diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ed3b3fd947b25e..e851f245a1cb63 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,4 +14,148 @@ # See the License for the specific language governing permissions and # limitations under the License. # -* @yiguolei + +# for more syntax help and usage example +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax + +be/src/agent/be_exec_version_manager.cpp @BiteTheDDDDt @zclllyybb +fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman +**/pom.xml @CalvinKirs @morningman +fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @dataroaring @morningman @yiguolei @xiaokang +fe/fe-core/src/main/java/org/apache/doris/fs @CalvinKirs +fe/fe-core/src/main/java/org/apache/doris/fsv2 @CalvinKirs +be/src/vec/functions @zclllyybb +be/**/CMakeLists.txt @zclllyybb @BiteTheDDDDt +be/src/olap/rowset/segment_v2/variant @eldenmoon @csun5285 +be/src/storage/index/inverted/ @airborne12 @eldenmoon @csun5285 +be/src/exprs/function/function_search.h @airborne12 @eldenmoon @csun5285 +be/src/exprs/function/function_search.cpp @airborne12 @eldenmoon @csun5285 +be/src/exprs/function/match.cpp @airborne12 @eldenmoon @csun5285 +be/src/exprs/function/match.h @airborne12 @eldenmoon @csun5285 +be/src/exprs/vsearch.h @airborne12 @eldenmoon @csun5285 +be/src/exprs/vsearch.cpp @airborne12 @eldenmoon @csun5285 +.claude/ @zclllyybb +AGENTS.md @zclllyybb + +be/src/cloud @liaoxin01 @luwei16 @gavinchou +be/src/io @liaoxin01 @gavinchou @morningman @Gabriel39 +be/src/io/cache @liaoxin01 @gavinchou @morningman @Gabriel39 +be/src/io/fs @liaoxin01 @gavinchou @morningman @Gabriel39 +be/src/io/tools @liaoxin01 @gavinchou +be/src/format_v2 @Gabriel39 @yiguolei +be/src/load @liaoxin01 @gavinchou +be/src/load/channel @liaoxin01 @gavinchou +be/src/load/delta_writer @liaoxin01 @gavinchou +be/src/load/group_commit @liaoxin01 @gavinchou +be/src/load/memtable @liaoxin01 @gavinchou +be/src/load/routine_load @liaoxin01 @gavinchou +be/src/load/stream_load @liaoxin01 @gavinchou +be/src/storage @gavinchou +be/src/storage/olap_common.h @gavinchou @yiguolei +be/src/storage/cache @liaoxin01 @gavinchou +be/src/storage/compaction @luwei16 @gavinchou +be/src/storage/delete @luwei16 @gavinchou +be/src/storage/index @airborne12 @eldenmoon @csun5285 +be/src/storage/iterator @yiguolei @gavinchou +be/src/storage/predicate @yiguolei @gavinchou +be/src/storage/rowset @gavinchou +be/src/storage/schema_change @luwei16 @gavinchou +be/src/storage/segment @yiguolei @liaoxin01 @gavinchou @eldenmoon @csun5285 @airborne12 +be/src/storage/snapshot @deardeng @luwei16 @gavinchou +be/src/storage/tablet @gavinchou +be/src/storage/task @deardeng @gavinchou +be/src/storage/txn @liaoxin01 @gavinchou +be/test/cloud @luwei16 @liaoxin01 @gavinchou +be/test/io @liaoxin01 @gavinchou @morningman @Gabriel39 +be/test/io/cache @liaoxin01 @gavinchou @morningman @Gabriel39 +be/test/io/client @gavinchou +be/test/io/fs @luwei16 @liaoxin01 @gavinchou @morningman @Gabriel39 +be/test/load @liaoxin01 @gavinchou +be/test/load/channel @liaoxin01 @gavinchou +be/test/load/delta_writer @liaoxin01 @gavinchou +be/test/load/memtable @liaoxin01 @gavinchou +be/test/olap @gavinchou +be/test/olap/rowset @gavinchou +be/test/storage @gavinchou +be/test/storage/cache @liaoxin01 @gavinchou +be/test/storage/compaction @luwei16 @gavinchou +be/test/storage/delete @gavinchou +be/test/storage/index @airborne12 @eldenmoon @csun5285 +be/test/storage/iterator @yiguolei @gavinchou +be/test/storage/predicate @yiguolei @gavinchou +be/test/storage/rowset @gavinchou +be/test/storage/schema_change @luwei16 @gavinchou +be/test/storage/segment @yiguolei @gavinchou @eldenmoon @csun5285 @airborne12 +be/test/storage/snapshot @deardeng @luwei16 @gavinchou +be/test/storage/tablet @gavinchou +be/test/storage/txn @liaoxin01 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/alter @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/backup @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/binlog @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/clone @deardeng @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud @liaoxin01 @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/alter @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/backup @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/catalog @morningman @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/common @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/datasource @morningman @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/load @liaoxin01 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/master @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/persist @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/qe @morrySnow @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/rpc @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/snapshot @luwei16 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/stage @liaoxin01 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/storage @luwei16 @liaoxin01 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/system @deardeng @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cloud/transaction @mymeiyi @liaoxin01 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/cooldown @liaoxin01 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/fs @calvinkirs @morningman @liaoxin01 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/ha @mymeiyi @gavinchou +fe/fe-core/src/main/java/org/apache/doris/journal @mymeiyi @gavinchou +fe/fe-core/src/main/java/org/apache/doris/load @liaoxin01 @gavinchou +fe/fe-core/src/main/java/org/apache/doris/master @mymeiyi @deardeng @gavinchou +fe/fe-core/src/main/java/org/apache/doris/mtmv @morrySnow @seawinde +fe/fe-core/src/main/java/org/apache/doris/nereids @morrySnow @924060929 @englefly @starocean999 +fe/fe-core/src/main/java/org/apache/doris/persist @mymeiyi @gavinchou +fe/fe-core/src/main/java/org/apache/doris/persist/gson @mymeiyi @gavinchou +fe/fe-core/src/main/java/org/apache/doris/persist/meta @mymeiyi @gavinchou +fe/fe-core/src/test/java/org/apache/doris/alter @luwei16 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/backup @luwei16 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/binlog @luwei16 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/clone @deardeng @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/backup @luwei16 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/cache @liaoxin01 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/catalog @morningman @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/common @liaoxin01 @luwei16 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/load @liaoxin01 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/master @deardeng @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/rpc @luwei16 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/stage @liaoxin01 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/storage @liaoxin01 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/system @deardeng @gavinchou +fe/fe-core/src/test/java/org/apache/doris/cloud/transaction @liaoxin01 @gavinchou +fe/fe-core/src/test/java/org/apache/doris/fs @calvinkirs @morningman @gavinchou @Gabriel39 +fe/fe-core/src/test/java/org/apache/doris/journal @mymeiyi @gavinchou +fe/fe-core/src/test/java/org/apache/doris/load @liaoxin01 @gavinchou +fe/fe-filesystem @calvinkirs @morningman @gavinchou +fe/fe-sql-parser/src/main/antlr4 @morrySnow @starocean999 @924060929 @englefly +cloud @gavinchou +cloud/cmake @liaoxin01 @luwei16 @gavinchou +cloud/conf @liaoxin01 @luwei16 @gavinchou +cloud/script @gavinchou @luwei16 +cloud/src @gavinchou +cloud/src/common @liaoxin01 @luwei16 @gavinchou +cloud/src/gen-cpp @liaoxin01 @luwei16 @gavinchou +cloud/src/meta-service @liaoxin01 @luwei16 @gavinchou +cloud/src/meta-store @luwei16 @gavinchou +cloud/src/rate-limiter @luwei16 @gavinchou +cloud/src/recycler @liaoxin01 @gavinchou +cloud/src/resource-manager @deardeng @gavinchou +cloud/src/snapshot @luwei16 @gavinchou +cloud/test @liaoxin01 @luwei16 @gavinchou +gensrc/proto/cloud.proto @luwei16 @gavinchou +gensrc/proto/file_cache.proto @liaoxin01 @gavinchou +gensrc/proto/olap_file.proto @yiguolei @liaoxin01 @gavinchou +gensrc/proto/segment_v2.proto @yiguolei @liaoxin01 @gavinchou @eldenmoon @csun5285 @airborne12 diff --git a/.gitignore b/.gitignore index 91c21130cd8bee..f6ba9c52f7091f 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,4 @@ compile_commands.json .github .worktrees/ +.worktree_initialized diff --git a/be/src/common/check.cpp b/be/src/common/check.cpp new file mode 100644 index 00000000000000..99408738af1dfe --- /dev/null +++ b/be/src/common/check.cpp @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/check.h" + +#include "common/exception.h" +#include "common/status.h" + +namespace doris { + +void doris_check_fail(std::string_view message) { + throw Exception(Status::FatalError("{}", message)); +} + +} // namespace doris diff --git a/be/src/common/check.h b/be/src/common/check.h new file mode 100644 index 00000000000000..1b4e6d51992946 --- /dev/null +++ b/be/src/common/check.h @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace doris { + +[[noreturn]] void doris_check_fail(std::string_view message); + +namespace detail { +template +concept OstreamPrintable = requires(std::ostream& os, const T& value) { os << value; }; + +class DorisCheckMessage { +public: + explicit DorisCheckMessage(std::string_view message) { _stream << message; } + + template + DorisCheckMessage& operator<<(const T& value) { + _stream << value; + return *this; + } + + DorisCheckMessage& operator<<(std::ostream& (*func)(std::ostream&)) { + func(_stream); + return *this; + } + + DorisCheckMessage& operator<<(std::ios& (*func)(std::ios&)) { + func(_stream); + return *this; + } + + DorisCheckMessage& operator<<(std::ios_base& (*func)(std::ios_base&)) { + func(_stream); + return *this; + } + + [[noreturn]] void fail() { doris_check_fail(_stream.str()); } + +private: + std::ostringstream _stream; +}; + +class DorisCheckMessageVoidify { +public: + [[noreturn]] void operator&(DorisCheckMessage& message) const { message.fail(); } + [[noreturn]] void operator&(DorisCheckMessage&& message) const { message.fail(); } +}; + +class DorisCheckResult { +public: + explicit DorisCheckResult(bool ok) : _ok(ok) {} + explicit DorisCheckResult(std::string message) : _ok(false), _message(std::move(message)) {} + + bool ok() const { return _ok; } + const std::string& message() const { return _message; } + +private: + bool _ok; + std::string _message; +}; + +template +std::string doris_check_value_to_string(const T& value) { + if constexpr (std::is_same_v, std::nullptr_t>) { + return "nullptr"; + } else if constexpr (std::is_same_v, bool>) { + return value ? "true" : "false"; + } else if constexpr (OstreamPrintable) { + std::ostringstream oss; + oss << std::boolalpha << value; + return oss.str(); + } else { + return ""; + } +} + +template +DorisCheckResult doris_check_binary_op_result(const Lhs& lhs, const Rhs& rhs, + std::string_view lhs_expr, std::string_view rhs_expr, + std::string_view op_expr, Comparator comparator) { + if (static_cast(comparator(lhs, rhs))) { + return DorisCheckResult(true); + } + return DorisCheckResult(fmt::format("Check failed: {} {} {} ({} vs {})", lhs_expr, op_expr, + rhs_expr, doris_check_value_to_string(lhs), + doris_check_value_to_string(rhs))); +} +} // namespace detail + +} // namespace doris + +// core in Debug mode, exception in Release mode. +#define DORIS_CHECK(stmt) \ + if (bool _doris_check_ok = static_cast(stmt); _doris_check_ok) { \ + } else [[unlikely]] \ + ::doris::detail::DorisCheckMessageVoidify() & \ + ::doris::detail::DorisCheckMessage("Check failed: " #stmt) + +// Use DORIS_CHECK_* only for invariants that must also be checked in Release builds. +// Keep DCHECK_* in loops or other hot paths where Release checks would add overhead. +#ifndef NDEBUG +#define DORIS_CHECK_EQ(val1, val2) DCHECK_EQ(val1, val2) +#define DORIS_CHECK_NE(val1, val2) DCHECK_NE(val1, val2) +#define DORIS_CHECK_LT(val1, val2) DCHECK_LT(val1, val2) +#define DORIS_CHECK_LE(val1, val2) DCHECK_LE(val1, val2) +#define DORIS_CHECK_GT(val1, val2) DCHECK_GT(val1, val2) +#define DORIS_CHECK_GE(val1, val2) DCHECK_GE(val1, val2) +#else +#define DORIS_CHECK_BINARY_OP(val1, val2, op, op_str) \ + if (auto _doris_check_result = ::doris::detail::doris_check_binary_op_result( \ + (val1), (val2), #val1, #val2, op_str, \ + [](const auto& _doris_check_lhs, const auto& _doris_check_rhs) { \ + return _doris_check_lhs op _doris_check_rhs; \ + }); \ + _doris_check_result.ok()) { \ + } else [[unlikely]] \ + ::doris::detail::DorisCheckMessageVoidify() & \ + ::doris::detail::DorisCheckMessage(_doris_check_result.message()) + +#define DORIS_CHECK_EQ(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, ==, "==") +#define DORIS_CHECK_NE(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, !=, "!=") +#define DORIS_CHECK_LT(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, <, "<") +#define DORIS_CHECK_LE(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, <=, "<=") +#define DORIS_CHECK_GT(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, >, ">") +#define DORIS_CHECK_GE(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, >=, ">=") +#endif diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5c390c7de29fbb..59dffdf1ac34e3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1050,9 +1050,8 @@ DEFINE_mInt32(merged_hdfs_min_io_size, "8192"); // OrcReader DEFINE_mInt32(orc_natural_read_size_mb, "8"); -DEFINE_mInt64(big_column_size_buffer, "65535"); -DEFINE_mInt64(small_column_size_buffer, "100"); - +DEFINE_Validator(orc_natural_read_size_mb, + [](const int config) -> bool { return config > 0 && config <= 1024; }); // Perform the always_true check at intervals determined by runtime_filter_sampling_frequency DEFINE_mInt32(runtime_filter_sampling_frequency, "32"); DEFINE_mInt32(execution_max_rpc_timeout_sec, "3600"); diff --git a/be/src/common/status.h b/be/src/common/status.h index 62521ddee5cb2a..abbc87aa585d56 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -17,6 +17,7 @@ #include #include +#include "common/check.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" #include "common/expected.h" @@ -770,13 +771,6 @@ using ResultError = unexpected; }) // core in Debug mode, exception in Release mode. -#define DORIS_CHECK(stmt) \ - do { \ - if (!static_cast(stmt)) [[unlikely]] { \ - throw Exception(Status::FatalError(fmt::format("Check failed: {}", #stmt))); \ - } \ - } while (false) - } // namespace doris // specify formatter for Status diff --git a/be/src/core/data_type/data_type_timestamptz.h b/be/src/core/data_type/data_type_timestamptz.h index 4a3fba0616cc45..b386402cb49696 100644 --- a/be/src/core/data_type/data_type_timestamptz.h +++ b/be/src/core/data_type/data_type_timestamptz.h @@ -56,6 +56,10 @@ class DataTypeTimeStampTz final : public DataTypeNumberBaseset_scale(_scale); + } + void to_pb_column_meta(PColumnMeta* col_meta) const override { DataTypeNumberBase::to_pb_column_meta(col_meta); col_meta->mutable_decimal_param()->set_scale(_scale); diff --git a/be/src/core/data_type_serde/data_type_array_serde.h b/be/src/core/data_type_serde/data_type_array_serde.h index ceb97b7d57f4a0..4a74649336e586 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.h +++ b/be/src/core/data_type_serde/data_type_array_serde.h @@ -94,6 +94,7 @@ class DataTypeArraySerDe : public DataTypeSerDe { const cctz::time_zone& ctz) const override; Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 7a0dfd163c70c6..999046503fb9c8 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -28,6 +28,7 @@ #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" +#include "core/data_type_serde/decoded_column_view.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" @@ -44,6 +45,95 @@ namespace doris { static const int64_t micro_to_nano_second = 1000; #include "common/compile_check_begin.h" +namespace { + +#pragma pack(1) +struct DecodedInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; + + int64_t to_timestamp_micros() const { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_IN_DAY = 86400000000; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + return (julian_day - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + + nanos_of_day / NANOS_PER_MICROSECOND; + } +}; +#pragma pack() +static_assert(sizeof(DecodedInt96Timestamp) == 12); + +Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, + int64_t timestamp_micros) { + static constexpr int64_t MICROS_PER_SECOND = 1000000; + static constexpr int64_t MICROS_PER_MINUTE = MICROS_PER_SECOND * 60; + static constexpr int64_t MICROS_PER_HOUR = MICROS_PER_MINUTE * 60; + static constexpr int64_t MICROS_PER_DAY = MICROS_PER_HOUR * 24; + static const int64_t EPOCH_DAYNR = calc_daynr(1970, 1, 1); + + int64_t days_since_epoch = timestamp_micros / MICROS_PER_DAY; + int64_t micros_of_day = timestamp_micros % MICROS_PER_DAY; + if (micros_of_day < 0) { + micros_of_day += MICROS_PER_DAY; + --days_since_epoch; + } + + const int64_t daynr = EPOCH_DAYNR + days_since_epoch; + if (daynr <= 0) { + return Status::DataQualityError( + "Decoded DATETIMEV2 timestamp is out of range: micros={}, daynr={}", + timestamp_micros, daynr); + } + + DateV2Value datetime_value; + if (!datetime_value.get_date_from_daynr(static_cast(daynr))) { + return Status::DataQualityError( + "Decoded DATETIMEV2 timestamp is out of range: micros={}, daynr={}", + timestamp_micros, daynr); + } + + const auto hour = static_cast(micros_of_day / MICROS_PER_HOUR); + micros_of_day %= MICROS_PER_HOUR; + const auto minute = static_cast(micros_of_day / MICROS_PER_MINUTE); + micros_of_day %= MICROS_PER_MINUTE; + const auto second = static_cast(micros_of_day / MICROS_PER_SECOND); + const auto microsecond = static_cast(micros_of_day % MICROS_PER_SECOND); + datetime_value.unchecked_set_time(datetime_value.year(), datetime_value.month(), + datetime_value.day(), hour, minute, second, microsecond); + data.push_back(datetime_value); + return Status::OK(); +} + +void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, + int64_t timestamp_micros, + const cctz::time_zone& timezone) { + static constexpr int64_t MICROS_PER_SECOND = 1000000; + + int64_t epoch_seconds = timestamp_micros / MICROS_PER_SECOND; + int64_t micros_of_second = timestamp_micros % MICROS_PER_SECOND; + if (micros_of_second < 0) { + micros_of_second += MICROS_PER_SECOND; + --epoch_seconds; + } + + DateV2Value datetime_value; + datetime_value.from_unixtime(epoch_seconds, timezone); + datetime_value.set_microsecond(static_cast(micros_of_second)); + data.push_back(datetime_value); +} + +int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { + if (view.time_unit == DecodedTimeUnit::MILLIS) { + return value * 1000; + } + if (view.time_unit == DecodedTimeUnit::NANOS) { + return value / 1000; + } + return value; +} + +} // namespace + // NOLINTBEGIN(readability-function-size) // NOLINTBEGIN(readability-function-cognitive-complexity) Status DataTypeDateTimeV2SerDe::from_string_batch(const ColumnString& col_str, @@ -453,6 +543,59 @@ Status DataTypeDateTimeV2SerDe::read_column_from_arrow(IColumn& column, return Status::OK(); } +Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( + IColumn& column, const DecodedColumnView& view) const { + if (view.value_kind != DecodedValueKind::INT64 && view.value_kind != DecodedValueKind::INT96) { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("DATETIMEV2 decoded reader expects INT64 or INT96 source")); + } + if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); + } + auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); + if (view.value_kind == DecodedValueKind::INT96) { + const auto* values = reinterpret_cast(view.values); + static const auto utc_timezone = cctz::utc_time_zone(); + const auto& timezone = view.timezone == nullptr ? utc_timezone : *view.timezone; + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(DateV2Value()); + continue; + } + append_datetimev2_from_utc_epoch_micros(data, values[row].to_timestamp_micros(), + timezone); + } + return Status::OK(); + } + + const auto* values = reinterpret_cast(view.values); + static const auto utc_timezone = cctz::utc_time_zone(); + const auto& timezone = view.timezone == nullptr ? utc_timezone : *view.timezone; + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(DateV2Value()); + continue; + } + const int64_t timestamp_micros = decoded_timestamp_micros(view, values[row]); + if (view.timestamp_is_adjusted_to_utc) { + append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); + } else { + auto st = append_datetimev2_from_epoch_micros(data, timestamp_micros); + if (!st.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return st; + } + } + } + return Status::OK(); +} + Status DataTypeDateTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.h b/be/src/core/data_type_serde/data_type_datetimev2_serde.h index 0389432a621730..e089d789ccee7d 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.h @@ -88,6 +88,9 @@ class DataTypeDateTimeV2SerDe : public DataTypeNumberSerDe(column).get_data(); + const auto* values = reinterpret_cast(view.values); + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(DateV2Value()); + continue; + } + DateV2Value date_v2; + date_v2.get_date_from_daynr(values[row] + date_threshold); + data.push_back(date_v2); + } + return Status::OK(); +} + Status DataTypeDateV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.h b/be/src/core/data_type_serde/data_type_datev2_serde.h index 0375f9be4b4b23..bc02b61b520193 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datev2_serde.h @@ -86,6 +86,9 @@ class DataTypeDateV2SerDe : public DataTypeNumberSerDe +NativeType decode_big_endian_signed_integer(const uint8_t* data, int length) { + if constexpr (std::is_same_v) { + NativeType value = data != nullptr && length > 0 && (data[0] & 0x80) != 0 ? NativeType(-1) + : NativeType(0); + for (int i = 0; i < length; ++i) { + value = (value << 8) + NativeType(data[i]); + } + return value; + } else { + using UnsignedNativeType = + std::conditional_t, unsigned __int128, + std::make_unsigned_t>; + UnsignedNativeType value = data != nullptr && length > 0 && (data[0] & 0x80) != 0 + ? static_cast(-1) + : 0; + for (int i = 0; i < length; ++i) { + value = static_cast((value << 8) | data[i]); + } + return static_cast(value); + } +} + +template +bool decoded_decimal_value_fits(const typename PrimitiveTypeTraits::CppType::NativeType& value, + UInt32 precision) { + return value >= min_decimal_value(precision).value && + value <= max_decimal_value(precision).value; +} + +template +bool decoded_decimal_int_value_fits(Int128 value, UInt32 precision) { + using NativeType = typename PrimitiveTypeTraits::CppType::NativeType; + if constexpr (std::is_same_v) { + const auto wide_value = wide::Int256(value); + return decoded_decimal_value_fits(wide_value, precision); + } else { + return value >= static_cast(min_decimal_value(precision).value) && + value <= static_cast(max_decimal_value(precision).value); + } +} + +template +Status read_decimal_decoded_value(const DecodedColumnView& view, UInt32 precision, int64_t row, + typename PrimitiveTypeTraits::CppType* result) { + using FieldType = typename PrimitiveTypeTraits::CppType; + using NativeType = typename FieldType::NativeType; + NativeType native_value; + if (view.value_kind == DecodedValueKind::INT32) { + const auto* values = reinterpret_cast(view.values); + const auto value = static_cast(values[row]); + if (!decoded_decimal_int_value_fits(value, precision)) { + return Status::DataQualityError("Decoded decimal value is out of range"); + } + native_value = NativeType(value); + } else if (view.value_kind == DecodedValueKind::INT64) { + const auto* values = reinterpret_cast(view.values); + const auto value = static_cast(values[row]); + if (!decoded_decimal_int_value_fits(value, precision)) { + return Status::DataQualityError("Decoded decimal value is out of range"); + } + native_value = NativeType(value); + } else { + const auto& value = (*view.binary_values)[row]; + const auto length = view.value_kind == DecodedValueKind::FIXED_BINARY + ? view.fixed_length + : cast_set(value.size); + if (length > static_cast(sizeof(NativeType))) { + return Status::DataQualityError("Decoded decimal binary value is too wide: length={}", + length); + } + native_value = decode_big_endian_signed_integer( + reinterpret_cast(value.data), length); + } + if (!decoded_decimal_value_fits(native_value, precision)) { + return Status::DataQualityError("Decoded decimal value is out of range"); + } + *result = FieldType {native_value}; + return Status::OK(); +} + +template +Status read_decimal_decoded_values(IColumn& column, const DecodedColumnView& view, + UInt32 precision) { + if (view.value_kind == DecodedValueKind::INT32 || view.value_kind == DecodedValueKind::INT64) { + if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); + } + } else if (view.binary_values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded binary values are null for {}", column.get_name()); + } + auto& data = assert_cast&>(column).get_data(); + const auto old_size = data.size(); + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(typename PrimitiveTypeTraits::CppType()); + continue; + } + if (view.value_kind == DecodedValueKind::BINARY || + view.value_kind == DecodedValueKind::FIXED_BINARY) { + const auto& value = (*view.binary_values)[row]; + const auto length = view.value_kind == DecodedValueKind::FIXED_BINARY + ? view.fixed_length + : cast_set(value.size); + if (value.data == nullptr && length > 0) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + return Status::Corruption("Decoded decimal binary value is null for {} at row {}", + column.get_name(), row); + } + } + typename PrimitiveTypeTraits::CppType value; + auto st = read_decimal_decoded_value(view, precision, row, &value); + if (!st.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + st.prepend(fmt::format( + "Failed to decode decimal value for {} at row {}: ", column.get_name(), row)); + return st; + } + data.push_back(value); + } + return Status::OK(); +} + +} // namespace template Status DataTypeDecimalSerDe::from_string_batch(const ColumnString& str, ColumnNullable& column, @@ -373,6 +506,24 @@ Status DataTypeDecimalSerDe::read_column_from_arrow(IColumn& column, return Status::OK(); } +template +Status DataTypeDecimalSerDe::read_column_from_decoded_values( + IColumn& column, const DecodedColumnView& view) const { + if constexpr (T == TYPE_DECIMAL32 || T == TYPE_DECIMAL64 || T == TYPE_DECIMAL128I || + T == TYPE_DECIMAL256) { + if (view.value_kind == DecodedValueKind::INT32 || + view.value_kind == DecodedValueKind::INT64 || + view.value_kind == DecodedValueKind::BINARY || + view.value_kind == DecodedValueKind::FIXED_BINARY) { + return read_decimal_decoded_values(column, view, precision); + } + } + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("Unsupported decoded values for {} from source kind {}", + get_name(), static_cast(view.value_kind))); +} + template Status DataTypeDecimalSerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 140c8e3a292a21..9f8fd58799b753 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -108,6 +108,9 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { const cctz::time_zone& ctz) const override; Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const override; + Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/data_type_map_serde.h b/be/src/core/data_type_serde/data_type_map_serde.h index b4aa88f3cbec1f..478fb155afbbbf 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.h +++ b/be/src/core/data_type_serde/data_type_map_serde.h @@ -85,6 +85,7 @@ class DataTypeMapSerDe : public DataTypeSerDe { const cctz::time_zone& ctz) const override; Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index 6ca2e07b7b0188..b805467bf15a9b 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -22,7 +22,7 @@ #include #include -#include +#include #include "core/assert_cast.h" #include "core/column/column.h" @@ -31,10 +31,12 @@ #include "core/column/column_vector.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/data_type_string_serde.h" +#include "core/data_type_serde/decoded_column_view.h" #include "exprs/function/cast/cast_base.h" #include "format/transformer/vcsv_transformer.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" +#include "util/simd/bits.h" namespace doris { class Arena; @@ -352,6 +354,39 @@ Status DataTypeNullableSerDe::read_column_from_arrow(IColumn& column, ctz); } +Status DataTypeNullableSerDe::read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const { + auto& nullable_column = assert_cast(column); + auto& null_map = nullable_column.get_null_map_data(); + const auto old_size = null_map.size(); + auto& nested_column = nullable_column.get_nested_column(); + const auto old_nested_size = nested_column.size(); + null_map.resize(null_map.size() + view.row_count); + if (view.null_map == nullptr) { + // No null value + memset(null_map.data() + old_size, 0, view.row_count); + } else { + // TODO: skip if no null in map + auto* dst = null_map.data() + old_size; + memcpy(dst, view.null_map, view.row_count); + // If there are all null values, we can skip reading nested column and just insert defaults. + if (simd::count_zero_num(reinterpret_cast(view.null_map), view.row_count) == + 0) { + nested_column.insert_many_defaults(view.row_count); + return Status::OK(); + } + } + DecodedColumnView nested_view = view; + nested_view.conversion_failure_null_map = &null_map; + nested_view.conversion_failure_null_map_offset = old_size; + auto st = nested_serde->read_column_from_decoded_values(nested_column, nested_view); + if (!st.ok()) { + null_map.resize(old_size); + nested_column.resize(old_nested_size); + } + return st; +} + bool DataTypeNullableSerDe::write_column_to_mysql_text(const IColumn& column, BufferWritable& bw, int64_t row_idx, const FormatOptions& options) const { diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.h b/be/src/core/data_type_serde/data_type_nullable_serde.h index 31ad0e0c0413e7..49639a2a149517 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.h +++ b/be/src/core/data_type_serde/data_type_nullable_serde.h @@ -87,6 +87,9 @@ class DataTypeNullableSerDe : public DataTypeSerDe { const cctz::time_zone& ctz) const override; Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const override; + Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index f82b33769c58db..8a77894bfb1039 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -20,6 +20,8 @@ #include #include +#include +#include #include "common/exception.h" #include "common/status.h" @@ -27,6 +29,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/decoded_column_view.h" #include "core/packed_int128.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -42,8 +45,138 @@ #include "util/to_string.h" namespace doris { -#include "common/compile_check_begin.h" -// Type map的基本结构 +namespace { + +template +const NativeType* decoded_values_as(const DecodedColumnView& view) { + return reinterpret_cast(view.values); +} + +template +bool decoded_number_value_fits(SourceType value) { + if constexpr (std::is_floating_point_v) { + return true; + } else if constexpr (std::is_same_v) { + return value == SourceType(0) || value == SourceType(1); + } else if constexpr (std::is_signed_v) { + const auto int128_value = static_cast(value); + return int128_value >= static_cast(std::numeric_limits::lowest()) && + int128_value <= static_cast(std::numeric_limits::max()); + } else { + const auto uint128_value = static_cast(value); + if constexpr (std::is_signed_v) { + return uint128_value <= + static_cast(std::numeric_limits::max()); + } else { + return uint128_value <= + static_cast(std::numeric_limits::max()); + } + } +} + +template +Status read_number_decoded_values(IColumn& column, const DecodedColumnView& view) { + if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); + } + auto& data = + assert_cast::ColumnType&>(column).get_data(); + const auto old_size = data.size(); + const auto* values = decoded_values_as(view); + for (int64_t row = 0; row < view.row_count; ++row) { + using DorisCppType = typename PrimitiveTypeTraits::CppType; + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(DorisCppType()); + continue; + } + if (!decoded_number_value_fits(values[row])) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Decoded value is out of range for {} at row {}", + column.get_name(), row); + } + data.push_back(static_cast(values[row])); + } + return Status::OK(); +} + +template +Status read_logical_integer_decoded_values_as(IColumn& column, const DecodedColumnView& view) { + if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); + } + auto& data = + assert_cast::ColumnType&>(column).get_data(); + const auto old_size = data.size(); + const auto* values = decoded_values_as(view); + for (int64_t row = 0; row < view.row_count; ++row) { + using DorisCppType = typename PrimitiveTypeTraits::CppType; + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(DorisCppType()); + continue; + } + const auto logical_value = static_cast(values[row]); + if (!decoded_number_value_fits(logical_value)) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return Status::DataQualityError( + "Decoded logical integer value is out of range for {} at row {}", + column.get_name(), row); + } + data.push_back(static_cast(logical_value)); + } + return Status::OK(); +} + +template +Status read_integer_decoded_values(IColumn& column, const DecodedColumnView& view) { + if (view.logical_integer_bit_width <= 0) { + return read_number_decoded_values(column, view); + } + + if (view.logical_integer_is_signed) { + switch (view.logical_integer_bit_width) { + case 8: + return read_logical_integer_decoded_values_as(column, + view); + case 16: + return read_logical_integer_decoded_values_as(column, + view); + case 32: + return read_logical_integer_decoded_values_as(column, + view); + case 64: + return read_logical_integer_decoded_values_as(column, + view); + default: + return Status::NotSupported("Unsupported decoded logical integer bit width {} for {}", + view.logical_integer_bit_width, column.get_name()); + } + } + + switch (view.logical_integer_bit_width) { + case 8: + return read_logical_integer_decoded_values_as(column, view); + case 16: + return read_logical_integer_decoded_values_as(column, view); + case 32: + return read_logical_integer_decoded_values_as(column, view); + case 64: + return read_logical_integer_decoded_values_as(column, view); + default: + return Status::NotSupported("Unsupported decoded logical integer bit width {} for {}", + view.logical_integer_bit_width, column.get_name()); + } +} + +} // namespace +// Basic structure of the type map. template struct TypeMap { using KeyType = Key; @@ -51,21 +184,21 @@ struct TypeMap { using Next = TypeMap; }; -// Type map的末端 +// End marker of the type map. template <> struct TypeMap {}; -// TypeMapLookup 前向声明 +// Forward declaration of TypeMapLookup. template struct TypeMapLookup; -// Type map查找：找到匹配的键时的情况 +// Type map lookup when the key matches. template struct TypeMapLookup> { using ValueType = Value; }; -// Type map查找：递归查找 +// Type map lookup by recursive search. template struct TypeMapLookup> { using ValueType = typename TypeMapLookup>::ValueType; @@ -157,6 +290,42 @@ Status DataTypeNumberSerDe::write_column_to_arrow(const IColumn& column, cons return Status::OK(); } +template +Status DataTypeNumberSerDe::read_column_from_decoded_values( + IColumn& column, const DecodedColumnView& view) const { + if constexpr (T == TYPE_BOOLEAN) { + if (view.value_kind == DecodedValueKind::BOOL) { + return read_number_decoded_values(column, view); + } + } else if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT) { + if (view.value_kind == DecodedValueKind::INT32) { + return read_integer_decoded_values(column, view); + } + if (view.value_kind == DecodedValueKind::UINT32) { + return read_integer_decoded_values(column, view); + } + if (view.value_kind == DecodedValueKind::INT64) { + return read_integer_decoded_values(column, view); + } + if (view.value_kind == DecodedValueKind::UINT64) { + return read_integer_decoded_values(column, view); + } + } else if constexpr (T == TYPE_FLOAT) { + if (view.value_kind == DecodedValueKind::FLOAT) { + return read_number_decoded_values(column, view); + } + } else if constexpr (T == TYPE_DOUBLE) { + if (view.value_kind == DecodedValueKind::DOUBLE) { + return read_number_decoded_values(column, view); + } + } + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("Unsupported decoded values for {} from source kind {}", + get_name(), static_cast(view.value_kind))); +} + template Status DataTypeNumberSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const { diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index cc33fe2b684f2d..16c4ccd48e5c8a 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -118,6 +118,10 @@ class DataTypeNumberSerDe : public DataTypeSerDe { Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const override; + Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index 5fa8fa2fab8a10..ff3f3c450244d0 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -16,25 +16,923 @@ // under the License. #include "core/data_type_serde/data_type_serde.h" +#include + +#include +#include +#include +#include +#include +#include + #include "common/cast_set.h" +#include "common/check.h" +#include "common/consts.h" #include "common/exception.h" #include "common/status.h" +#include "core/assert_cast.h" #include "core/column/column.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" #include "core/data_type_serde/data_type_array_serde.h" +#include "core/data_type_serde/data_type_datetimev2_serde.h" +#include "core/data_type_serde/data_type_datev2_serde.h" #include "core/data_type_serde/data_type_decimal_serde.h" #include "core/data_type_serde/data_type_jsonb_serde.h" +#include "core/data_type_serde/data_type_nullable_serde.h" #include "core/data_type_serde/data_type_number_serde.h" #include "core/data_type_serde/data_type_string_serde.h" +#include "core/data_type_serde/data_type_timestamptz_serde.h" #include "core/field.h" +#include "core/types.h" +#include "core/value/timestamptz_value.h" +#include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_base.h" #include "runtime/descriptors.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" namespace doris { -#include "common/compile_check_begin.h" +namespace { + +constexpr int DECIMAL_PRECISION_FOR_HIVE11 = BeConsts::MAX_DECIMAL128_PRECISION; +constexpr int32_t DORIS_DATE_EPOCH_DAYNR = 719528; + +bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row) { + return batch.hasNulls && !batch.notNull[row]; +} + +size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows) { + if (selected_rows == nullptr) { + return rows; + } + return selected_rows->size(); +} + +size_t orc_source_row_at(size_t row, const std::vector* selected_rows) { + if (selected_rows == nullptr) { + return row; + } + return (*selected_rows)[row]; +} + +DecodedColumnView make_orc_decoded_view(const OrcDecodedColumnView& orc_view, + DecodedValueKind value_kind) { + DecodedColumnView view; + view.value_kind = value_kind; + view.row_count = cast_set(orc_decode_row_count(orc_view.rows, orc_view.selected_rows)); + view.timezone = orc_view.timezone; + return view; +} + +void fill_orc_decoded_null_map(const ::orc::ColumnVectorBatch& batch, size_t rows, + const std::vector* selected_rows, NullMap* null_map) { + DORIS_CHECK(null_map != nullptr); + if (!batch.hasNulls) { + return; + } + const auto output_rows = orc_decode_row_count(rows, selected_rows); + null_map->resize(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + (*null_map)[row] = !batch.notNull[orc_source_row_at(row, selected_rows)]; + } +} + +void append_orc_null_map(const ::orc::ColumnVectorBatch& batch, size_t rows, + const std::vector* selected_rows, NullMap* null_map) { + DORIS_CHECK(null_map != nullptr); + const auto output_rows = orc_decode_row_count(rows, selected_rows); + const auto old_size = null_map->size(); + null_map->resize(old_size + output_rows); + if (batch.hasNulls) { + for (size_t row = 0; row < output_rows; ++row) { + (*null_map)[old_size + row] = !batch.notNull[orc_source_row_at(row, selected_rows)]; + } + return; + } + std::memset(null_map->data() + old_size, 0, output_rows); +} + +size_t trim_right_spaces(const char* value, size_t length) { + while (length > 0 && value[length - 1] == ' ') { + --length; + } + return length; +} + +Status append_orc_string_ref(const ::orc::Type& file_type, const char* data, int64_t length, + std::vector& binary_values) { + if (length < 0) { + return Status::Corruption("Invalid negative ORC string length {}", length); + } + auto value_length = static_cast(length); + if (file_type.getKind() == ::orc::TypeKind::CHAR) { + value_length = trim_right_spaces(data, value_length); + } + binary_values.emplace_back(value_length == 0 ? "" : data, value_length); + return Status::OK(); +} + +Int128 to_int128(::orc::Int128 value) { + const auto high_bits = static_cast<__uint128_t>(static_cast(value.getHighBits())); + const auto low_bits = static_cast<__uint128_t>(value.getLowBits()); + return static_cast((high_bits << 64) | low_bits); +} + +::orc::Int128 to_orc_int128(Int128 value) { + const auto unsigned_value = static_cast<__uint128_t>(value); + return ::orc::Int128(static_cast(static_cast(unsigned_value >> 64)), + static_cast(unsigned_value)); +} + +Status scale_decimal_value(Int128 value, int32_t source_scale, int32_t target_scale, + Int128* scaled_value) { + DORIS_CHECK(scaled_value != nullptr); + if (source_scale == target_scale) { + *scaled_value = value; + return Status::OK(); + } + if (source_scale < target_scale) { + bool overflow = false; + const auto scaled = ::orc::scaleUpInt128ByPowerOfTen(to_orc_int128(value), + target_scale - source_scale, overflow); + if (overflow) { + return Status::DataQualityError( + "ORC decimal value overflows when scaling from {} to {}", source_scale, + target_scale); + } + *scaled_value = to_int128(scaled); + return Status::OK(); + } + *scaled_value = to_int128( + ::orc::scaleDownInt128ByPowerOfTen(to_orc_int128(value), source_scale - target_scale)); + return Status::OK(); +} + +void fill_decimal_big_endian_value(Int128 value, std::array* bytes) { + DORIS_CHECK(bytes != nullptr); + const auto unsigned_value = static_cast<__uint128_t>(value); + for (size_t byte_idx = 0; byte_idx < bytes->size(); ++byte_idx) { + const auto shift = (bytes->size() - byte_idx - 1) * 8; + (*bytes)[byte_idx] = static_cast(unsigned_value >> shift); + } +} + +Status read_decoded_values(const DataTypeSerDe& serde, IColumn& column, DecodedColumnView* view) { + DORIS_CHECK(view != nullptr); + RETURN_IF_ERROR(serde.read_column_from_decoded_values(column, *view)); + return Status::OK(); +} + +template +void fill_selected_values(const SourceType* source_values, size_t rows, + const std::vector* selected_rows, + std::vector* selected_values) { + DORIS_CHECK(source_values != nullptr); + DORIS_CHECK(selected_values != nullptr); + const auto output_rows = orc_decode_row_count(rows, selected_rows); + selected_values->resize(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + (*selected_values)[row] = source_values[orc_source_row_at(row, selected_rows)]; + } +} + +template +Status decode_fixed_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view, DecodedValueKind value_kind) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC scalar batch type {}", + orc_view.batch->toString()); + } + auto view = make_orc_decoded_view(orc_view, value_kind); + NullMap null_map; + fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + std::vector selected_values; + if (orc_view.selected_rows == nullptr) { + view.values = reinterpret_cast(orc_batch->data.data()); + } else { + fill_selected_values(orc_batch->data.data(), orc_view.rows, orc_view.selected_rows, + &selected_values); + view.values = reinterpret_cast(selected_values.data()); + } + RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); + return Status::OK(); +} + +Status decode_float_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC float batch type {}", + orc_view.batch->toString()); + } + auto view = make_orc_decoded_view(orc_view, DecodedValueKind::FLOAT); + NullMap null_map; + fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::vector float_values; + float_values.resize(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + float_values[row] = + static_cast(orc_batch->data[orc_source_row_at(row, orc_view.selected_rows)]); + } + view.values = reinterpret_cast(float_values.data()); + RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); + return Status::OK(); +} + +Status decode_boolean_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC boolean batch type {}", + orc_view.batch->toString()); + } + auto view = make_orc_decoded_view(orc_view, DecodedValueKind::BOOL); + NullMap null_map; + fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::unique_ptr bool_values = std::make_unique(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + bool_values[row] = orc_batch->data[orc_source_row_at(row, orc_view.selected_rows)] != 0; + } + view.values = reinterpret_cast(bool_values.get()); + RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); + return Status::OK(); +} + +Status decode_string_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view) { + DORIS_CHECK(orc_view.file_type != nullptr); + if (const auto* encoded_batch = + dynamic_cast(orc_view.batch); + encoded_batch != nullptr && encoded_batch->isEncoded) { + if (encoded_batch->dictionary == nullptr) { + return Status::InternalError("Encoded ORC string batch has no dictionary"); + } + auto view = make_orc_decoded_view(orc_view, DecodedValueKind::BINARY); + NullMap null_map; + fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, + &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::vector binary_values; + binary_values.reserve(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_source_row_at(row, orc_view.selected_rows); + if (orc_row_is_null(*orc_view.batch, source_row)) { + binary_values.emplace_back("", 0); + continue; + } + char* data = nullptr; + int64_t length = 0; + encoded_batch->dictionary->getValueByIndex(encoded_batch->index[source_row], data, + length); + RETURN_IF_ERROR( + append_orc_string_ref(*orc_view.file_type, data, length, binary_values)); + } + view.binary_values = &binary_values; + RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); + return Status::OK(); + } + + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC string batch type {}", + orc_view.batch->toString()); + } + auto view = make_orc_decoded_view(orc_view, DecodedValueKind::BINARY); + NullMap null_map; + fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::vector binary_values; + binary_values.reserve(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_source_row_at(row, orc_view.selected_rows); + if (orc_row_is_null(*orc_view.batch, source_row)) { + binary_values.emplace_back("", 0); + continue; + } + RETURN_IF_ERROR(append_orc_string_ref(*orc_view.file_type, orc_batch->data[source_row], + orc_batch->length[source_row], binary_values)); + } + view.binary_values = &binary_values; + RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); + return Status::OK(); +} + +Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC date batch type {}", + orc_view.batch->toString()); + } + auto view = make_orc_decoded_view(orc_view, DecodedValueKind::INT32); + NullMap null_map; + fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::vector date_values; + date_values.resize(output_rows); + auto& date_dict = date_day_offset_dict::get(); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_source_row_at(row, orc_view.selected_rows); + const auto date = date_dict[cast_set(orc_batch->data[source_row])]; + date_values[row] = cast_set(date.daynr() - DORIS_DATE_EPOCH_DAYNR); + } + view.values = reinterpret_cast(date_values.data()); + RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); + return Status::OK(); +} + +Status decode_decimal_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view, int32_t target_scale) { + DORIS_CHECK(orc_view.file_type != nullptr); + auto view = make_orc_decoded_view(orc_view, DecodedValueKind::FIXED_BINARY); + view.decimal_precision = orc_view.file_type->getPrecision() == 0 + ? DECIMAL_PRECISION_FOR_HIVE11 + : cast_set(orc_view.file_type->getPrecision()); + NullMap null_map; + fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + view.fixed_length = sizeof(Int128); + + std::vector binary_values; + std::vector> decimal_values; + const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + decimal_values.resize(output_rows); + binary_values.reserve(output_rows); + if (const auto* decimal64_batch = + dynamic_cast(orc_view.batch); + decimal64_batch != nullptr) { + view.decimal_scale = decimal64_batch->scale; + for (size_t row = 0; row < output_rows; ++row) { + Int128 value = 0; + const auto source_row = orc_source_row_at(row, orc_view.selected_rows); + if (!orc_row_is_null(*orc_view.batch, source_row)) { + RETURN_IF_ERROR(scale_decimal_value(decimal64_batch->values[source_row], + decimal64_batch->scale, target_scale, &value)); + } + fill_decimal_big_endian_value(value, &decimal_values[row]); + binary_values.emplace_back(reinterpret_cast(decimal_values[row].data()), + decimal_values[row].size()); + } + view.binary_values = &binary_values; + RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); + return Status::OK(); + } + + const auto* decimal128_batch = + dynamic_cast(orc_view.batch); + if (decimal128_batch == nullptr) { + return Status::InternalError("Unexpected ORC decimal batch type {}", + orc_view.batch->toString()); + } + view.decimal_scale = decimal128_batch->scale; + for (size_t row = 0; row < output_rows; ++row) { + Int128 value = 0; + const auto source_row = orc_source_row_at(row, orc_view.selected_rows); + if (!orc_row_is_null(*orc_view.batch, source_row)) { + RETURN_IF_ERROR(scale_decimal_value(to_int128(decimal128_batch->values[source_row]), + decimal128_batch->scale, target_scale, &value)); + } + fill_decimal_big_endian_value(value, &decimal_values[row]); + binary_values.emplace_back(reinterpret_cast(decimal_values[row].data()), + decimal_values[row].size()); + } + view.binary_values = &binary_values; + RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); + return Status::OK(); +} + +Status append_orc_offsets(ColumnArray::Offsets64& doris_offsets, + const ::orc::DataBuffer& orc_offsets, size_t rows, + size_t* element_size, const std::vector* selected_rows, + std::vector* element_selection) { + DORIS_CHECK(element_size != nullptr); + if (selected_rows != nullptr) { + DORIS_CHECK(element_selection != nullptr); + const auto prev_offset = doris_offsets.empty() ? 0 : doris_offsets.back(); + ColumnArray::Offset64 current_offset = prev_offset; + element_selection->clear(); + for (size_t row = 0; row < selected_rows->size(); ++row) { + const auto source_row = (*selected_rows)[row]; + DORIS_CHECK(source_row < rows); + const auto begin_offset = orc_offsets[source_row]; + const auto end_offset = orc_offsets[source_row + 1]; + if (end_offset < begin_offset) { + return Status::Corruption("Invalid ORC offsets"); + } + const auto delta = static_cast(end_offset - begin_offset); + for (size_t element_idx = 0; element_idx < delta; ++element_idx) { + element_selection->push_back(static_cast(begin_offset) + element_idx); + } + current_offset += static_cast(delta); + doris_offsets.push_back(current_offset); + } + *element_size = element_selection->size(); + return Status::OK(); + } + + const auto prev_offset = doris_offsets.empty() ? 0 : doris_offsets.back(); + const auto base_offset = orc_offsets[0]; + for (size_t idx = 1; idx <= rows; ++idx) { + const auto delta = orc_offsets[idx] - base_offset; + if (delta < 0) { + return Status::Corruption("Invalid ORC offsets"); + } + doris_offsets.push_back(prev_offset + static_cast(delta)); + } + const auto total_delta = orc_offsets[rows] - base_offset; + if (total_delta < 0) { + return Status::Corruption("Invalid ORC offsets"); + } + *element_size = static_cast(total_delta); + return Status::OK(); +} + +int64_t find_struct_child_index(const ::orc::Type& type, const std::string& field_name) { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::STRUCT); + for (uint64_t child_idx = 0; child_idx < type.getSubtypeCount(); ++child_idx) { + if (type.getFieldName(child_idx) == field_name) { + return static_cast(child_idx); + } + } + return -1; +} + +Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view, + const cctz::time_zone& timezone) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC timestamp batch type {}", + orc_view.batch->toString()); + } + auto& data = assert_cast(nested_column).get_data(); + const size_t old_data_size = data.size(); + const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + data.resize(old_data_size + output_rows); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_source_row_at(row, orc_view.selected_rows); + if (orc_row_is_null(*orc_view.batch, source_row)) { + data[old_data_size + row] = DateV2Value {}; + continue; + } + auto& value = + reinterpret_cast&>(data[old_data_size + row]); + value.from_unixtime(orc_batch->data[source_row], timezone); + value.set_microsecond(cast_set(orc_batch->nanoseconds[source_row] / 1000)); + } + return Status::OK(); +} + +Status decode_timestamp_tz_orc_values(IColumn& nested_column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC timestamp batch type {}", + orc_view.batch->toString()); + } + auto& data = assert_cast(nested_column).get_data(); + const size_t old_data_size = data.size(); + const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + data.resize(old_data_size + output_rows); + static const auto utc_time_zone = cctz::utc_time_zone(); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_source_row_at(row, orc_view.selected_rows); + if (orc_row_is_null(*orc_view.batch, source_row)) { + data[old_data_size + row] = TimestampTzValue {}; + continue; + } + auto& value = data[old_data_size + row]; + value.from_unixtime(orc_batch->data[source_row], utc_time_zone); + value.set_microsecond(cast_set(orc_batch->nanoseconds[source_row] / 1000)); + } + return Status::OK(); +} + +OrcDecodedColumnView make_child_orc_view(const OrcDecodedColumnView& parent_view, + const ::orc::Type* file_type, + const ::orc::Type* selected_type, + const ::orc::ColumnVectorBatch* batch, size_t rows, + const std::vector* selected_rows) { + OrcDecodedColumnView child_view = parent_view; + child_view.file_type = file_type; + child_view.selected_type = selected_type; + child_view.batch = batch; + child_view.rows = rows; + child_view.selected_rows = selected_rows; + return child_view; +} + +Status read_orc_child_column(const DataTypeSerDeSPtr& child_serde, MutableColumnPtr& child_column, + const OrcDecodedColumnView& child_view) { + DORIS_CHECK(child_serde != nullptr); + RETURN_IF_ERROR(child_serde->read_column_from_orc(*child_column, child_view)); + return Status::OK(); +} + +Status decode_list_orc_values(const DataTypeSerDeSPtr& nested_serde, IColumn& nested_column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_list = dynamic_cast(orc_view.batch); + if (orc_list == nullptr) { + return Status::InternalError("Unexpected ORC list batch type {}", + orc_view.batch->toString()); + } + DORIS_CHECK(orc_view.file_type != nullptr); + DORIS_CHECK(orc_view.selected_type != nullptr); + DORIS_CHECK(orc_view.file_type->getSubtypeCount() == 1); + DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == 1); + DORIS_CHECK(orc_list->elements != nullptr); + const auto* file_element_type = orc_view.file_type->getSubtype(0); + const auto* selected_element_type = orc_view.selected_type->getSubtype(0); + DORIS_CHECK(file_element_type != nullptr); + DORIS_CHECK(selected_element_type != nullptr); + + auto& array_column = assert_cast(nested_column); + size_t element_size = 0; + std::vector element_selection; + RETURN_IF_ERROR(append_orc_offsets(array_column.get_offsets(), orc_list->offsets, orc_view.rows, + &element_size, orc_view.selected_rows, &element_selection)); + auto element_column = array_column.get_data_ptr()->assert_mutable(); + const auto child_rows = orc_view.selected_rows == nullptr + ? element_size + : static_cast(orc_list->elements->numElements); + const auto* child_selection = orc_view.selected_rows == nullptr ? nullptr : &element_selection; + auto child_view = make_child_orc_view(orc_view, file_element_type, selected_element_type, + orc_list->elements.get(), child_rows, child_selection); + RETURN_IF_ERROR(read_orc_child_column(nested_serde, element_column, child_view)); + array_column.get_data_ptr() = std::move(element_column); + return Status::OK(); +} + +Status decode_map_orc_values(const DataTypeSerDeSPtr& key_serde, + const DataTypeSerDeSPtr& value_serde, IColumn& nested_column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_map = dynamic_cast(orc_view.batch); + if (orc_map == nullptr) { + return Status::InternalError("Unexpected ORC map batch type {}", + orc_view.batch->toString()); + } + DORIS_CHECK(orc_view.file_type != nullptr); + DORIS_CHECK(orc_view.selected_type != nullptr); + DORIS_CHECK(orc_view.file_type->getSubtypeCount() == 2); + DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == 2); + DORIS_CHECK(orc_map->keys != nullptr); + DORIS_CHECK(orc_map->elements != nullptr); + auto& map_column = assert_cast(nested_column); + size_t element_size = 0; + std::vector element_selection; + RETURN_IF_ERROR(append_orc_offsets(map_column.get_offsets(), orc_map->offsets, orc_view.rows, + &element_size, orc_view.selected_rows, &element_selection)); + + const auto* file_key_type = orc_view.file_type->getSubtype(0); + const auto* selected_key_type = orc_view.selected_type->getSubtype(0); + DORIS_CHECK(file_key_type != nullptr); + DORIS_CHECK(selected_key_type != nullptr); + const auto child_rows = orc_view.selected_rows == nullptr + ? element_size + : static_cast(orc_map->keys->numElements); + const auto* child_selection = orc_view.selected_rows == nullptr ? nullptr : &element_selection; + auto key_column = map_column.get_keys_ptr()->assert_mutable(); + auto key_view = make_child_orc_view(orc_view, file_key_type, selected_key_type, + orc_map->keys.get(), child_rows, child_selection); + RETURN_IF_ERROR(read_orc_child_column(key_serde, key_column, key_view)); + map_column.get_keys_ptr() = std::move(key_column); + + const auto* file_value_type = orc_view.file_type->getSubtype(1); + const auto* selected_value_type = orc_view.selected_type->getSubtype(1); + DORIS_CHECK(file_value_type != nullptr); + DORIS_CHECK(selected_value_type != nullptr); + auto value_column = map_column.get_values_ptr()->assert_mutable(); + auto value_view = make_child_orc_view( + orc_view, file_value_type, selected_value_type, orc_map->elements.get(), + orc_view.selected_rows == nullptr ? element_size + : static_cast(orc_map->elements->numElements), + child_selection); + RETURN_IF_ERROR(read_orc_child_column(value_serde, value_column, value_view)); + map_column.get_values_ptr() = std::move(value_column); + return Status::OK(); +} + +Status decode_struct_orc_values(const DataTypeSerDeSPtrs& elem_serdes_ptrs, IColumn& nested_column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_struct = dynamic_cast(orc_view.batch); + if (orc_struct == nullptr) { + return Status::InternalError("Unexpected ORC struct batch type {}", + orc_view.batch->toString()); + } + DORIS_CHECK(orc_view.file_type != nullptr); + DORIS_CHECK(orc_view.selected_type != nullptr); + DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == orc_struct->fields.size()); + auto& struct_column = assert_cast(nested_column); + DORIS_CHECK(struct_column.tuple_size() == orc_view.selected_type->getSubtypeCount()); + DORIS_CHECK(elem_serdes_ptrs.size() == orc_view.selected_type->getSubtypeCount()); + + for (uint64_t selected_idx = 0; selected_idx < orc_view.selected_type->getSubtypeCount(); + ++selected_idx) { + const auto field_name = orc_view.selected_type->getFieldName(selected_idx); + const auto file_child_idx = find_struct_child_index(*orc_view.file_type, field_name); + if (file_child_idx < 0) { + return Status::InternalError("Selected ORC field {} is not in file struct", field_name); + } + const auto* file_child_type = + orc_view.file_type->getSubtype(static_cast(file_child_idx)); + const auto* selected_child_type = orc_view.selected_type->getSubtype(selected_idx); + DORIS_CHECK(file_child_type != nullptr); + DORIS_CHECK(selected_child_type != nullptr); + DORIS_CHECK(selected_idx < orc_struct->fields.size()); + auto child_column = + struct_column.get_column_ptr(static_cast(selected_idx))->assert_mutable(); + auto child_view = make_child_orc_view(orc_view, file_child_type, selected_child_type, + orc_struct->fields[selected_idx], orc_view.rows, + orc_view.selected_rows); + RETURN_IF_ERROR( + read_orc_child_column(elem_serdes_ptrs[selected_idx], child_column, child_view)); + struct_column.get_column_ptr(static_cast(selected_idx)) = std::move(child_column); + } + return Status::OK(); +} + +} // namespace + DataTypeSerDe::~DataTypeSerDe() = default; +bool decoded_column_view_can_null_on_conversion_failure(const DecodedColumnView& view) { + return !view.enable_strict_mode && view.conversion_failure_null_map != nullptr; +} + +void decoded_column_view_insert_null_on_conversion_failure(IColumn& column, + const DecodedColumnView& view, + int64_t row) { + DORIS_CHECK(decoded_column_view_can_null_on_conversion_failure(view)); + DORIS_CHECK(row >= 0); + DORIS_CHECK(row < view.row_count); + DORIS_CHECK(view.conversion_failure_null_map_offset >= 0); + const auto null_map_row = view.conversion_failure_null_map_offset + row; + DORIS_CHECK(null_map_row >= 0); + DORIS_CHECK(static_cast(null_map_row) < view.conversion_failure_null_map->size()); + column.insert_default(); + (*view.conversion_failure_null_map)[null_map_row] = 1; +} + +Status decoded_column_view_handle_conversion_failure(IColumn& column, const DecodedColumnView& view, + const Status& status) { + if (!decoded_column_view_can_null_on_conversion_failure(view)) { + return status; + } + for (int64_t row = 0; row < view.row_count; ++row) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + } + return Status::OK(); +} + +Status DataTypeSerDe::read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("read_column_from_decoded_values is not supported for {}", + get_name())); +} + +Status DataTypeSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + return Status::NotSupported("read_column_from_orc is not supported for {}", get_name()); +} + +Status DataTypeNullableSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.selected_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == view.selected_type->getKind()); + auto& nullable_column = assert_cast(column); + const auto output_rows = orc_decode_row_count(view.rows, view.selected_rows); + if (output_rows == 0) { + return Status::OK(); + } + + auto& null_map = nullable_column.get_null_map_data(); + const auto old_null_map_size = null_map.size(); + auto& nested_column = nullable_column.get_nested_column(); + const auto old_nested_size = nested_column.size(); + append_orc_null_map(*view.batch, view.rows, view.selected_rows, &null_map); + auto st = nested_serde->read_column_from_orc(nested_column, view); + if (!st.ok()) { + null_map.resize(old_null_map_size); + nested_column.resize(old_nested_size); + } + return st; +} + +template +Status DataTypeNumberSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + + if constexpr (T == TYPE_BOOLEAN) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::BOOLEAN); + return decode_boolean_orc_values(*this, column, view); + } else if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT) { + if constexpr (T == TYPE_TINYINT) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::BYTE); + } else if constexpr (T == TYPE_SMALLINT) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::SHORT); + } else if constexpr (T == TYPE_INT) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::INT); + } else { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::LONG); + } + return decode_fixed_orc_values<::orc::LongVectorBatch, int64_t>(*this, column, view, + DecodedValueKind::INT64); + } else if constexpr (T == TYPE_FLOAT) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::FLOAT); + return decode_float_orc_values(*this, column, view); + } else if constexpr (T == TYPE_DOUBLE) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DOUBLE); + return decode_fixed_orc_values<::orc::DoubleVectorBatch, double>(*this, column, view, + DecodedValueKind::DOUBLE); + } + return DataTypeSerDe::read_column_from_orc(column, view); +} + +template +Status DataTypeStringSerDeBase::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + const auto kind = view.file_type->getKind(); + DORIS_CHECK(kind == ::orc::TypeKind::STRING || kind == ::orc::TypeKind::BINARY || + kind == ::orc::TypeKind::VARCHAR || kind == ::orc::TypeKind::CHAR); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_string_orc_values(*this, column, view); +} + +template +Status DataTypeDecimalSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DECIMAL); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_decimal_orc_values(*this, column, view, cast_set(scale)); +} + +Status DataTypeDateV2SerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DATE); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_date_orc_values(*this, column, view); +} + +Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + const auto kind = view.file_type->getKind(); + DORIS_CHECK(kind == ::orc::TypeKind::TIMESTAMP || kind == ::orc::TypeKind::TIMESTAMP_INSTANT); + DORIS_CHECK(view.timezone != nullptr); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_timestamp_orc_values(column, view, *view.timezone); +} + +Status DataTypeTimeStampTzSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::TIMESTAMP_INSTANT); + DORIS_CHECK(view.enable_mapping_timestamp_tz); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_timestamp_tz_orc_values(column, view); +} + +Status DataTypeArraySerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::LIST); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_list_orc_values(nested_serde, column, view); +} + +Status DataTypeMapSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::MAP); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_map_orc_values(key_serde, value_serde, column, view); +} + +Status DataTypeStructSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::STRUCT); + if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_struct_orc_values(elem_serdes_ptrs, column, view); +} + +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeNumberSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; + +template Status DataTypeStringSerDeBase::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeStringSerDeBase::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeStringSerDeBase::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; + +template Status DataTypeDecimalSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeDecimalSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeDecimalSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeDecimalSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; +template Status DataTypeDecimalSerDe::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const; + +Status DataTypeSerDe::read_field_from_decoded_value(const IDataType& data_type, Field* field, + const DecodedColumnView& view) const { + DORIS_CHECK(field != nullptr); + DORIS_CHECK(view.row_count == 1); + auto column = data_type.create_column(); + RETURN_IF_ERROR(read_column_from_decoded_values(*column, view)); + DORIS_CHECK(column->size() == 1); + column->get(0, *field); + return Status::OK(); +} + DataTypeSerDeSPtrs create_data_type_serdes(const DataTypes& types) { DataTypeSerDeSPtrs serdes; serdes.reserve(types.size()); diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index 4876f2e0b27e20..2c32504984a237 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -27,6 +27,7 @@ #include "common/cast_set.h" #include "common/status.h" #include "core/column/column_nullable.h" +#include "core/data_type_serde/decoded_column_view.h" #include "core/field.h" #include "core/string_buffer.hpp" #include "core/types.h" @@ -41,6 +42,7 @@ namespace cctz { class time_zone; } // namespace cctz namespace orc { +class Type; struct ColumnVectorBatch; } // namespace orc @@ -112,6 +114,16 @@ struct FieldInfo { int precision = 0; }; +struct OrcDecodedColumnView { + const orc::Type* file_type = nullptr; + const orc::Type* selected_type = nullptr; + const orc::ColumnVectorBatch* batch = nullptr; + size_t rows = 0; + const std::vector* selected_rows = nullptr; + const cctz::time_zone* timezone = nullptr; + bool enable_mapping_timestamp_tz = false; +}; + // Deserialize means read from different file format or memory format, // for example read from arrow, read from parquet. // Serialize means write the column cell or the total column into another @@ -486,6 +498,14 @@ class DataTypeSerDe { int64_t start, int64_t end, const cctz::time_zone& ctz) const = 0; + // Read already decoded column values into a Doris column. The input view is format-neutral: + // file readers translate their decoder output into DecodedColumnView, while SerDe owns + // the Doris-type-specific materialization into IColumn. + virtual Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const; + virtual Status read_field_from_decoded_value(const IDataType& data_type, Field* field, + const DecodedColumnView& view) const; + // ORC serializer virtual Status write_column_to_orc(const std::string& timezone, const IColumn& column, const NullMap* null_map, @@ -493,6 +513,7 @@ class DataTypeSerDe { int64_t end, Arena& arena, const FormatOptions& options) const = 0; // ORC deserializer + virtual Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const; virtual void set_return_object_as_string(bool value) { _return_object_as_string = value; } diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 5b5a79efef5248..7f9a82c23cc6ac 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -22,11 +22,40 @@ #include "core/column/column_string.h" #include "core/data_type/define_primitive_type.h" +#include "core/data_type_serde/decoded_column_view.h" #include "util/jsonb_document_cast.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" namespace doris { +namespace { + +template +Status read_string_decoded_values(IColumn& column, const DecodedColumnView& view) { + if (view.binary_values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded binary values are null for {}", column.get_name()); + } + auto& string_column = assert_cast(column); + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + string_column.insert_default(); + continue; + } + const auto& value = (*view.binary_values)[row]; + if (value.data == nullptr && value.size > 0) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + return Status::Corruption("Decoded string binary value is null for {} at row {}", + column.get_name(), row); + } + string_column.insert_data(value.data, value.size); + } + return Status::OK(); +} + +} // namespace namespace { @@ -430,6 +459,19 @@ Status DataTypeStringSerDeBase::read_column_from_arrow( return Status::OK(); } +template +Status DataTypeStringSerDeBase::read_column_from_decoded_values( + IColumn& column, const DecodedColumnView& view) const { + if (view.value_kind != DecodedValueKind::BINARY && + view.value_kind != DecodedValueKind::FIXED_BINARY) { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("Unsupported decoded values for {} from source kind {}", + get_name(), static_cast(view.value_kind))); + } + return read_string_decoded_values(column, view); +} + template Status DataTypeStringSerDeBase::write_column_to_orc( const std::string& timezone, const IColumn& column, const NullMap* null_map, diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index 1b07739b8f6c17..6c64b1276c3afb 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -204,6 +204,10 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const override; + Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, const FormatOptions& options) const override { diff --git a/be/src/core/data_type_serde/data_type_struct_serde.h b/be/src/core/data_type_serde/data_type_struct_serde.h index 70053fdeb31f25..dd3ff38f1609d6 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.h +++ b/be/src/core/data_type_serde/data_type_struct_serde.h @@ -86,6 +86,7 @@ class DataTypeStructSerDe : public DataTypeSerDe { const cctz::time_zone& ctz) const override; Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index c6ff002dc802dc..c40e671793c848 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -20,12 +20,38 @@ #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" +#include "core/data_type_serde/decoded_column_view.h" #include "core/value/time_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_time_impl.hpp" namespace doris { -#include "common/compile_check_begin.h" +namespace { + +TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64_t row) { + int64_t micros = 0; + if (view.value_kind == DecodedValueKind::INT32) { + const auto* values = reinterpret_cast(view.values); + micros = static_cast(values[row]) * 1000; + } else { + const auto* values = reinterpret_cast(view.values); + micros = values[row]; + if (view.time_unit == DecodedTimeUnit::MILLIS) { + micros *= 1000; + } else if (view.time_unit == DecodedTimeUnit::NANOS) { + micros /= 1000; + } + } + const bool negative = micros < 0; + const int64_t abs_micros = std::abs(micros); + return TimeValue::make_time( + abs_micros / TimeValue::ONE_HOUR_MICROSECONDS, + (abs_micros % TimeValue::ONE_HOUR_MICROSECONDS) / TimeValue::ONE_MINUTE_MICROSECONDS, + (abs_micros % TimeValue::ONE_MINUTE_MICROSECONDS) / TimeValue::ONE_SECOND_MICROSECONDS, + abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); +} + +} // namespace Status DataTypeTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, @@ -146,6 +172,27 @@ Status DataTypeTimeV2SerDe::from_string_strict_mode(StringRef& str, IColumn& col return Status::OK(); } +Status DataTypeTimeV2SerDe::read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const { + if (view.value_kind != DecodedValueKind::INT32 && view.value_kind != DecodedValueKind::INT64) { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("TIMEV2 decoded reader expects INT32 or INT64 source")); + } + if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); + } + auto& data = assert_cast(column).get_data(); + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(TimeValue::TimeType()); + continue; + } + data.push_back(read_time_decoded_value(view, row)); + } + return Status::OK(); +} + template Status DataTypeTimeV2SerDe::from_int_batch(const typename IntDataType::ColumnType& int_col, ColumnNullable& target_col) const { diff --git a/be/src/core/data_type_serde/data_type_time_serde.h b/be/src/core/data_type_serde/data_type_time_serde.h index dafaa600eb49e0..c3b8ab41a0fdd5 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.h +++ b/be/src/core/data_type_serde/data_type_time_serde.h @@ -68,6 +68,8 @@ class DataTypeTimeV2SerDe : public DataTypeNumberSerDe Status from_decimal_strict_mode_batch(const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col) const; + Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const override; int get_scale() const override { return _scale; } protected: diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp index e8c26f6db68e75..abc8b86700023a 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp @@ -18,14 +18,64 @@ #include "core/data_type_serde/data_type_timestamptz_serde.h" #include +#include #include "core/data_type/primitive_type.h" +#include "core/data_type_serde/decoded_column_view.h" #include "core/value/timestamptz_value.h" #include "exprs/function/cast/cast_parameters.h" #include "exprs/function/cast/cast_to_string.h" #include "exprs/function/cast/cast_to_timestamptz.h" namespace doris { +namespace { + +#pragma pack(1) +struct DecodedInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; + + int64_t to_timestamp_micros() const { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_IN_DAY = 86400000000; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + return (julian_day - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + + nanos_of_day / NANOS_PER_MICROSECOND; + } +}; +#pragma pack() +static_assert(sizeof(DecodedInt96Timestamp) == 12); + +void append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& data, + int64_t timestamp_micros) { + static constexpr int64_t MICROS_PER_SECOND = 1000000; + static const auto UTC = cctz::utc_time_zone(); + + int64_t epoch_seconds = timestamp_micros / MICROS_PER_SECOND; + int64_t micros_of_second = timestamp_micros % MICROS_PER_SECOND; + if (micros_of_second < 0) { + micros_of_second += MICROS_PER_SECOND; + --epoch_seconds; + } + + TimestampTzValue timestamp_tz; + timestamp_tz.from_unixtime(epoch_seconds, UTC); + timestamp_tz.set_microsecond(static_cast(micros_of_second)); + data.push_back(timestamp_tz); +} + +int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { + if (view.time_unit == DecodedTimeUnit::MILLIS) { + return value * 1000; + } + if (view.time_unit == DecodedTimeUnit::NANOS) { + return value / 1000; + } + return value; +} + +} // namespace + // The implementation of these functions mainly refers to data_type_datetimev2_serde.cpp Status DataTypeTimeStampTzSerDe::from_string(StringRef& str, IColumn& column, @@ -246,6 +296,41 @@ Status DataTypeTimeStampTzSerDe::write_column_to_orc(const std::string& timezone return Status::OK(); } +Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( + IColumn& column, const DecodedColumnView& view) const { + if (view.value_kind != DecodedValueKind::INT64 && view.value_kind != DecodedValueKind::INT96) { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("TIMESTAMPTZ decoded reader expects INT64 or INT96 source")); + } + if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); + } + + auto& data = assert_cast(column).get_data(); + if (view.value_kind == DecodedValueKind::INT96) { + const auto* values = reinterpret_cast(view.values); + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(TimestampTzValue()); + continue; + } + append_timestamptz_from_utc_epoch_micros(data, values[row].to_timestamp_micros()); + } + return Status::OK(); + } + + const auto* values = reinterpret_cast(view.values); + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(TimestampTzValue()); + continue; + } + append_timestamptz_from_utc_epoch_micros(data, decoded_timestamp_micros(view, values[row])); + } + return Status::OK(); +} + std::string DataTypeTimeStampTzSerDe::to_olap_string(const Field& field) const { return CastToString::from_timestamptz(field.get(), 6); } diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.h b/be/src/core/data_type_serde/data_type_timestamptz_serde.h index 0a595935d8fdd6..23d57f57fc8dac 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.h +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.h @@ -22,6 +22,7 @@ #include #include "core/data_type_serde/data_type_number_serde.h" +#include "core/data_type_serde/decoded_column_view.h" #include "core/types.h" #include "core/value/time_value.h" @@ -72,6 +73,10 @@ class DataTypeTimeStampTzSerDe : public DataTypeNumberSerDe +#include +#include + +#include "common/status.h" +#include "core/column/column_nullable.h" +#include "core/string_ref.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { + +class IColumn; + +// Physical value source type for a decoded column batch. +// This enum describes only generic memory layouts, not format-specific types such as +// Parquet/ORC/Arrow. +enum class DecodedValueKind { + BOOL, + INT32, + UINT32, + INT64, + UINT64, + INT96, + FLOAT, + DOUBLE, + BINARY, + FIXED_BINARY, +}; + +enum class DecodedTimeUnit { + UNKNOWN, + MILLIS, + MICROS, + NANOS, +}; + +struct DecodedColumnView { + DecodedValueKind value_kind = DecodedValueKind::INT32; + DecodedTimeUnit time_unit = DecodedTimeUnit::UNKNOWN; + int64_t row_count = 0; + // Optional logical integer annotation. value_kind still describes the physical buffer layout. + int logical_integer_bit_width = -1; + int decimal_precision = -1; + int decimal_scale = -1; + int fixed_length = -1; + bool logical_integer_is_signed = true; + bool timestamp_is_adjusted_to_utc = false; + const uint8_t* values = nullptr; + const uint8_t* null_map = nullptr; + const std::vector* binary_values = nullptr; + const cctz::time_zone* timezone = nullptr; + bool enable_strict_mode = false; + NullMap* conversion_failure_null_map = nullptr; + int64_t conversion_failure_null_map_offset = 0; +}; + +inline bool decoded_column_view_row_is_null(const DecodedColumnView& view, int64_t row) { + return view.null_map != nullptr && view.null_map[row] != 0; +} + +inline bool decoded_column_view_has_non_null_value(const DecodedColumnView& view) { + if (view.null_map == nullptr) { + return view.row_count > 0; + } + + // TODO(gabriel): optimize null map check with SIMD or bitset if needed. + for (int64_t row = 0; row < view.row_count; ++row) { + if (view.null_map[row] == 0) { + return true; + } + } + return false; +} + +bool decoded_column_view_can_null_on_conversion_failure(const DecodedColumnView& view); + +void decoded_column_view_insert_null_on_conversion_failure(IColumn& column, + const DecodedColumnView& view, + int64_t row); + +Status decoded_column_view_handle_conversion_failure(IColumn& column, const DecodedColumnView& view, + const Status& status); + +} // namespace doris diff --git a/be/src/exec/connector/jni_connector.cpp b/be/src/exec/connector/jni_connector.cpp index 40259d35df4450..a3e20f600ed038 100644 --- a/be/src/exec/connector/jni_connector.cpp +++ b/be/src/exec/connector/jni_connector.cpp @@ -841,6 +841,9 @@ void JniConnector::_collect_profile_before_close() { return; } + const auto update_peak = [](int64_t previous, int64_t current) { + return current > previous; + }; for (const auto& metric : statistics_result) { std::vector type_and_name = split(metric.first, ":"); if (type_and_name.size() != 2) { @@ -848,22 +851,49 @@ void JniConnector::_collect_profile_before_close() { << "'metricType:metricName'"; continue; } - long metric_value = std::stol(metric.second); + int64_t metric_value = std::stoll(metric.second); RuntimeProfile::Counter* scanner_counter; if (type_and_name[0] == "timer") { scanner_counter = ADD_CHILD_TIMER(_profile, type_and_name[1], _connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); } else if (type_and_name[0] == "counter") { scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::UNIT, _connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); } else if (type_and_name[0] == "bytes") { scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::BYTES, _connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); + } else if (type_and_name[0] == "timer_gauge") { + scanner_counter = + ADD_CHILD_TIMER(_profile, type_and_name[1], _connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "gauge") { + scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::UNIT, + _connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "bytes_gauge") { + scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::BYTES, + _connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "timer_peak") { + auto* scanner_peak_counter = _profile->add_conditition_counter( + type_and_name[1], TUnit::TIME_NS, update_peak, _connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); + } else if (type_and_name[0] == "peak") { + auto* scanner_peak_counter = _profile->add_conditition_counter( + type_and_name[1], TUnit::UNIT, update_peak, _connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); + } else if (type_and_name[0] == "bytes_peak") { + auto* scanner_peak_counter = _profile->add_conditition_counter( + type_and_name[1], TUnit::BYTES, update_peak, _connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); } else { - LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter or bytes"; + LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter, bytes, " + << "timer_gauge, gauge, bytes_gauge, timer_peak, peak or bytes_peak"; continue; } - COUNTER_UPDATE(scanner_counter, metric_value); } } } diff --git a/be/src/exec/operator/file_scan_operator.cpp b/be/src/exec/operator/file_scan_operator.cpp index 34fc77687eeeba..abe89da95842b2 100644 --- a/be/src/exec/operator/file_scan_operator.cpp +++ b/be/src/exec/operator/file_scan_operator.cpp @@ -24,6 +24,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/scan_operator.h" #include "exec/scan/file_scanner.h" +#include "exec/scan/file_scanner_v2.h" #include "exec/scan/scanner_context.h" #include "format/format_common.h" #include "storage/storage_engine.h" @@ -54,6 +55,18 @@ PushDownType FileScanLocalState::_should_push_down_binary_predicate( } } +bool FileScanLocalState::_push_down_topn(const RuntimePredicate& predicate) { + if (!predicate.target_is_slot(_parent->node_id())) { + return false; + } + auto& p = _parent->cast(); + const auto slot_id = predicate.get_texpr(_parent->node_id()).nodes[0].slot_ref.slot_id; + auto* slot = p._slot_id_to_slot_desc[slot_id]; + DCHECK(slot != nullptr); + // External readers do not fully support VARBINARY column predicates yet. + return slot->type()->get_primitive_type() != TYPE_VARBINARY; +} + int FileScanLocalState::max_scanners_concurrency(RuntimeState* state) const { // For select * from table limit 10; should just use one thread. if (should_run_serial()) { @@ -91,6 +104,28 @@ ScannerScheduler* FileScanLocalState::scan_scheduler(RuntimeState* state) const return state->get_query_ctx()->get_remote_scan_scheduler(); } +#ifdef BE_TEST +bool FileScanLocalState::TEST_should_use_file_scanner_v2(const TQueryOptions& query_options, + bool is_load, + const TFileScanRangeParams& scan_params) { + return _should_use_file_scanner_v2(query_options, is_load, scan_params); +} +#endif + +bool FileScanLocalState::_should_use_file_scanner_v2(const TQueryOptions& query_options, + bool is_load, + const TFileScanRangeParams& scan_params) { + const bool is_transactional_hive = + scan_params.__isset.table_format_params && + scan_params.table_format_params.table_format_type == "transactional_hive"; + // JNI reader selection is stored per split, but this scan-level selector cannot inspect the + // split yet. Older FEs may omit both the scan-level Paimon marker and split-level reader_type, + // so keep JNI scans on V1 until scanner selection can distinguish every compatibility shape. + return query_options.__isset.enable_file_scanner_v2 && query_options.enable_file_scanner_v2 && + !is_load && scan_params.format_type != TFileFormatType::FORMAT_WAL && + scan_params.format_type != TFileFormatType::FORMAT_JNI && !is_transactional_hive; +} + Status FileScanLocalState::_init_scanners(std::list* scanners) { if (_split_source->num_scan_ranges() == 0) { _eos = true; @@ -108,11 +143,31 @@ Status FileScanLocalState::_init_scanners(std::list* scanners) { std::min(ScannerScheduler::default_remote_scan_thread_num() / p.parallelism(state()), _max_scanners); shard_num = std::max(shard_num, 1U); - _kv_cache.reset(new ShardedKVCache(shard_num)); + _kv_cache = std::make_unique(shard_num); + const TFileScanRangeParams* scan_params = nullptr; + if (state()->get_query_ctx() != nullptr && + state()->get_query_ctx()->file_scan_range_params_map.count(parent_id()) > 0) { + scan_params = &state()->get_query_ctx()->file_scan_range_params_map[parent_id()]; + } else { + scan_params = _split_source->get_params(); + } + const bool is_load = + state()->desc_tbl().get_tuple_descriptor(scan_params->src_tuple_id) != nullptr; + // TODO: Use scanner v2 for all queries. + const bool use_file_scanner_v2 = + _should_use_file_scanner_v2(state()->query_options(), is_load, *scan_params); + _operator_profile->add_info_string("UseScannerV2", use_file_scanner_v2 ? "true" : "false"); for (int i = 0; i < _max_scanners; ++i) { - std::unique_ptr scanner = FileScanner::create_unique( - state(), this, p._limit, _split_source, _scanner_profile.get(), _kv_cache.get(), - &p._colname_to_slot_id); + ScannerSPtr scanner; + if (use_file_scanner_v2) { + scanner = FileScannerV2::create_shared(state(), this, p._limit, _split_source, + _scanner_profile.get(), _kv_cache.get(), + &p._colname_to_slot_id); + } else { + scanner = FileScanner::create_shared(state(), this, p._limit, _split_source, + _scanner_profile.get(), _kv_cache.get(), + &p._colname_to_slot_id); + } RETURN_IF_ERROR(scanner->init(state(), _conjuncts)); scanners->push_back(std::move(scanner)); } @@ -182,6 +237,9 @@ Status FileScanLocalState::init(RuntimeState* state, LocalStateInfo& info) { SCOPED_TIMER(_init_timer); auto& p = _parent->cast(); _output_tuple_id = p._output_tuple_id; + _condition_cache_hit_counter = ADD_COUNTER(custom_profile(), "ConditionCacheHit", TUnit::UNIT); + _condition_cache_filtered_rows_counter = + ADD_COUNTER(custom_profile(), "ConditionCacheFilteredRows", TUnit::UNIT); return Status::OK(); } diff --git a/be/src/exec/operator/file_scan_operator.h b/be/src/exec/operator/file_scan_operator.h index 91d4580a1a517c..2aaeebe049cf64 100644 --- a/be/src/exec/operator/file_scan_operator.h +++ b/be/src/exec/operator/file_scan_operator.h @@ -31,6 +31,7 @@ namespace doris { #include "common/compile_check_begin.h" class FileScanner; +class FileScannerV2; } // namespace doris namespace doris { @@ -55,20 +56,23 @@ class FileScanLocalState final : public ScanLocalState { int max_scanners_concurrency(RuntimeState* state) const override; int min_scanners_concurrency(RuntimeState* state) const override; ScannerScheduler* scan_scheduler(RuntimeState* state) const override; +#ifdef BE_TEST + static bool TEST_should_use_file_scanner_v2(const TQueryOptions& query_options, bool is_load, + const TFileScanRangeParams& scan_params); +#endif private: friend class FileScanner; + friend class FileScannerV2; PushDownType _should_push_down_bloom_filter() const override { return PushDownType::UNACCEPTABLE; } PushDownType _should_push_down_topn_filter() const override { return PushDownType::PARTIAL_ACCEPTABLE; } - bool _push_down_topn(const RuntimePredicate& predicate) override { - // For external table/ file scan, first try push down the predicate, - // and then determine whether it can be pushed down within the (parquet/orc) reader. - return true; - } + bool _push_down_topn(const RuntimePredicate& predicate) override; + static bool _should_use_file_scanner_v2(const TQueryOptions& query_options, bool is_load, + const TFileScanRangeParams& scan_params); PushDownType _should_push_down_is_null_predicate(VectorizedFnCall* fn_call) const override { return fn_call->fn().name.function_name == "is_null_pred" || @@ -91,6 +95,8 @@ class FileScanLocalState final : public ScanLocalState { // KVCache _kv_cache; std::unique_ptr _kv_cache; TupleId _output_tuple_id = -1; + RuntimeProfile::Counter* _condition_cache_hit_counter = nullptr; + RuntimeProfile::Counter* _condition_cache_filtered_rows_counter = nullptr; }; class FileScanOperatorX final : public ScanOperatorX { diff --git a/be/src/exec/operator/result_sink_operator.h b/be/src/exec/operator/result_sink_operator.h index 790aaa59cf65b1..84c86f1127e319 100644 --- a/be/src/exec/operator/result_sink_operator.h +++ b/be/src/exec/operator/result_sink_operator.h @@ -46,7 +46,7 @@ struct ResultFileOptions { TParquetCompressionType::type parquet_commpression_type; TParquetVersion::type parquet_version; bool parquert_disable_dictionary = false; - bool enable_int96_timestamps = false; + bool enable_int96_timestamps = true; //note: use outfile with parquet format, have deprecated 9:schema and 10:file_properties //But in order to consider the compatibility when upgrading, so add a bool to check //Now the code version is 1.1.2, so when the version is after 1.2, could remove this code. diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp new file mode 100644 index 00000000000000..b215212b6d861b --- /dev/null +++ b/be/src/exec/scan/access_path_parser.cpp @@ -0,0 +1,479 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/scan/access_path_parser.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "runtime/descriptors.h" +#include "util/string_util.h" + +namespace doris { +namespace { + +bool is_scanner_materialized_virtual_column(const std::string& column_name) { + return column_name == BeConsts::ICEBERG_ROWID_COL; +} + +bool parse_non_negative_int(std::string_view value, int32_t* result) { + DORIS_CHECK(result != nullptr); + int32_t parsed = -1; + const auto* begin = value.data(); + const auto* end = begin + value.size(); + const auto [ptr, ec] = std::from_chars(begin, end, parsed); + if (ec != std::errc() || ptr != end || parsed < 0) { + return false; + } + *result = parsed; + return true; +} + +std::string access_path_to_string(const std::vector& path) { + return fmt::format("{}", fmt::join(path, ".")); +} + +format::ColumnDefinition* find_or_add_child(format::ColumnDefinition* parent, int32_t id, + std::string name, DataTypePtr type) { + DORIS_CHECK(parent != nullptr); + for (auto& child : parent->children) { + if ((child.has_identifier_field_id() && child.get_identifier_field_id() == id) || + child.name == name) { + return &child; + } + } + parent->children.push_back({ + .identifier = Field::create_field(id), + .name = std::move(name), + .type = std::move(type), + .children = {}, + .default_expr = nullptr, + .is_partition_key = false, + }); + return &parent->children.back(); +} + +void inherit_schema_metadata(format::ColumnDefinition* column, + const format::ColumnDefinition* schema_column) { + if (column == nullptr || schema_column == nullptr) { + return; + } + column->name_mapping = schema_column->name_mapping; +} + +const format::ColumnDefinition* find_schema_child_by_path( + const format::ColumnDefinition* schema_column, const std::string& child_path) { + if (schema_column == nullptr) { + return nullptr; + } + int32_t parsed_field_id = -1; + if (parse_non_negative_int(child_path, &parsed_field_id)) { + const auto child_it = std::ranges::find_if( + schema_column->children, [&](const format::ColumnDefinition& child) { + return child.has_identifier_field_id() && + child.get_identifier_field_id() == parsed_field_id; + }); + return child_it == schema_column->children.end() ? nullptr : &*child_it; + } + const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { + if (to_lower(child.name) == to_lower(child_path)) { + return true; + } + return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(child_path); + }); + }); + return child_it == schema_column->children.end() ? nullptr : &*child_it; +} + +int32_t schema_field_id(const format::ColumnDefinition* schema_column) { + if (schema_column == nullptr || !schema_column->has_identifier_field_id()) { + return -1; + } + return schema_column->get_identifier_field_id(); +} + +int32_t schema_field_id_or(const format::ColumnDefinition* schema_column, int32_t fallback) { + const auto field_id = schema_field_id(schema_column); + return field_id >= 0 ? field_id : fallback; +} + +std::string schema_field_name_or(const format::ColumnDefinition* schema_column, + std::string fallback) { + return schema_column == nullptr || schema_column->name.empty() ? std::move(fallback) + : schema_column->name; +} + +struct AccessPathNode { + bool project_all = false; + std::map children; +}; + +void merge_access_path_node(AccessPathNode* dst, const AccessPathNode& src) { + DORIS_CHECK(dst != nullptr); + if (dst->project_all) { + return; + } + if (src.project_all) { + dst->project_all = true; + dst->children.clear(); + return; + } + for (const auto& [path, child] : src.children) { + merge_access_path_node(&dst->children[path], child); + } +} + +void insert_access_path(AccessPathNode* root, const std::vector& path, + size_t path_idx) { + DORIS_CHECK(root != nullptr); + if (root->project_all) { + return; + } + if (path_idx >= path.size()) { + root->project_all = true; + root->children.clear(); + return; + } + insert_access_path(&root->children[path[path_idx]], path, path_idx + 1); +} + +Status build_nested_children_from_access_node(format::ColumnDefinition* column, + const DataTypePtr& type, const AccessPathNode& node, + const std::string& path, + const format::ColumnDefinition* schema_column); + +// Expand a full complex-column projection into table-schema children when the table format provides +// an external/current schema. Without this, `SELECT complex_col` or `SELECT *` leaves +// ColumnDefinition::children empty, so ColumnMapper treats the root complex column as a scalar +// mapping and later tries to cast the old file shape to the current table shape directly. +// +// Examples: +// - STRUCT country/city projected from an old file STRUCT country/population/location should +// create children country and city, so city can be materialized as missing/default. +// - ARRAY> should create the array element wrapper and then the element +// struct children item and quantity. +// - MAP> should create semantic children key/value directly, then +// expand the value struct children full_name and age. Do not introduce a physical entries +// wrapper here: ColumnMapper and TableReader treat MAP children as [key, value]. +Status build_all_nested_children_from_schema(format::ColumnDefinition* column, + const DataTypePtr& type, const std::string& path, + const format::ColumnDefinition* schema_column) { + DORIS_CHECK(column != nullptr); + + const auto nested_type = remove_nullable(type); + AccessPathNode project_all; + project_all.project_all = true; + switch (nested_type->get_primitive_type()) { + case TYPE_STRUCT: { + const auto& struct_type = assert_cast(*nested_type); + for (size_t field_idx = 0; field_idx < struct_type.get_elements().size(); ++field_idx) { + const auto field_name = struct_type.get_element_name(field_idx); + const auto* schema_child = find_schema_child_by_path(schema_column, field_name); + auto* child = find_or_add_child( + column, schema_field_id_or(schema_child, cast_set(field_idx)), + schema_field_name_or(schema_child, field_name), + struct_type.get_element(field_idx)); + inherit_schema_metadata(child, schema_child); + RETURN_IF_ERROR(build_nested_children_from_access_node( + child, child->type, project_all, path + "." + child->name, schema_child)); + } + return Status::OK(); + } + case TYPE_ARRAY: { + const auto& array_type = assert_cast(*nested_type); + const auto* element_schema = schema_column != nullptr && !schema_column->children.empty() + ? &schema_column->children[0] + : nullptr; + auto* child = find_or_add_child(column, schema_field_id_or(element_schema, 0), "element", + array_type.get_nested_type()); + inherit_schema_metadata(child, element_schema); + return build_nested_children_from_access_node(child, child->type, project_all, path + ".*", + element_schema); + } + case TYPE_MAP: { + const auto& map_type = assert_cast(*nested_type); + const auto* key_schema = schema_column != nullptr && !schema_column->children.empty() + ? &schema_column->children[0] + : nullptr; + const auto* value_schema = schema_column != nullptr && schema_column->children.size() > 1 + ? &schema_column->children[1] + : nullptr; + auto* key_child = find_or_add_child(column, schema_field_id_or(key_schema, 0), "key", + map_type.get_key_type()); + inherit_schema_metadata(key_child, key_schema); + RETURN_IF_ERROR(build_nested_children_from_access_node( + key_child, key_child->type, project_all, path + ".KEYS", key_schema)); + auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value", + map_type.get_value_type()); + inherit_schema_metadata(value_child, value_schema); + RETURN_IF_ERROR(build_nested_children_from_access_node( + value_child, value_child->type, project_all, path + ".VALUES", value_schema)); + return Status::OK(); + } + default: + return Status::OK(); + } +} + +Status build_struct_children_from_access_node(format::ColumnDefinition* column, + const DataTypeStruct& struct_type, + const AccessPathNode& node, const std::string& path, + const format::ColumnDefinition* schema_column) { + DORIS_CHECK(column != nullptr); + for (const auto& [child_path, child_node] : node.children) { + // Struct children are resolved by name or schema field id. We do not treat a numeric + // child token as a struct ordinal, because `col.0` becomes ambiguous once the struct + // evolves. Position-based access needs a separate design if it is required later. + if (child_path == "OFFSET" || child_path == "*" || child_path == "KEYS" || + child_path == "VALUES") { + return Status::NotSupported( + "AccessPathParser does not support access path {} for slot {}", + path + "." + child_path, column->name); + } + + // Prefer the table/schema ColumnDefinition because it carries field ids and aliases. + // Fallback to the struct type name only for formats without external schema metadata. + const auto* schema_child = find_schema_child_by_path(schema_column, child_path); + int32_t field_id = schema_field_id(schema_child); + std::string field_name = schema_child == nullptr ? child_path : schema_child->name; + DataTypePtr field_type = schema_child == nullptr ? nullptr : schema_child->type; + if (field_id < 0 || field_type == nullptr) { + for (size_t field_idx = 0; field_idx < struct_type.get_elements().size(); ++field_idx) { + if (to_lower(struct_type.get_element_name(field_idx)) == to_lower(field_name)) { + field_id = cast_set(field_idx); + field_name = struct_type.get_element_name(field_idx); + field_type = struct_type.get_element(field_idx); + break; + } + } + } + + if (field_id < 0 || field_type == nullptr) { + return Status::NotSupported( + "AccessPathParser does not support access path {} for slot {}", + path + "." + child_path, column->name); + } + // TODO: For TVF Parquet files without field ids, this fallback uses the struct ordinal as + // the table child identifier. BY_NAME mapping should instead keep a string identifier and + // let TableColumnMapper resolve the file-local child id from the Parquet schema. + auto* child = find_or_add_child(column, field_id, field_name, field_type); + inherit_schema_metadata(child, schema_child); + RETURN_IF_ERROR(build_nested_children_from_access_node( + child, child->type, child_node, path + "." + child_path, schema_child)); + } + return Status::OK(); +} + +Status build_map_children_from_access_node(format::ColumnDefinition* column, + const DataTypeMap& map_type, const AccessPathNode& node, + const std::string& path, + const format::ColumnDefinition* schema_column) { + DORIS_CHECK(column != nullptr); + AccessPathNode key_node; + AccessPathNode value_node; + bool need_key = false; + bool need_value = false; + + for (const auto& [child_path, child_node] : node.children) { + if (child_path == "OFFSET") { + return Status::NotSupported( + "AccessPathParser does not support access path {} for slot {}", + path + "." + child_path, column->name); + } + if (child_path == "KEYS") { + need_key = true; + merge_access_path_node(&key_node, child_node); + continue; + } + if (child_path == "VALUES") { + need_key = true; + key_node.project_all = true; + key_node.children.clear(); + need_value = true; + merge_access_path_node(&value_node, child_node); + continue; + } + if (child_path == "*") { + need_key = true; + key_node.project_all = true; + key_node.children.clear(); + need_value = true; + merge_access_path_node(&value_node, child_node); + continue; + } + return Status::NotSupported("AccessPathParser does not support access path {} for slot {}", + path + "." + child_path, column->name); + } + if (need_key && !need_value) { + // A key-only MAP projection is not independently materializable yet. FileScannerV2 can + // describe a projection such as `m.KEYS`, but the downstream file block -> table block path + // still builds a ColumnMap from key column + value column + offsets. If the value child is + // omitted here, TableReader/ColumnMapper cannot reconstruct a valid table MAP column even + // though the query only needs keys. + // + // Example: + // SELECT map_keys(m) FROM t; + // or + // SELECT * FROM t WHERE array_contains(map_keys(m), 'k1'); + // + // The access path only asks for `m.KEYS`, but the scan still has to read `m.VALUES` as a + // temporary full projection until map materialization supports constructing a table MAP + // from keys only. + need_value = true; + value_node.project_all = true; + value_node.children.clear(); + } + + if (!need_key && !need_value) { + return Status::OK(); + } + + const auto* key_schema = schema_column != nullptr && !schema_column->children.empty() + ? &schema_column->children[0] + : nullptr; + const auto* value_schema = schema_column != nullptr && schema_column->children.size() > 1 + ? &schema_column->children[1] + : nullptr; + if (need_key) { + auto* key_child = find_or_add_child(column, schema_field_id_or(key_schema, 0), "key", + map_type.get_key_type()); + inherit_schema_metadata(key_child, key_schema); + RETURN_IF_ERROR(build_nested_children_from_access_node(key_child, key_child->type, key_node, + path + ".KEYS", key_schema)); + } + if (need_value) { + auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value", + map_type.get_value_type()); + inherit_schema_metadata(value_child, value_schema); + RETURN_IF_ERROR(build_nested_children_from_access_node( + value_child, value_child->type, value_node, path + ".VALUES", value_schema)); + } + return Status::OK(); +} + +Status build_nested_children_from_access_node(format::ColumnDefinition* column, + const DataTypePtr& type, const AccessPathNode& node, + const std::string& path, + const format::ColumnDefinition* schema_column) { + DORIS_CHECK(column != nullptr); + if (node.project_all || node.children.empty()) { + return build_all_nested_children_from_schema(column, type, path, schema_column); + } + + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_STRUCT: + return build_struct_children_from_access_node( + column, assert_cast(*nested_type), node, path, + schema_column); + case TYPE_ARRAY: { + if (node.children.size() != 1 || !node.children.contains("*")) { + return Status::NotSupported( + "AccessPathParser does not support access path {} for slot {}", path, + column->name); + } + const auto& array_type = assert_cast(*nested_type); + const auto* element_schema = schema_column != nullptr && !schema_column->children.empty() + ? &schema_column->children[0] + : nullptr; + auto* child = find_or_add_child(column, schema_field_id_or(element_schema, 0), "element", + array_type.get_nested_type()); + inherit_schema_metadata(child, element_schema); + return build_nested_children_from_access_node(child, child->type, node.children.at("*"), + path + ".*", element_schema); + } + case TYPE_MAP: + return build_map_children_from_access_node( + column, assert_cast(*nested_type), node, path, schema_column); + default: + return Status::NotSupported("AccessPathParser does not support access path {} for slot {}", + path, column->name); + } +} + +} // namespace + +Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, + const std::vector& access_paths, + const format::ColumnDefinition* schema_column) { + DORIS_CHECK(column != nullptr); + if (is_scanner_materialized_virtual_column(column->name)) { + return Status::OK(); + } + if (!is_complex_type(remove_nullable(column->type)->get_primitive_type())) { + return Status::OK(); + } + + AccessPathNode root; + // Build tree for AccessPathNode. + // For example, for access paths ["a.b", "a.c", "d"], the tree will be: + // root + // ├── a + // │ ├── b + // │ └── c + // └── d + for (const auto& access_path : access_paths) { + // TODO: Support META access paths if needed. Currently AccessPathParser only supports + // DATA access paths. + if (access_path.type != TAccessPathType::DATA || !access_path.__isset.data_access_path) { + return Status::NotSupported( + "AccessPathParser only supports DATA access paths for slot {}", column->name); + } + const auto& path = access_path.data_access_path.path; + if (path.empty()) { + insert_access_path(&root, path, 0); + continue; + } + int32_t top_level_id = -1; + if (to_lower(path.front()) != to_lower(column->name) && + (!parse_non_negative_int(path.front(), &top_level_id) || + !column->has_identifier_field_id() || + top_level_id != column->get_identifier_field_id())) { + return Status::NotSupported("AccessPathParser access path {} does not match slot {}", + access_path_to_string(path), column->name); + } + insert_access_path(&root, path, 1); + } + // Recursively build nested children for the column based on the AccessPathNode tree. + return build_nested_children_from_access_node(column, column->type, root, column->name, + schema_column); +} + +Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, + const SlotDescriptor* slot_desc, + const format::ColumnDefinition* schema_column) { + DORIS_CHECK(column != nullptr); + DORIS_CHECK(slot_desc != nullptr); + return build_nested_children(column, slot_desc->all_access_paths(), schema_column); +} + +} // namespace doris diff --git a/be/src/exec/scan/access_path_parser.h b/be/src/exec/scan/access_path_parser.h new file mode 100644 index 00000000000000..1aa4c5b89d492a --- /dev/null +++ b/be/src/exec/scan/access_path_parser.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "format_v2/column_data.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { + +class SlotDescriptor; + +class AccessPathParser { +public: + static Status build_nested_children(format::ColumnDefinition* column, + const SlotDescriptor* slot_desc, + const format::ColumnDefinition* schema_column); + + static Status build_nested_children(format::ColumnDefinition* column, + const std::vector& access_paths, + const format::ColumnDefinition* schema_column); +}; + +} // namespace doris diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 3c7a52a28936bf..b1ec7d6a47eee5 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -331,6 +331,11 @@ void FileScanner::_init_runtime_filter_partition_prune_ctxs() { auto impl = conjunct->root()->get_impl(); // If impl is not null, which means this a conjuncts from runtime filter. auto expr = impl ? impl : conjunct->root(); + // Preserve a safe prefix of the row-level conjunct order. Considering later predicates + // after an unsafe one could prune the split before the unsafe predicate is evaluated. + if (!expr->is_safe_to_execute_on_selected_rows()) { + break; + } if (_check_partition_prune_expr(expr)) { _runtime_filter_partition_prune_ctxs.emplace_back(conjunct); } diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 833ca9419bba13..88bf97df142403 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -66,6 +66,19 @@ class FileScanner : public Scanner { static const std::string FileReadBytesProfile; static const std::string FileReadTimeProfile; +#ifdef BE_TEST + void TEST_init_runtime_filter_partition_prune_ctxs( + const VExprContextSPtrs& conjuncts, + const std::unordered_map& partition_slot_index_map) { + _conjuncts = conjuncts; + _partition_slot_index_map = partition_slot_index_map; + _init_runtime_filter_partition_prune_ctxs(); + } + const VExprContextSPtrs& TEST_runtime_filter_partition_prune_ctxs() const { + return _runtime_filter_partition_prune_ctxs; + } +#endif + FileScanner(RuntimeState* state, FileScanLocalState* parent, int64_t limit, std::shared_ptr split_source, RuntimeProfile* profile, ShardedKVCache* kv_cache, diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp new file mode 100644 index 00000000000000..8d4f5443fd980d --- /dev/null +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -0,0 +1,1023 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/scan/file_scanner_v2.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/consts.h" +#include "common/metrics/doris_metrics.h" +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/column_with_type_and_name.h" +#include "core/column/column.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/string_ref.h" +#include "exec/common/util.hpp" +#include "exec/operator/scan_operator.h" +#include "exec/scan/access_path_parser.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vruntimefilter_wrapper.h" +#include "exprs/vslot_ref.h" +#include "format/format_common.h" +#include "format_v2/column_mapper.h" +#include "format_v2/jni/iceberg_sys_table_reader.h" +#include "format_v2/jni/jdbc_reader.h" +#include "format_v2/jni/max_compute_jni_reader.h" +#include "format_v2/jni/trino_connector_jni_reader.h" +#include "format_v2/table/hive_reader.h" +#include "format_v2/table/hudi_reader.h" +#include "format_v2/table/iceberg_reader.h" +#include "format_v2/table/paimon_reader.h" +#include "format_v2/table/remote_doris_reader.h" +#include "format_v2/table_reader.h" +#include "io/cache/block_file_cache_profile.h" +#include "io/fs/file_meta_cache.h" +#include "io/io_common.h" +#include "runtime/descriptors.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "service/backend_options.h" +#include "storage/id_manager.h" + +namespace doris { +namespace { + +std::string table_format_name(const TFileRangeDesc& range) { + return range.__isset.table_format_params ? range.table_format_params.table_format_type + : "NotSet"; +} + +TFileFormatType::type get_range_format_type(const TFileScanRangeParams& params, + const TFileRangeDesc& range) { + return range.__isset.format_type ? range.format_type : params.format_type; +} + +bool is_supported_table_format(const TFileRangeDesc& range) { + const auto table_format = table_format_name(range); + if (table_format == "hudi" && range.__isset.table_format_params && + range.table_format_params.__isset.hudi_params && + range.table_format_params.hudi_params.__isset.delta_logs && + !range.table_format_params.hudi_params.delta_logs.empty()) { + // Hudi MOR splits need log-file merge semantics and must stay on the existing JNI path. + // FileScannerV2 currently supports native Parquet data files only. + return false; + } + return table_format == "NotSet" || table_format == "tvf" || table_format == "hive" || + table_format == "iceberg" || table_format == "paimon" || table_format == "hudi"; +} + +bool is_supported_arrow_table_format(const TFileRangeDesc& range) { + return table_format_name(range) == "remote_doris"; +} + +bool is_supported_jni_table_format(const TFileRangeDesc& range) { + const auto table_format = table_format_name(range); + if (table_format == "paimon") { + return range.__isset.table_format_params && + range.table_format_params.__isset.paimon_params && + range.table_format_params.paimon_params.__isset.reader_type && + range.table_format_params.paimon_params.reader_type == TPaimonReaderType::PAIMON_JNI; + } + return table_format == "jdbc" || table_format == "iceberg" || table_format == "hudi" || + table_format == "max_compute" || table_format == "trino_connector"; +} + +bool is_csv_format(TFileFormatType::type format_type) { + switch (format_type) { + case TFileFormatType::FORMAT_CSV_PLAIN: + case TFileFormatType::FORMAT_CSV_GZ: + case TFileFormatType::FORMAT_CSV_BZ2: + case TFileFormatType::FORMAT_CSV_LZ4FRAME: + case TFileFormatType::FORMAT_CSV_LZ4BLOCK: + case TFileFormatType::FORMAT_CSV_LZOP: + case TFileFormatType::FORMAT_CSV_DEFLATE: + case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK: + case TFileFormatType::FORMAT_PROTO: + return true; + default: + return false; + } +} + +bool is_text_format(TFileFormatType::type format_type) { + return format_type == TFileFormatType::FORMAT_TEXT; +} + +bool is_json_format(TFileFormatType::type format_type) { + return format_type == TFileFormatType::FORMAT_JSON; +} + +bool is_native_format(TFileFormatType::type format_type) { + return format_type == TFileFormatType::FORMAT_NATIVE; +} + +bool is_partition_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) { + if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) || + column_name == BeConsts::ICEBERG_ROWID_COL) { + return false; + } + return !slot_info.is_file_slot; +} + +bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) { + if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) || + column_name == BeConsts::ICEBERG_ROWID_COL) { + return false; + } + // CSV and other non-self-describing formats need FE slot descriptors for only the columns that + // are physically read from the file. Partition/default/virtual columns stay in TableReader's + // mapping layer and are materialized after the file-local block is read. New FE provides an + // explicit category; old FE falls back to `is_file_slot`. + return slot_info.is_file_slot; +} + +Status rewrite_slot_refs_to_global_index( + VExprSPtr* expr, + const std::unordered_map& slot_id_to_global_index) { + DORIS_CHECK(expr != nullptr); + if (*expr == nullptr) { + return Status::OK(); + } + if (auto* runtime_filter = dynamic_cast(expr->get()); + runtime_filter != nullptr) { + auto impl = runtime_filter->get_impl(); + DORIS_CHECK(impl != nullptr); + RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&impl, slot_id_to_global_index)); + runtime_filter->set_impl(std::move(impl)); + return Status::OK(); + } + if ((*expr)->is_slot_ref()) { + const auto* slot_ref = assert_cast(expr->get()); + const auto global_index_it = slot_id_to_global_index.find(slot_ref->slot_id()); + if (global_index_it == slot_id_to_global_index.end()) { + DORIS_CHECK(slot_ref->slot_id() >= 0); + const auto global_index = format::GlobalIndex(cast_set(slot_ref->slot_id())); + *expr = VSlotRef::create_shared(cast_set(global_index.value()), + cast_set(global_index.value()), -1, + slot_ref->data_type(), slot_ref->column_name()); + RETURN_IF_ERROR(expr->get()->prepare(nullptr, RowDescriptor(), nullptr)); + return Status::OK(); + } + const auto global_index = global_index_it->second; + *expr = VSlotRef::create_shared(cast_set(global_index.value()), + cast_set(global_index.value()), -1, + slot_ref->data_type(), slot_ref->column_name()); + RETURN_IF_ERROR(expr->get()->prepare(nullptr, RowDescriptor(), nullptr)); + return Status::OK(); + } + auto children = (*expr)->children(); + for (auto& child : children) { + if (child == nullptr) { + continue; + } + RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&child, slot_id_to_global_index)); + } + (*expr)->set_children(std::move(children)); + return Status::OK(); +} + +} // namespace + +#ifdef BE_TEST +FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile, + std::unique_ptr table_reader) + : Scanner(state, profile), _table_reader(std::move(table_reader)) {} + +Status FileScannerV2::TEST_validate_scan_range(const TFileScanRangeParams& params, + const TFileRangeDesc& range) { + return _validate_scan_range(params, range); +} + +Status FileScannerV2::TEST_to_file_format(TFileFormatType::type format_type, + format::FileFormat* file_format) { + return _to_file_format(format_type, file_format); +} + +bool FileScannerV2::TEST_is_partition_slot(const TFileScanSlotInfo& slot_info, + const std::string& column_name) { + return is_partition_slot(slot_info, column_name); +} + +bool FileScannerV2::TEST_is_data_file_slot(const TFileScanSlotInfo& slot_info, + const std::string& column_name) { + return is_data_file_slot(slot_info, column_name); +} + +Status FileScannerV2::TEST_rewrite_slot_refs_to_global_index( + VExprSPtr* expr, + const std::unordered_map& slot_id_to_global_index) { + return rewrite_slot_refs_to_global_index(expr, slot_id_to_global_index); +} + +FileScannerV2::RealtimeCounterDeltas FileScannerV2::TEST_collect_realtime_counter_deltas( + const io::FileReaderStats& file_reader_stats, + const io::FileCacheStatistics& file_cache_statistics, + UncachedReaderBytesStorage uncached_reader_bytes_storage, int64_t* last_read_bytes, + int64_t* last_read_rows, int64_t* last_bytes_read_from_local, + int64_t* last_bytes_read_from_remote) { + return _collect_realtime_counter_deltas(file_reader_stats, file_cache_statistics, + uncached_reader_bytes_storage, last_read_bytes, + last_read_rows, last_bytes_read_from_local, + last_bytes_read_from_remote); +} + +void FileScannerV2::TEST_report_file_cache_profile( + RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) { + _report_file_cache_profile(profile, file_cache_statistics); +} + +bool FileScannerV2::TEST_should_skip_not_found(const Status& status, bool ignore_not_found) { + return _should_skip_not_found(status, ignore_not_found); +} +#endif + +bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFileRangeDesc& range) { + const auto format_type = get_range_format_type(params, range); + if (format_type == TFileFormatType::FORMAT_PARQUET || + format_type == TFileFormatType::FORMAT_ORC) { + return is_supported_table_format(range); + } else if (format_type == TFileFormatType::FORMAT_ARROW) { + return is_supported_arrow_table_format(range); + } else if (format_type == TFileFormatType::FORMAT_JNI) { + return is_supported_jni_table_format(range); + } else if (is_csv_format(format_type) || is_text_format(format_type) || + is_json_format(format_type) || is_native_format(format_type)) { + return is_supported_table_format(range); + } else { + LOG(WARNING) << "Unsupported file format type " << format_type << " for file scanner v2"; + return false; + } +} + +Status FileScannerV2::_validate_scan_range(const TFileScanRangeParams& params, + const TFileRangeDesc& range) { + if (!is_supported(params, range)) { + return Status::NotSupported( + "FileScannerV2 does not support table format {} with file format {}", + table_format_name(range), to_string(get_range_format_type(params, range))); + } + return Status::OK(); +} + +FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_state, int64_t limit, + std::shared_ptr split_source, + RuntimeProfile* profile, ShardedKVCache* kv_cache, + const std::unordered_map* colname_to_slot_id) + : Scanner(state, local_state, limit, profile), + _split_source(std::move(split_source)), + _kv_cache(kv_cache) { + (void)colname_to_slot_id; + if (state->get_query_ctx() != nullptr && + state->get_query_ctx()->file_scan_range_params_map.count(local_state->parent_id()) > 0) { + _params = &(state->get_query_ctx()->file_scan_range_params_map[local_state->parent_id()]); + } else { + _params = _split_source->get_params(); + } +} + +Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { + RETURN_IF_ERROR(Scanner::init(state, conjuncts)); + _get_block_timer = + ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerV2GetBlockTime", 1); + _not_found_file_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), + "NotFoundFileNum", TUnit::UNIT, 1); + _file_counter = + ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT, 1); + _file_read_bytes_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), + "FileReadBytes", TUnit::BYTES, 1); + _file_read_calls_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), + "FileReadCalls", TUnit::UNIT, 1); + _file_read_time_counter = + ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileReadTime", 1); + _adaptive_batch_predicted_rows_counter = ADD_COUNTER_WITH_LEVEL( + _local_state->scanner_profile(), "AdaptiveBatchPredictedRows", TUnit::UNIT, 1); + _adaptive_batch_actual_bytes_counter = ADD_COUNTER_WITH_LEVEL( + _local_state->scanner_profile(), "AdaptiveBatchActualBytes", TUnit::BYTES, 1); + _adaptive_batch_probe_count_counter = ADD_COUNTER_WITH_LEVEL( + _local_state->scanner_profile(), "AdaptiveBatchProbeCount", TUnit::UNIT, 1); + _file_cache_statistics = std::make_unique(); + _file_reader_stats = std::make_unique(); + RETURN_IF_ERROR(_init_io_ctx()); + _io_ctx->file_cache_stats = _file_cache_statistics.get(); + _io_ctx->file_reader_stats = _file_reader_stats.get(); + _io_ctx->is_disposable = _state->query_options().disable_file_cache; + return Status::OK(); +} + +Status FileScannerV2::_open_impl(RuntimeState* state) { + RETURN_IF_CANCELLED(state); + RETURN_IF_ERROR(Scanner::_open_impl(state)); + RETURN_IF_ERROR(_get_next_scan_range(&_first_scan_range)); + if (_first_scan_range) { + RETURN_IF_ERROR(_create_table_reader_for_format(_current_range, &_table_reader)); + DORIS_CHECK(_table_reader != nullptr); + RETURN_IF_ERROR(_init_expr_ctxes()); + RETURN_IF_ERROR(_init_table_reader(_current_range)); + } + return Status::OK(); +} + +Status FileScannerV2::_get_next_scan_range(bool* has_next) { + DORIS_CHECK(has_next != nullptr); + RETURN_IF_ERROR(_split_source->get_next(has_next, &_current_range)); + if (*has_next) { + RETURN_IF_ERROR(_validate_scan_range(*_params, _current_range)); + } + return Status::OK(); +} + +Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* eof) { + while (true) { + RETURN_IF_CANCELLED(state); + if (!_has_prepared_split) { + RETURN_IF_ERROR(_prepare_next_split(eof)); + if (*eof) { + return Status::OK(); + } + } + + { + SCOPED_TIMER(_get_block_timer); + if (_should_run_adaptive_batch_size()) { + _table_reader->set_batch_size(_predict_reader_batch_rows()); + } + const auto status = _table_reader->get_block(block, eof); + if (_should_skip_not_found(status, config::ignore_not_found_file_in_external_table)) { + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_not_found_file_counter, 1); + _state->update_num_finished_scan_range(1); + _has_prepared_split = false; + block->clear_column_data(cast_set(_projected_columns.size())); + *eof = false; + continue; + } + RETURN_IF_ERROR(status); + } + if (*eof) { + _state->update_num_finished_scan_range(1); + _has_prepared_split = false; + *eof = false; + continue; + } + _update_adaptive_batch_size(*block); + return Status::OK(); + } +} + +Status FileScannerV2::_prepare_next_split(bool* eos) { + while (true) { + bool has_next = _first_scan_range; + if (!_first_scan_range) { + RETURN_IF_ERROR(_get_next_scan_range(&has_next)); + } + _first_scan_range = false; + if (!has_next || _should_stop) { + *eos = true; + return Status::OK(); + } + DORIS_CHECK(_table_reader != nullptr); + _current_range_path = _current_range.path; + + const auto format_type = get_range_format_type(*_params, _current_range); + _init_adaptive_batch_size_state(format_type); + if (_should_run_adaptive_batch_size()) { + // JNI readers open eagerly in prepare_split(). Seed the probe size first so readers + // such as Paimon also use it for their first physical read batch. + _table_reader->set_batch_size(_predict_reader_batch_rows()); + } + std::map partition_values; + RETURN_IF_ERROR(_generate_partition_values(_current_range, &partition_values)); + const auto status = + _prepare_table_reader_split(_current_range, std::move(partition_values)); + if (_should_skip_not_found(status, config::ignore_not_found_file_in_external_table)) { + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_not_found_file_counter, 1); + _state->update_num_finished_scan_range(1); + continue; + } + RETURN_IF_ERROR(status); + if (_table_reader->current_split_pruned()) { + _state->update_num_finished_scan_range(1); + continue; + } + COUNTER_UPDATE(_file_counter, 1); + _has_prepared_split = true; + *eos = false; + return Status::OK(); + } +} + +Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { + const auto format_type = get_range_format_type(*_params, range); + format::FileFormat file_format; + RETURN_IF_ERROR(_to_file_format(format_type, &file_format)); + DORIS_CHECK(_table_reader != nullptr); + + VExprContextSPtrs table_conjuncts; + RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); + RETURN_IF_ERROR(_table_reader->init({ + .projected_columns = _projected_columns, + .conjuncts = std::move(table_conjuncts), + .format = file_format, + .scan_params = const_cast(_params), + .io_ctx = _io_ctx, + .runtime_state = _state, + .scanner_profile = _local_state->scanner_profile(), + .file_slot_descs = &_file_slot_descs, + .push_down_agg_type = _local_state->get_push_down_agg_type(), + .condition_cache_digest = _local_state->get_condition_cache_digest(), + })); + return Status::OK(); +} + +Status FileScannerV2::_create_table_reader_for_format( + const TFileRangeDesc& range, std::unique_ptr* reader) const { + DORIS_CHECK(reader != nullptr); + const auto table_format = table_format_name(range); + if (table_format == "NotSet" || table_format == "tvf") { + *reader = std::make_unique(); + } else if (table_format == "hive") { + *reader = format::hive::HiveReader::create_unique(); + } else if (table_format == "iceberg") { + if (get_range_format_type(*_params, range) == TFileFormatType::FORMAT_JNI) { + *reader = std::make_unique(); + } else { + *reader = std::make_unique(); + } + } else if (table_format == "paimon") { + *reader = std::make_unique(); + } else if (table_format == "hudi") { + *reader = std::make_unique(); + } else if (table_format == "jdbc") { + *reader = std::make_unique(); + } else if (table_format == "max_compute") { + const auto* mc_desc = + static_cast(_output_tuple_desc->table_desc()); + RETURN_IF_ERROR(mc_desc->init_status()); + *reader = std::make_unique(mc_desc); + } else if (table_format == "trino_connector") { + *reader = std::make_unique(); + } else if (table_format == "remote_doris") { + *reader = std::make_unique(); + } else { + return Status::NotSupported("FileScannerV2 does not support table format {}", table_format); + } + return Status::OK(); +} + +Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range, + std::map partition_values) { + format::FileFormat current_split_format; + RETURN_IF_ERROR(_to_file_format(get_range_format_type(*_params, range), ¤t_split_format)); + VExprContextSPtrs conjuncts; + RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts)); + VExprContextSPtrs partition_prune_conjuncts; + if (_state->query_options().enable_runtime_filter_partition_prune) { + RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts)); + } + RETURN_IF_ERROR(_table_reader->prepare_split({ + .partition_values = std::move(partition_values), + .conjuncts = std::move(conjuncts), + .partition_prune_conjuncts = std::move(partition_prune_conjuncts), + // A metadata COUNT split may span scheduler turns. Do not enter that irreversible + // synthetic-row path while a runtime filter can still arrive between batches. + .all_runtime_filters_applied = _applied_rf_num == _total_rf_num, + .cache = _kv_cache, + .current_range = range, + .current_split_format = current_split_format, + .global_rowid_context = _create_global_rowid_context(range), + })); + return Status::OK(); +} + +bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not_found) { + return ignore_not_found && status.is(); +} + +bool FileScannerV2::_should_enable_file_meta_cache() const { + return ExecEnv::GetInstance()->file_meta_cache()->enabled() && + _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; +} + +std::optional FileScannerV2::_create_global_rowid_context( + const TFileRangeDesc& range) const { + if (!_need_global_rowid_column) { + return std::nullopt; + } + auto& id_file_map = _state->get_id_file_map(); + DORIS_CHECK(id_file_map != nullptr); + const auto file_id = id_file_map->get_file_mapping_id( + std::make_shared(_local_state->cast().parent_id(), + range, _should_enable_file_meta_cache())); + return format::GlobalRowIdContext { + .version = IdManager::ID_VERSION, + .backend_id = BackendOptions::get_backend_id(), + .file_id = file_id, + }; +} + +Status FileScannerV2::_generate_partition_values( + const TFileRangeDesc& range, std::map* partition_values) const { + DORIS_CHECK(partition_values != nullptr); + partition_values->clear(); + if (!range.__isset.columns_from_path_keys || !range.__isset.columns_from_path) { + return Status::OK(); + } + DORIS_CHECK(range.columns_from_path_keys.size() == range.columns_from_path.size()); + for (size_t idx = 0; idx < range.columns_from_path_keys.size(); ++idx) { + const auto& key = range.columns_from_path_keys[idx]; + const auto it = _partition_slot_descs.find(key); + if (it == _partition_slot_descs.end()) { + continue; + } + const auto& value = range.columns_from_path[idx]; + const bool is_null = range.__isset.columns_from_path_is_null && + idx < range.columns_from_path_is_null.size() && + range.columns_from_path_is_null[idx]; + Field field; + DORIS_CHECK(it->second.slot_desc != nullptr); + RETURN_IF_ERROR(_parse_partition_value(it->second.slot_desc, value, is_null, &field)); + partition_values->emplace(it->second.canonical_name, std::move(field)); + } + return Status::OK(); +} + +Status FileScannerV2::_parse_partition_value(const SlotDescriptor* slot_desc, + const std::string& value, bool is_null, + Field* field) const { + DORIS_CHECK(slot_desc != nullptr); + DORIS_CHECK(field != nullptr); + if (is_null) { + *field = Field::create_field(Null()); + return Status::OK(); + } + const auto data_type = remove_nullable(slot_desc->get_data_type_ptr()); + auto column = data_type->create_column(); + auto serde = data_type->get_serde(); + DataTypeSerDe::FormatOptions options; + options.converted_from_string = true; + StringRef ref(value.data(), value.size()); + RETURN_IF_ERROR(serde->from_string(ref, *column, options)); + DORIS_CHECK(column->size() == 1); + *field = (*column)[0]; + return Status::OK(); +} + +Status FileScannerV2::_init_expr_ctxes() { + _slot_id_to_desc.clear(); + _slot_id_to_global_index.clear(); + _partition_slot_descs.clear(); + _file_slot_descs.clear(); + for (const auto* slot_desc : _output_tuple_desc->slots()) { + _slot_id_to_desc.emplace(slot_desc->id(), slot_desc); + } + DORIS_CHECK(_table_reader != nullptr); + RETURN_IF_ERROR(_build_projected_columns(*_table_reader)); + return Status::OK(); +} + +Status FileScannerV2::_build_projected_columns(const format::TableReader& table_reader) { + _projected_columns.clear(); + _projected_columns.reserve(_params->required_slots.size()); + _need_global_rowid_column = false; + format::ProjectedColumnBuildContext build_context { + .scan_params = _params, + .range = &_current_range, + .runtime_state = _state, + }; + + for (size_t slot_idx = 0; slot_idx < _params->required_slots.size(); ++slot_idx) { + const auto& slot_info = _params->required_slots[slot_idx]; + const auto it = _slot_id_to_desc.find(slot_info.slot_id); + if (it == _slot_id_to_desc.end()) { + return Status::InternalError("Unknown source slot descriptor, slot_id={}", + slot_info.slot_id); + } + auto column = _build_table_column(it->second); + if (column.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) { + _need_global_rowid_column = true; + } + RETURN_IF_ERROR(_build_default_expr(slot_info, &column.default_expr)); + build_context.schema_column.reset(); + RETURN_IF_ERROR(table_reader.annotate_projected_column(slot_info, &build_context, &column)); + // Build nested children from access paths generated by the slot's access-path + // expressions. A projected column can therefore contain only a subset of the schema + // column's nested children. + RETURN_IF_ERROR(AccessPathParser::build_nested_children( + &column, it->second, + build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr)); + if (is_partition_slot(slot_info, column.name)) { + column.is_partition_key = true; + _partition_slot_descs.emplace( + column.name, + PartitionSlotInfo {.slot_desc = it->second, .canonical_name = column.name}); + for (const auto& alias : column.name_mapping) { + _partition_slot_descs.emplace( + alias, + PartitionSlotInfo {.slot_desc = it->second, .canonical_name = column.name}); + } + } else if (is_data_file_slot(slot_info, column.name)) { + _file_slot_descs.push_back(const_cast(it->second)); + } + const auto global_index = format::GlobalIndex(slot_idx); + _slot_id_to_global_index.emplace(slot_info.slot_id, global_index); + _projected_columns.push_back(std::move(column)); + } + RETURN_IF_ERROR(table_reader.validate_projected_columns(build_context)); + return Status::OK(); +} + +Status FileScannerV2::_build_default_expr(const TFileScanSlotInfo& slot_info, + VExprContextSPtr* ctx) const { + DORIS_CHECK(ctx != nullptr); + if (_params->__isset.default_value_of_src_slot) { + const auto it = _params->default_value_of_src_slot.find(slot_info.slot_id); + if (it != _params->default_value_of_src_slot.end() && !it->second.nodes.empty()) { + return VExpr::create_expr_tree(it->second, *ctx); + } + } + return Status::OK(); +} + +format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor* slot_desc) { + DORIS_CHECK(slot_desc != nullptr); + format::ColumnDefinition column; + // TODO(gabriel): why always BY_NAME here? + column.identifier = Field::create_field(slot_desc->col_name()); + column.name = slot_desc->col_name(); + column.type = slot_desc->get_data_type_ptr(); + return column; +} + +Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const { + DORIS_CHECK(conjuncts != nullptr); + conjuncts->clear(); + conjuncts->reserve(_conjuncts.size()); + for (const auto& conjunct : _conjuncts) { + VExprSPtr root; + RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); + RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&root, _slot_id_to_global_index)); + conjuncts->push_back(VExprContext::create_shared(std::move(root))); + } + return Status::OK(); +} + +TFileFormatType::type FileScannerV2::_get_current_format_type() const { + return get_range_format_type(*_params, _current_range); +} + +Status FileScannerV2::_to_file_format(TFileFormatType::type format_type, + format::FileFormat* file_format) { + DORIS_CHECK(file_format != nullptr); + switch (format_type) { + case TFileFormatType::FORMAT_PARQUET: + *file_format = format::FileFormat::PARQUET; + return Status::OK(); + case TFileFormatType::FORMAT_ORC: + *file_format = format::FileFormat::ORC; + return Status::OK(); + case TFileFormatType::FORMAT_JNI: + *file_format = format::FileFormat::JNI; + return Status::OK(); + case TFileFormatType::FORMAT_CSV_PLAIN: + case TFileFormatType::FORMAT_CSV_GZ: + case TFileFormatType::FORMAT_CSV_BZ2: + case TFileFormatType::FORMAT_CSV_LZ4FRAME: + case TFileFormatType::FORMAT_CSV_LZ4BLOCK: + case TFileFormatType::FORMAT_CSV_LZOP: + case TFileFormatType::FORMAT_CSV_DEFLATE: + case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK: + case TFileFormatType::FORMAT_PROTO: + *file_format = format::FileFormat::CSV; + return Status::OK(); + case TFileFormatType::FORMAT_TEXT: + *file_format = format::FileFormat::TEXT; + return Status::OK(); + case TFileFormatType::FORMAT_JSON: + *file_format = format::FileFormat::JSON; + return Status::OK(); + case TFileFormatType::FORMAT_NATIVE: + *file_format = format::FileFormat::NATIVE; + return Status::OK(); + case TFileFormatType::FORMAT_ARROW: + *file_format = format::FileFormat::ARROW; + return Status::OK(); + default: + return Status::NotSupported("FileScannerV2 does not support file format {}", + to_string(format_type)); + } +} + +Status FileScannerV2::_init_io_ctx() { + _io_ctx = std::make_shared(); + _io_ctx->query_id = &_state->query_id(); + return Status::OK(); +} + +void FileScannerV2::_reset_adaptive_batch_size_state() { + _block_size_predictor.reset(); + COUNTER_SET(_adaptive_batch_predicted_rows_counter, int64_t(0)); + COUNTER_SET(_adaptive_batch_actual_bytes_counter, int64_t(0)); +} + +void FileScannerV2::_init_adaptive_batch_size_state(TFileFormatType::type format_type) { + _reset_adaptive_batch_size_state(); + if (!_should_enable_adaptive_batch_size(format_type)) { + return; + } + + // V2 native file readers do not have reliable row-width hints before the first batch. Start + // every split with a small probe, then learn bytes-per-row from the materialized table block + // and keep later batches close to RuntimeState::preferred_block_size_bytes(). + _block_size_predictor = std::make_unique( + _state->preferred_block_size_bytes(), 0.0, ADAPTIVE_BATCH_INITIAL_PROBE_ROWS, + _state->batch_size()); +} + +bool FileScannerV2::_should_enable_adaptive_batch_size(TFileFormatType::type format_type) const { + if (!config::enable_adaptive_batch_size) { + return false; + } + switch (format_type) { + case TFileFormatType::FORMAT_PARQUET: + case TFileFormatType::FORMAT_ORC: + case TFileFormatType::FORMAT_CSV_PLAIN: + case TFileFormatType::FORMAT_CSV_GZ: + case TFileFormatType::FORMAT_CSV_BZ2: + case TFileFormatType::FORMAT_CSV_LZ4FRAME: + case TFileFormatType::FORMAT_CSV_LZ4BLOCK: + case TFileFormatType::FORMAT_CSV_LZOP: + case TFileFormatType::FORMAT_CSV_DEFLATE: + case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK: + case TFileFormatType::FORMAT_PROTO: + case TFileFormatType::FORMAT_TEXT: + case TFileFormatType::FORMAT_JSON: + case TFileFormatType::FORMAT_JNI: + return true; + default: + return false; + } +} + +bool FileScannerV2::_should_run_adaptive_batch_size() const { + // COUNT pushdown emits synthetic rows from file metadata and does not materialize file columns, + // so there is no useful row-width sample to learn from. + return _block_size_predictor != nullptr && + _local_state->get_push_down_agg_type() != TPushAggOp::type::COUNT; +} + +size_t FileScannerV2::_predict_reader_batch_rows() { + DORIS_CHECK(_block_size_predictor != nullptr); + // Before history exists this returns the probe row count; after update(), it returns roughly + // preferred_block_size_bytes / EWMA(bytes_per_row), capped by RuntimeState::batch_size(). + const size_t predicted_rows = _block_size_predictor->predict_next_rows(); + COUNTER_SET(_adaptive_batch_predicted_rows_counter, static_cast(predicted_rows)); + return predicted_rows; +} + +void FileScannerV2::_update_adaptive_batch_size(const Block& block) { + if (!_should_run_adaptive_batch_size()) { + return; + } + COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(block.bytes())); + if (block.rows() == 0) { + return; + } + // The sample is taken after TableReader has finalized file-local columns to table columns. + // This matches the memory shape seen by upstream operators and catches very wide nested + // columns, such as map/string payloads, after the first probe batch. + if (!_block_size_predictor->has_history()) { + COUNTER_UPDATE(_adaptive_batch_probe_count_counter, 1); + } + _block_size_predictor->update(block); +} + +Status FileScannerV2::close(RuntimeState* state) { + if (_is_closed) { + return Status::OK(); + } + if (_table_reader != nullptr) { + const auto close_status = _table_reader->close(); + if (!close_status.ok()) { + return close_status; + } + _report_condition_cache_profile(); + _table_reader.reset(); + } + return Scanner::close(state); +} + +void FileScannerV2::try_stop() { + Scanner::try_stop(); + if (_io_ctx) { + _io_ctx->should_stop = true; + } +} + +void FileScannerV2::update_realtime_counters() { + if (_file_reader_stats == nullptr) { + return; + } + DORIS_CHECK(_file_cache_statistics != nullptr); + const int64_t bytes_read = cast_set(_file_reader_stats->read_bytes); + auto* local_state = static_cast(_local_state); + const auto file_type = + _current_range.__isset.file_type + ? _current_range.file_type + : (_params != nullptr && _params->__isset.file_type ? _params->file_type + : TFileType::FILE_LOCAL); + const auto deltas = _collect_realtime_counter_deltas( + *_file_reader_stats, *_file_cache_statistics, _uncached_reader_bytes_storage(file_type), + &_last_read_bytes, &_last_read_rows, &_last_bytes_read_from_local, + &_last_bytes_read_from_remote); + + COUNTER_UPDATE(local_state->_scan_bytes, deltas.scan_bytes); + COUNTER_UPDATE(local_state->_scan_rows, deltas.scan_rows); + + _state->get_query_ctx()->resource_ctx()->io_context()->update_scan_rows(deltas.scan_rows); + _state->get_query_ctx()->resource_ctx()->io_context()->update_scan_bytes(deltas.scan_bytes); + _state->get_query_ctx()->resource_ctx()->io_context()->update_scan_bytes_from_local_storage( + deltas.scan_bytes_from_local_storage); + _state->get_query_ctx()->resource_ctx()->io_context()->update_scan_bytes_from_remote_storage( + deltas.scan_bytes_from_remote_storage); + + COUNTER_SET(_file_read_bytes_counter, bytes_read); + COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); + COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + + DorisMetrics::instance()->query_scan_bytes->increment(deltas.scan_bytes); + DorisMetrics::instance()->query_scan_rows->increment(deltas.scan_rows); + DorisMetrics::instance()->query_scan_bytes_from_local->increment( + deltas.scan_bytes_from_local_storage); + DorisMetrics::instance()->query_scan_bytes_from_remote->increment( + deltas.scan_bytes_from_remote_storage); +} + +FileScannerV2::RealtimeCounterDeltas FileScannerV2::_collect_realtime_counter_deltas( + const io::FileReaderStats& file_reader_stats, + const io::FileCacheStatistics& file_cache_statistics, + UncachedReaderBytesStorage uncached_reader_bytes_storage, int64_t* last_read_bytes, + int64_t* last_read_rows, int64_t* last_bytes_read_from_local, + int64_t* last_bytes_read_from_remote) { + DORIS_CHECK(last_read_bytes != nullptr); + DORIS_CHECK(last_read_rows != nullptr); + DORIS_CHECK(last_bytes_read_from_local != nullptr); + DORIS_CHECK(last_bytes_read_from_remote != nullptr); + + const int64_t read_bytes = cast_set(file_reader_stats.read_bytes); + const int64_t read_rows = cast_set(file_reader_stats.read_rows); + const int64_t bytes_read_from_local = file_cache_statistics.bytes_read_from_local; + const int64_t bytes_read_from_remote = file_cache_statistics.bytes_read_from_remote; + DORIS_CHECK(read_bytes >= *last_read_bytes); + DORIS_CHECK(read_rows >= *last_read_rows); + DORIS_CHECK(bytes_read_from_local >= *last_bytes_read_from_local); + DORIS_CHECK(bytes_read_from_remote >= *last_bytes_read_from_remote); + + RealtimeCounterDeltas deltas; + deltas.scan_rows = read_rows - *last_read_rows; + deltas.scan_bytes = read_bytes - *last_read_bytes; + // Peer cache is a known cache source, but it is not remote object storage. + const bool has_cache_source_stats = file_cache_statistics.num_local_io_total != 0 || + file_cache_statistics.num_remote_io_total != 0 || + file_cache_statistics.num_peer_io_total != 0 || + bytes_read_from_local != 0 || bytes_read_from_remote != 0 || + file_cache_statistics.bytes_read_from_peer != 0; + if (!has_cache_source_stats) { + switch (uncached_reader_bytes_storage) { + case UncachedReaderBytesStorage::LOCAL: + deltas.scan_bytes_from_local_storage = deltas.scan_bytes; + break; + case UncachedReaderBytesStorage::REMOTE: + deltas.scan_bytes_from_remote_storage = deltas.scan_bytes; + break; + case UncachedReaderBytesStorage::NONE: + break; + } + } else { + deltas.scan_bytes_from_local_storage = bytes_read_from_local - *last_bytes_read_from_local; + deltas.scan_bytes_from_remote_storage = + bytes_read_from_remote - *last_bytes_read_from_remote; + } + + *last_read_bytes = read_bytes; + *last_read_rows = read_rows; + *last_bytes_read_from_local = bytes_read_from_local; + *last_bytes_read_from_remote = bytes_read_from_remote; + return deltas; +} + +FileScannerV2::UncachedReaderBytesStorage FileScannerV2::_uncached_reader_bytes_storage( + TFileType::type file_type) { + switch (file_type) { + case TFileType::FILE_LOCAL: + return UncachedReaderBytesStorage::LOCAL; + case TFileType::FILE_STREAM: + return UncachedReaderBytesStorage::NONE; + case TFileType::FILE_BROKER: + case TFileType::FILE_S3: + case TFileType::FILE_HDFS: + case TFileType::FILE_NET: + case TFileType::FILE_HTTP: + return UncachedReaderBytesStorage::REMOTE; + } + DORIS_CHECK(false) << "unknown file type: " << file_type; + return UncachedReaderBytesStorage::NONE; +} + +void FileScannerV2::_collect_profile_before_close() { + _report_file_reader_predicate_filtered_rows(); + Scanner::_collect_profile_before_close(); + if (config::enable_file_cache && _state->query_options().enable_file_cache && + _profile != nullptr) { + _report_file_cache_profile(_profile, *_file_cache_statistics); + _state->get_query_ctx()->resource_ctx()->io_context()->update_bytes_write_into_cache( + _file_cache_statistics->bytes_write_into_cache); + } + if (_file_reader_stats != nullptr) { + COUNTER_SET(_file_read_bytes_counter, cast_set(_file_reader_stats->read_bytes)); + COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); + COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + } + // Query profiles can be collected before Scanner::close() runs. Publish condition-cache + // counters here as well, using deltas so this method and close() cannot double count. + _report_condition_cache_profile(); +} + +void FileScannerV2::_report_file_cache_profile( + RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) { + io::FileCacheProfileReporter cache_profile(profile); + cache_profile.update(&file_cache_statistics); +} + +bool FileScannerV2::_should_update_load_counters() const { + if (_is_load) { + return true; + } + // TVF based loads (e.g. http_stream, group commit relay) plan the load source as a + // tvf query scan without src tuple desc, so _is_load is false. But rows filtered by + // the load's WHERE clause still need to be reported as unselected rows. FILE_STREAM + // is only reachable from such load entries, never from normal queries, so use it to + // identify these scanners. + return (_params != nullptr && _params->__isset.file_type && + _params->file_type == TFileType::FILE_STREAM) || + (_current_range.__isset.file_type && _current_range.file_type == TFileType::FILE_STREAM); +} + +void FileScannerV2::_report_file_reader_predicate_filtered_rows() { + const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0; + const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows; + if (filtered_delta > 0) { + // File readers can evaluate localized conjuncts before a block reaches Scanner. Count + // those rows as scanner-level unselected rows so load statistics stay identical no matter + // whether a predicate is pushed down or evaluated by Scanner::_filter_output_block(). + _counter.num_rows_unselected += filtered_delta; + _reported_predicate_filtered_rows = filtered_rows; + } +} + +void FileScannerV2::_report_condition_cache_profile() { + auto* local_state = static_cast(_local_state); + const int64_t hit_count = + _table_reader != nullptr ? _table_reader->condition_cache_hit_count() : 0; + const int64_t hit_delta = hit_count - _reported_condition_cache_hit_count; + if (hit_delta > 0) { + COUNTER_UPDATE(local_state->_condition_cache_hit_counter, hit_delta); + _reported_condition_cache_hit_count = hit_count; + } + const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->condition_cache_filtered_rows : 0; + const int64_t filtered_delta = filtered_rows - _reported_condition_cache_filtered_rows; + if (filtered_delta > 0) { + COUNTER_UPDATE(local_state->_condition_cache_filtered_rows_counter, filtered_delta); + _reported_condition_cache_filtered_rows = filtered_rows; + } +} + +} // namespace doris diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h new file mode 100644 index 00000000000000..2bddc5d5e69e6c --- /dev/null +++ b/be/src/exec/scan/file_scanner_v2.h @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/factory_creator.h" +#include "common/status.h" +#include "core/block/block.h" +#include "exec/operator/file_scan_operator.h" +#include "exec/scan/scanner.h" +#include "exec/scan/split_source_connector.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/column_mapper.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/Descriptors_types.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "storage/segment/adaptive_block_size_predictor.h" + +namespace doris { + +class RuntimeState; +class SlotDescriptor; +class TFileRangeDesc; +class TFileScanRangeParams; +class ShardedKVCache; + +class FileScannerV2 final : public Scanner { + ENABLE_FACTORY_CREATOR(FileScannerV2); + +public: + static constexpr const char* NAME = "FileScannerV2"; + static constexpr size_t ADAPTIVE_BATCH_INITIAL_PROBE_ROWS = 32; + + struct RealtimeCounterDeltas { + int64_t scan_rows = 0; + int64_t scan_bytes = 0; + int64_t scan_bytes_from_local_storage = 0; + int64_t scan_bytes_from_remote_storage = 0; + }; + + enum class UncachedReaderBytesStorage { LOCAL, REMOTE, NONE }; + + static bool is_supported(const TFileScanRangeParams& params, const TFileRangeDesc& range); +#ifdef BE_TEST + FileScannerV2(RuntimeState* state, RuntimeProfile* profile, + std::unique_ptr table_reader); + static Status TEST_validate_scan_range(const TFileScanRangeParams& params, + const TFileRangeDesc& range); + static Status TEST_to_file_format(TFileFormatType::type format_type, + format::FileFormat* file_format); + static bool TEST_is_partition_slot(const TFileScanSlotInfo& slot_info, + const std::string& column_name); + static bool TEST_is_data_file_slot(const TFileScanSlotInfo& slot_info, + const std::string& column_name); + static Status TEST_rewrite_slot_refs_to_global_index( + VExprSPtr* expr, + const std::unordered_map& slot_id_to_global_index); + static RealtimeCounterDeltas TEST_collect_realtime_counter_deltas( + const io::FileReaderStats& file_reader_stats, + const io::FileCacheStatistics& file_cache_statistics, + UncachedReaderBytesStorage uncached_reader_bytes_storage, int64_t* last_read_bytes, + int64_t* last_read_rows, int64_t* last_bytes_read_from_local, + int64_t* last_bytes_read_from_remote); + static void TEST_report_file_cache_profile( + RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); + static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); +#endif + + FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, + std::shared_ptr split_source, RuntimeProfile* profile, + ShardedKVCache* kv_cache, + const std::unordered_map* colname_to_slot_id); + + Status init(RuntimeState* state, const VExprContextSPtrs& conjuncts) override; + Status _open_impl(RuntimeState* state) override; + Status close(RuntimeState* state) override; + void try_stop() override; + std::string get_name() override { return FileScannerV2::NAME; } + std::string get_current_scan_range_name() override { return _current_range_path; } + void update_realtime_counters() override; + +protected: + Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override; + void _collect_profile_before_close() override; + bool _should_update_load_counters() const override; + +private: + static Status _validate_scan_range(const TFileScanRangeParams& params, + const TFileRangeDesc& range); + Status _get_next_scan_range(bool* has_next); + TFileFormatType::type _get_current_format_type() const; + Status _init_io_ctx(); + Status _init_expr_ctxes(); + Status _prepare_next_split(bool* eos); + Status _init_table_reader(const TFileRangeDesc& range); + Status _create_table_reader_for_format(const TFileRangeDesc& range, + std::unique_ptr* reader) const; + Status _prepare_table_reader_split(const TFileRangeDesc& range, + std::map partition_values); + static bool _should_skip_not_found(const Status& status, bool ignore_not_found); + bool _should_enable_file_meta_cache() const; + std::optional _create_global_rowid_context( + const TFileRangeDesc& range) const; + Status _generate_partition_values(const TFileRangeDesc& range, + std::map* partition_values) const; + Status _parse_partition_value(const SlotDescriptor* slot_desc, const std::string& value, + bool is_null, Field* field) const; + Status _build_projected_columns(const format::TableReader& table_reader); + Status _build_default_expr(const TFileScanSlotInfo& slot_info, VExprContextSPtr* ctx) const; + static format::ColumnDefinition _build_table_column(const SlotDescriptor* slot_desc); + Status _build_table_conjuncts(VExprContextSPtrs* conjuncts) const; + static Status _to_file_format(TFileFormatType::type format_type, + format::FileFormat* file_format); + void _reset_adaptive_batch_size_state(); + void _init_adaptive_batch_size_state(TFileFormatType::type format_type); + bool _should_enable_adaptive_batch_size(TFileFormatType::type format_type) const; + bool _should_run_adaptive_batch_size() const; + size_t _predict_reader_batch_rows(); + void _update_adaptive_batch_size(const Block& block); + static RealtimeCounterDeltas _collect_realtime_counter_deltas( + const io::FileReaderStats& file_reader_stats, + const io::FileCacheStatistics& file_cache_statistics, + UncachedReaderBytesStorage uncached_reader_bytes_storage, int64_t* last_read_bytes, + int64_t* last_read_rows, int64_t* last_bytes_read_from_local, + int64_t* last_bytes_read_from_remote); + static UncachedReaderBytesStorage _uncached_reader_bytes_storage(TFileType::type file_type); + static void _report_file_cache_profile(RuntimeProfile* profile, + const io::FileCacheStatistics& file_cache_statistics); + void _report_file_reader_predicate_filtered_rows(); + void _report_condition_cache_profile(); + + struct PartitionSlotInfo { + const SlotDescriptor* slot_desc = nullptr; + std::string canonical_name; + }; + + const TFileScanRangeParams* _params = nullptr; + std::shared_ptr _split_source; + bool _first_scan_range = false; + bool _has_prepared_split = false; + TFileRangeDesc _current_range; + std::string _current_range_path; + + std::unique_ptr _table_reader; + std::vector _projected_columns; + // File formats without embedded schema, such as CSV, still need the FE slot descriptors in + // file-column order. This mirrors old FileScanner::_file_slot_descs and is passed only to + // readers that cannot derive their schema from file metadata. + std::vector _file_slot_descs; + bool _need_global_rowid_column = false; + std::unordered_map _slot_id_to_desc; + std::unordered_map _slot_id_to_global_index; + std::unordered_map _partition_slot_descs; + + std::unique_ptr _file_cache_statistics; + std::unique_ptr _file_reader_stats; + std::shared_ptr _io_ctx; + ShardedKVCache* _kv_cache = nullptr; + + RuntimeProfile::Counter* _get_block_timer = nullptr; + RuntimeProfile::Counter* _not_found_file_counter = nullptr; + RuntimeProfile::Counter* _file_counter = nullptr; + RuntimeProfile::Counter* _file_read_bytes_counter = nullptr; + RuntimeProfile::Counter* _file_read_calls_counter = nullptr; + RuntimeProfile::Counter* _file_read_time_counter = nullptr; + RuntimeProfile::Counter* _adaptive_batch_predicted_rows_counter = nullptr; + RuntimeProfile::Counter* _adaptive_batch_actual_bytes_counter = nullptr; + RuntimeProfile::Counter* _adaptive_batch_probe_count_counter = nullptr; + std::unique_ptr _block_size_predictor; + int64_t _reported_predicate_filtered_rows = 0; + int64_t _reported_condition_cache_hit_count = 0; + int64_t _reported_condition_cache_filtered_rows = 0; + int64_t _last_read_bytes = 0; + int64_t _last_read_rows = 0; + int64_t _last_bytes_read_from_local = 0; + int64_t _last_bytes_read_from_remote = 0; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/vhive_partition_writer.cpp b/be/src/exec/sink/writer/vhive_partition_writer.cpp index 4bfc728c8c9e16..4a358f728cbbf5 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/exec/sink/writer/vhive_partition_writer.cpp @@ -93,6 +93,8 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_ to_string(_hive_compress_type)); } } + // TODO: INT96 is kept for Hive 2/3 compatibility. Add an explicit option before + // changing the default Hive parquet timestamp encoding to standard logical types. ParquetFileOptions parquet_options = {parquet_compression_type, TParquetVersion::PARQUET_1_0, false, true}; _file_format_transformer = std::make_unique( diff --git a/be/src/exprs/expr_zonemap_filter.cpp b/be/src/exprs/expr_zonemap_filter.cpp index 19457beb37416b..95502a761b082a 100644 --- a/be/src/exprs/expr_zonemap_filter.cpp +++ b/be/src/exprs/expr_zonemap_filter.cpp @@ -31,6 +31,7 @@ #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "runtime/runtime_state.h" +#include "storage/index/bloom_filter/bloom_filter.h" namespace doris::expr_zonemap { namespace { @@ -51,8 +52,86 @@ bool value_in_range(const Field& value, const Field& min_value, const Field& max return value >= min_value && value <= max_value; } +bool dictionary_contains(const DictionaryEvalContext::SlotDictionary& dictionary, + const Field& value) { + return std::ranges::any_of(dictionary.values, [&](const Field& dictionary_value) { + return dictionary_value == value; + }); +} + +bool bloom_filter_may_contain(const BloomFilterEvalContext::SlotBloomFilter& slot_filter, + const Field& value) { + DORIS_CHECK(slot_filter.data_type != nullptr); + DORIS_CHECK(slot_filter.bloom_filter != nullptr); + const auto data_type = remove_nullable(slot_filter.data_type); + DORIS_CHECK(data_type != nullptr); + switch (data_type->get_primitive_type()) { + case TYPE_BOOLEAN: { + const bool typed_value = value.get(); + return slot_filter.bloom_filter->test_bytes(reinterpret_cast(&typed_value), + sizeof(typed_value)); + } + case TYPE_INT: { + const int32_t typed_value = value.get(); + return slot_filter.bloom_filter->test_bytes(reinterpret_cast(&typed_value), + sizeof(typed_value)); + } + case TYPE_BIGINT: { + const int64_t typed_value = value.get(); + return slot_filter.bloom_filter->test_bytes(reinterpret_cast(&typed_value), + sizeof(typed_value)); + } + case TYPE_FLOAT: { + const float typed_value = value.get(); + return slot_filter.bloom_filter->test_bytes(reinterpret_cast(&typed_value), + sizeof(typed_value)); + } + case TYPE_DOUBLE: { + const double typed_value = value.get(); + return slot_filter.bloom_filter->test_bytes(reinterpret_cast(&typed_value), + sizeof(typed_value)); + } + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: { + const auto& typed_value = value.get(); + return slot_filter.bloom_filter->test_bytes(typed_value.data(), typed_value.size()); + } + default: + return true; + } +} + +template +int single_slot_index(const VExprContextSPtr& ctx, Capability capability) { + DORIS_CHECK(ctx != nullptr); + const auto& root = ctx->root(); + DORIS_CHECK(root != nullptr); + if (!capability(root)) { + return -1; + } + + std::set slot_indexes; + root->collect_slot_column_ids(slot_indexes); + if (slot_indexes.size() != 1) { + return -1; + } + + return *slot_indexes.begin(); +} + } // namespace +const DictionaryEvalContext::SlotDictionary* DictionaryEvalContext::slot(int slot_index) const { + auto it = slots.find(slot_index); + return it == slots.end() ? nullptr : &it->second; +} + +const BloomFilterEvalContext::SlotBloomFilter* BloomFilterEvalContext::slot(int slot_index) const { + auto it = slots.find(slot_index); + return it == slots.end() ? nullptr : &it->second; +} + TExprNode create_texpr_node_from_hybrid_set_value(const void* data, const PrimitiveType& type, int precision, int scale) { if (is_string_type(type)) { @@ -235,24 +314,100 @@ ZoneMapFilterResult eval_in_zonemap(const ZoneMapEvalContext& ctx, const VExprSP return ZoneMapFilterResult::kNoMatch; } +ZoneMapFilterResult eval_eq_dictionary(const DictionaryEvalContext& ctx, + const SlotLiteral& slot_literal) { + auto dictionary = ctx.slot(slot_literal.slot_index); + if (dictionary == nullptr || dictionary->data_type == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + DORIS_CHECK(data_types_compatible(dictionary->data_type, slot_literal.slot_type)); + if (slot_literal.literal.is_null()) { + return ZoneMapFilterResult::kUnsupported; + } + return dictionary_contains(*dictionary, slot_literal.literal) ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; +} + +ZoneMapFilterResult eval_in_dictionary(const DictionaryEvalContext& ctx, const VExprSPtr& slot_expr, + bool is_not_in, const std::vector& values) { + if (is_not_in) { + return ZoneMapFilterResult::kUnsupported; + } + auto slot = std::dynamic_pointer_cast(slot_expr); + DORIS_CHECK(slot != nullptr); + auto dictionary = ctx.slot(slot->column_id()); + if (dictionary == nullptr || dictionary->data_type == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + DORIS_CHECK(data_types_compatible(dictionary->data_type, slot->data_type())); + if (values.empty()) { + return ZoneMapFilterResult::kNoMatch; + } + for (const auto& value : values) { + if (!value.is_null() && dictionary_contains(*dictionary, value)) { + return ZoneMapFilterResult::kMayMatch; + } + } + return ZoneMapFilterResult::kNoMatch; +} + +ZoneMapFilterResult eval_eq_bloom_filter(const BloomFilterEvalContext& ctx, + const SlotLiteral& slot_literal) { + auto slot_filter = ctx.slot(slot_literal.slot_index); + if (slot_filter == nullptr || slot_filter->data_type == nullptr || + slot_filter->bloom_filter == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + DORIS_CHECK(data_types_compatible(slot_filter->data_type, slot_literal.slot_type)); + if (slot_literal.literal.is_null()) { + return ZoneMapFilterResult::kUnsupported; + } + return bloom_filter_may_contain(*slot_filter, slot_literal.literal) + ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; +} + +ZoneMapFilterResult eval_in_bloom_filter(const BloomFilterEvalContext& ctx, + const VExprSPtr& slot_expr, bool is_not_in, + const std::vector& values) { + if (is_not_in) { + return ZoneMapFilterResult::kUnsupported; + } + auto slot = std::dynamic_pointer_cast(slot_expr); + DORIS_CHECK(slot != nullptr); + auto slot_filter = ctx.slot(slot->column_id()); + if (slot_filter == nullptr || slot_filter->data_type == nullptr || + slot_filter->bloom_filter == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + DORIS_CHECK(data_types_compatible(slot_filter->data_type, slot->data_type())); + if (values.empty()) { + return ZoneMapFilterResult::kNoMatch; + } + for (const auto& value : values) { + if (!value.is_null() && bloom_filter_may_contain(*slot_filter, value)) { + return ZoneMapFilterResult::kMayMatch; + } + } + return ZoneMapFilterResult::kNoMatch; +} + // Return the only slot ordinal referenced by a zonemap-evaluable expression. A negative result is // the conservative fallback marker for unsupported expressions, multi-slot expressions, or invalid // slot ordinals, so callers can skip schema-indexed zonemap pruning safely. int single_slot_zonemap_index(const VExprContextSPtr& ctx) { - DORIS_CHECK(ctx != nullptr); - const auto& root = ctx->root(); - DORIS_CHECK(root != nullptr); - if (!root->can_evaluate_zonemap_filter()) { - return -1; - } + return single_slot_index( + ctx, [](const VExprSPtr& expr) { return expr->can_evaluate_zonemap_filter(); }); +} - std::set slot_indexes; - root->collect_slot_column_ids(slot_indexes); - if (slot_indexes.size() != 1) { - return -1; - } +int single_slot_dictionary_index(const VExprContextSPtr& ctx) { + return single_slot_index( + ctx, [](const VExprSPtr& expr) { return expr->can_evaluate_dictionary_filter(); }); +} - return *slot_indexes.begin(); +int single_slot_bloom_filter_index(const VExprContextSPtr& ctx) { + return single_slot_index( + ctx, [](const VExprSPtr& expr) { return expr->can_evaluate_bloom_filter(); }); } bool is_expr_zonemap_filter_enabled(const RuntimeState* state) { diff --git a/be/src/exprs/expr_zonemap_filter.h b/be/src/exprs/expr_zonemap_filter.h index 2d8ff4d0c72207..a068a1e51477b8 100644 --- a/be/src/exprs/expr_zonemap_filter.h +++ b/be/src/exprs/expr_zonemap_filter.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -34,6 +35,10 @@ namespace doris { class HybridSetBase; class RuntimeState; class TExprNode; + +namespace segment_v2 { +class BloomFilter; +} // namespace segment_v2 } // namespace doris namespace doris::expr_zonemap { @@ -45,6 +50,31 @@ struct InZonemapMaterializedSet { Field max_value; }; +// Dictionary pruning evaluates file-level dictionary values, not row-level data. A kNoMatch result +// means no non-null dictionary entry can satisfy the expression, so the whole row group can be +// skipped only for dictionary-encoded columns whose dictionary contains all non-null values. +struct DictionaryEvalContext { + struct SlotDictionary { + DataTypePtr data_type; + std::vector values; + }; + + const SlotDictionary* slot(int slot_index) const; + std::map slots; +}; + +// Bloom-filter pruning can only disprove equality-style predicates. A kNoMatch result means every +// literal candidate required by the expression is definitely absent from the file bloom filter. +struct BloomFilterEvalContext { + struct SlotBloomFilter { + DataTypePtr data_type; + const segment_v2::BloomFilter* bloom_filter = nullptr; + }; + + const SlotBloomFilter* slot(int slot_index) const; + std::map slots; +}; + struct SlotLiteral { // Slot ordinal in the current expression binding. It is also the key used to look up the // corresponding reader-schema type and zone map from ZoneMapEvalContext. @@ -114,11 +144,33 @@ ZoneMapFilterResult eval_in_zonemap(const ZoneMapEvalContext& ctx, const VExprSP bool is_not_in, const std::vector& values, const Field& min_value, const Field& max_value); +ZoneMapFilterResult eval_eq_dictionary(const DictionaryEvalContext& ctx, + const SlotLiteral& slot_literal); + +ZoneMapFilterResult eval_in_dictionary(const DictionaryEvalContext& ctx, const VExprSPtr& slot_expr, + bool is_not_in, const std::vector& values); + +ZoneMapFilterResult eval_eq_bloom_filter(const BloomFilterEvalContext& ctx, + const SlotLiteral& slot_literal); + +ZoneMapFilterResult eval_in_bloom_filter(const BloomFilterEvalContext& ctx, + const VExprSPtr& slot_expr, bool is_not_in, + const std::vector& values); + // Return the only slot ordinal referenced by a zonemap-evaluable expression in its current // binding. Expressions that are unsupported by zonemap pruning, reference multiple slots, or use an // invalid negative slot ordinal return a negative value. int single_slot_zonemap_index(const VExprContextSPtr& ctx); +int single_slot_dictionary_index(const VExprContextSPtr& ctx); + +int single_slot_bloom_filter_index(const VExprContextSPtr& ctx); + bool is_expr_zonemap_filter_enabled(const RuntimeState* state); } // namespace doris::expr_zonemap + +namespace doris { +using DictionaryEvalContext = expr_zonemap::DictionaryEvalContext; +using BloomFilterEvalContext = expr_zonemap::BloomFilterEvalContext; +} // namespace doris diff --git a/be/src/exprs/function/function.cpp b/be/src/exprs/function/function.cpp index 6cec038864e8b7..c7b35e8260f209 100644 --- a/be/src/exprs/function/function.cpp +++ b/be/src/exprs/function/function.cpp @@ -413,5 +413,14 @@ ZoneMapFilterResult IFunctionBase::evaluate_zonemap_filter( return unsupported_zonemap_filter(ctx); } -#include "common/compile_check_end.h" +ZoneMapFilterResult IFunctionBase::evaluate_dictionary_filter( + const DictionaryEvalContext& ctx, const VExprSPtrs& function_arguments) const { + return ZoneMapFilterResult::kUnsupported; +} + +ZoneMapFilterResult IFunctionBase::evaluate_bloom_filter( + const BloomFilterEvalContext& ctx, const VExprSPtrs& function_arguments) const { + return ZoneMapFilterResult::kUnsupported; +} + } // namespace doris diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 171e889f79b21d..13d4bfd6bb573b 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -42,6 +42,7 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" #include "core/types.h" +#include "exprs/expr_zonemap_filter.h" #include "exprs/function_context.h" #include "exprs/vexpr_fwd.h" #include "storage/index/inverted/inverted_index_iterator.h" // IWYU pragma: keep @@ -236,6 +237,20 @@ class IFunctionBase { virtual bool can_evaluate_zonemap_filter(const VExprSPtrs& /*function_arguments*/) const { return false; } + + virtual ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx, const VExprSPtrs& function_arguments) const; + + virtual bool can_evaluate_dictionary_filter(const VExprSPtrs& /*function_arguments*/) const { + return false; + } + + virtual ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx, + const VExprSPtrs& function_arguments) const; + + virtual bool can_evaluate_bloom_filter(const VExprSPtrs& /*function_arguments*/) const { + return false; + } }; using FunctionBasePtr = std::shared_ptr; @@ -511,6 +526,24 @@ class DefaultFunction final : public IFunctionBase { return function->can_evaluate_zonemap_filter(function_arguments); } + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx, const VExprSPtrs& function_arguments) const override { + return function->evaluate_dictionary_filter(ctx, function_arguments); + } + + bool can_evaluate_dictionary_filter(const VExprSPtrs& function_arguments) const override { + return function->can_evaluate_dictionary_filter(function_arguments); + } + + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx, + const VExprSPtrs& function_arguments) const override { + return function->evaluate_bloom_filter(ctx, function_arguments); + } + + bool can_evaluate_bloom_filter(const VExprSPtrs& function_arguments) const override { + return function->can_evaluate_bloom_filter(function_arguments); + } + private: std::shared_ptr function; DataTypes arguments; diff --git a/be/src/exprs/function/functions_comparison.h b/be/src/exprs/function/functions_comparison.h index 062e5d857e390b..b5643e1ca5ebc5 100644 --- a/be/src/exprs/function/functions_comparison.h +++ b/be/src/exprs/function/functions_comparison.h @@ -374,6 +374,26 @@ inline bool can_evaluate(const VExprSPtrs& arguments) { return true; } +inline bool can_evaluate_equality(const VExprSPtrs& arguments, Op op) { + return op == Op::EQ && can_evaluate(arguments); +} + +inline ZoneMapFilterResult evaluate_dictionary(const DictionaryEvalContext& ctx, + const VExprSPtrs& arguments, Op op) { + DORIS_CHECK(op == Op::EQ); + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + DORIS_CHECK(slot_literal.has_value()); + return expr_zonemap::eval_eq_dictionary(ctx, *slot_literal); +} + +inline ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx, + const VExprSPtrs& arguments, Op op) { + DORIS_CHECK(op == Op::EQ); + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + DORIS_CHECK(slot_literal.has_value()); + return expr_zonemap::eval_eq_bloom_filter(ctx, *slot_literal); +} + inline std::optional op_from_name(std::string_view name) { if (name == NameEquals::name) { return Op::EQ; @@ -589,6 +609,30 @@ class FunctionComparison : public IFunction { comparison_zonemap_detail::can_evaluate(arguments); } + ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext& ctx, + const VExprSPtrs& arguments) const override { + auto op = comparison_zonemap_detail::op_from_name(name); + DORIS_CHECK(op.has_value()); + return comparison_zonemap_detail::evaluate_dictionary(ctx, arguments, *op); + } + + bool can_evaluate_dictionary_filter(const VExprSPtrs& arguments) const override { + auto op = comparison_zonemap_detail::op_from_name(name); + return op.has_value() && comparison_zonemap_detail::can_evaluate_equality(arguments, *op); + } + + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx, + const VExprSPtrs& arguments) const override { + auto op = comparison_zonemap_detail::op_from_name(name); + DORIS_CHECK(op.has_value()); + return comparison_zonemap_detail::evaluate_bloom_filter(ctx, arguments, *op); + } + + bool can_evaluate_bloom_filter(const VExprSPtrs& arguments) const override { + auto op = comparison_zonemap_detail::op_from_name(name); + return op.has_value() && comparison_zonemap_detail::can_evaluate_equality(arguments, *op); + } + /// Get result types by argument types. If the function does not apply to these arguments, throw an exception. DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { return std::make_shared(); diff --git a/be/src/exprs/function/match.cpp b/be/src/exprs/function/match.cpp index 89669af711af02..677a8edb836a50 100644 --- a/be/src/exprs/function/match.cpp +++ b/be/src/exprs/function/match.cpp @@ -187,10 +187,10 @@ inline doris::segment_v2::InvertedIndexQueryType FunctionMatchBase::get_query_ty return doris::segment_v2::InvertedIndexQueryType::UNKNOWN_QUERY; } -std::vector FunctionMatchBase::analyse_query_str_token( +std::vector FunctionMatchBase::analyse_query_str_token( const InvertedIndexAnalyzerCtx* analyzer_ctx, const std::string& match_query_str, const std::string& column_name) const { - std::vector query_tokens; + std::vector query_tokens; if (analyzer_ctx == nullptr) { return query_tokens; } @@ -225,11 +225,11 @@ std::vector FunctionMatchBase::analyse_query_str_token( return query_tokens; } -inline std::vector FunctionMatchBase::analyse_data_token( +inline std::vector FunctionMatchBase::analyse_data_token( const std::string& column_name, const InvertedIndexAnalyzerCtx* analyzer_ctx, const ColumnString* string_col, int32_t current_block_row_idx, const ColumnArray::Offsets64* array_offsets, int32_t& current_src_array_offset) const { - std::vector data_tokens; + std::vector data_tokens; if (analyzer_ctx == nullptr) { return data_tokens; } @@ -309,10 +309,10 @@ Status FunctionMatchAny::execute_match(FunctionContext* context, const std::stri // TODO: more efficient impl for (auto& term_info : query_tokens) { - auto it = - std::find_if(data_tokens.begin(), data_tokens.end(), [&](const TermInfo& info) { - return info.get_single_term() == term_info.get_single_term(); - }); + auto it = std::find_if(data_tokens.begin(), data_tokens.end(), + [&](const segment_v2::TermInfo& info) { + return info.get_single_term() == term_info.get_single_term(); + }); if (it != data_tokens.end()) { result[i] = true; break; @@ -350,10 +350,10 @@ Status FunctionMatchAll::execute_match(FunctionContext* context, const std::stri // TODO: more efficient impl auto find_count = 0; for (auto& term_info : query_tokens) { - auto it = - std::find_if(data_tokens.begin(), data_tokens.end(), [&](const TermInfo& info) { - return info.get_single_term() == term_info.get_single_term(); - }); + auto it = std::find_if(data_tokens.begin(), data_tokens.end(), + [&](const segment_v2::TermInfo& info) { + return info.get_single_term() == term_info.get_single_term(); + }); if (it != data_tokens.end()) { ++find_count; } else { @@ -398,9 +398,10 @@ Status FunctionMatchPhrase::execute_match(FunctionContext* context, const std::s auto data_it = data_tokens.begin(); while (data_it != data_tokens.end()) { // find position of first token - data_it = std::find_if(data_it, data_tokens.end(), [&](const TermInfo& info) { - return info.get_single_term() == query_tokens[0].get_single_term(); - }); + data_it = + std::find_if(data_it, data_tokens.end(), [&](const segment_v2::TermInfo& info) { + return info.get_single_term() == query_tokens[0].get_single_term(); + }); if (data_it != data_tokens.end()) { matched = true; auto data_it_next = ++data_it; diff --git a/be/src/exprs/function/match.h b/be/src/exprs/function/match.h index 7badc8d9220d7c..84d0e4374bf245 100644 --- a/be/src/exprs/function/match.h +++ b/be/src/exprs/function/match.h @@ -51,8 +51,6 @@ class FunctionContext; namespace doris { -using namespace segment_v2; - const std::string MATCH_ANY_FUNCTION = "match_any"; const std::string MATCH_ALL_FUNCTION = "match_all"; const std::string MATCH_PHRASE_FUNCTION = "match_phrase"; @@ -83,16 +81,14 @@ class FunctionMatchBase : public IFunction { doris::segment_v2::InvertedIndexQueryType get_query_type_from_fn_name() const; - std::vector analyse_query_str_token(const InvertedIndexAnalyzerCtx* analyzer_ctx, - const std::string& match_query_str, - const std::string& field_name) const; + std::vector analyse_query_str_token( + const InvertedIndexAnalyzerCtx* analyzer_ctx, const std::string& match_query_str, + const std::string& field_name) const; - std::vector analyse_data_token(const std::string& column_name, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnString* string_col, - int32_t current_block_row_idx, - const ColumnArray::Offsets64* array_offsets, - int32_t& current_src_array_offset) const; + std::vector analyse_data_token( + const std::string& column_name, const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnString* string_col, int32_t current_block_row_idx, + const ColumnArray::Offsets64* array_offsets, int32_t& current_src_array_offset) const; Status check(FunctionContext* context, const std::string& function_name) const; diff --git a/be/src/exprs/short_circuit_evaluation_expr.h b/be/src/exprs/short_circuit_evaluation_expr.h index bd845cb1cca1ab..050cdba34c84bc 100644 --- a/be/src/exprs/short_circuit_evaluation_expr.h +++ b/be/src/exprs/short_circuit_evaluation_expr.h @@ -63,6 +63,13 @@ class ShortCircuitIfExpr final : public ShortCircuitExpr { ~ShortCircuitIfExpr() override = default; const std::string& expr_name() const override { return IF_NAME; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto node = clone_texpr_node(); + node.__set_short_circuit_evaluation(true); + *cloned_expr = ShortCircuitIfExpr::create_shared(node); + return Status::OK(); + } Status execute_column(VExprContext* context, const Block* block, Selector* selector, size_t count, ColumnPtr& result_column) const override; @@ -76,6 +83,18 @@ class ShortCircuitCaseExpr final : public ShortCircuitExpr { ShortCircuitCaseExpr(const TExprNode& node); ~ShortCircuitCaseExpr() override = default; const std::string& expr_name() const override { return CASE_NAME; } + bool has_else_expr() const { return _has_else_expr; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto node = clone_texpr_node(); + TCaseExpr case_node; + case_node.__set_has_case_expr(false); + case_node.__set_has_else_expr(_has_else_expr); + node.__set_case_expr(case_node); + node.__set_short_circuit_evaluation(true); + *cloned_expr = ShortCircuitCaseExpr::create_shared(node); + return Status::OK(); + } Status execute_column(VExprContext* context, const Block* block, Selector* selector, size_t count, ColumnPtr& result_column) const override; @@ -91,6 +110,13 @@ class ShortCircuitIfNullExpr final : public ShortCircuitExpr { ~ShortCircuitIfNullExpr() override = default; const std::string& expr_name() const override { return IFNULL_NAME; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto node = clone_texpr_node(); + node.__set_short_circuit_evaluation(true); + *cloned_expr = ShortCircuitIfNullExpr::create_shared(node); + return Status::OK(); + } Status execute_column(VExprContext* context, const Block* block, Selector* selector, size_t count, ColumnPtr& result_column) const override; @@ -104,10 +130,17 @@ class ShortCircuitCoalesceExpr final : public ShortCircuitExpr { ShortCircuitCoalesceExpr(const TExprNode& node) : ShortCircuitExpr(node) {} ~ShortCircuitCoalesceExpr() override = default; const std::string& expr_name() const override { return COALESCE_NAME; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto node = clone_texpr_node(); + node.__set_short_circuit_evaluation(true); + *cloned_expr = ShortCircuitCoalesceExpr::create_shared(node); + return Status::OK(); + } Status execute_column(VExprContext* context, const Block* block, Selector* selector, size_t count, ColumnPtr& result_column) const override; private: inline static const std::string COALESCE_NAME = "coalesce"; }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/vbloom_predicate.h b/be/src/exprs/vbloom_predicate.h index c757d851aebcc9..f522fb4f29235b 100644 --- a/be/src/exprs/vbloom_predicate.h +++ b/be/src/exprs/vbloom_predicate.h @@ -59,6 +59,13 @@ class VBloomPredicate final : public VExpr { std::shared_ptr get_bloom_filter_func() const override { return _filter; } uint64_t get_digest(uint64_t seed) const override; + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto cloned = VBloomPredicate::create_shared(clone_texpr_node()); + cloned->set_filter(_filter); + *cloned_expr = std::move(cloned); + return Status::OK(); + } private: Status _do_execute(VExprContext* context, const Block* block, const uint8_t* __restrict filter, diff --git a/be/src/exprs/vcase_expr.h b/be/src/exprs/vcase_expr.h index 7c29c5bc65d654..554d11fae5834b 100644 --- a/be/src/exprs/vcase_expr.h +++ b/be/src/exprs/vcase_expr.h @@ -59,6 +59,17 @@ class VCaseExpr final : public VExpr { void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override; const std::string& expr_name() const override; std::string debug_string() const override; + bool has_else_expr() const { return _has_else_expr; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto node = clone_texpr_node(); + TCaseExpr case_node; + case_node.__set_has_case_expr(false); + case_node.__set_has_else_expr(_has_else_expr); + node.__set_case_expr(case_node); + *cloned_expr = VCaseExpr::create_shared(node); + return Status::OK(); + } private: template diff --git a/be/src/exprs/vcast_expr.h b/be/src/exprs/vcast_expr.h index 1b1e72f57657ec..eb4f0dfd354985 100644 --- a/be/src/exprs/vcast_expr.h +++ b/be/src/exprs/vcast_expr.h @@ -55,8 +55,14 @@ class VCastExpr : public VExpr { const std::string& expr_name() const override; std::string debug_string() const override; const DataTypePtr& get_target_type() const; + bool is_safe_to_execute_on_selected_rows() const override { return false; } virtual std::string cast_name() const { return "CAST"; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = VCastExpr::create_shared(clone_texpr_node()); + return Status::OK(); + } uint64_t get_digest(uint64_t seed) const override { auto res = VExpr::get_digest(seed); @@ -94,6 +100,16 @@ class TryCastExpr final : public VCastExpr { size_t count, ColumnPtr& result_column) const override; ~TryCastExpr() override = default; std::string cast_name() const override { return "TRY CAST"; } + bool is_safe_to_execute_on_selected_rows() const override { + return VExpr::is_safe_to_execute_on_selected_rows(); + } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto node = clone_texpr_node(); + node.__set_is_cast_nullable(_original_cast_return_is_nullable); + *cloned_expr = TryCastExpr::create_shared(node); + return Status::OK(); + } private: DataTypePtr original_cast_return_type() const; diff --git a/be/src/exprs/vcolumn_ref.h b/be/src/exprs/vcolumn_ref.h index b1f94e5d1c0a8d..a69611408a2c47 100644 --- a/be/src/exprs/vcolumn_ref.h +++ b/be/src/exprs/vcolumn_ref.h @@ -81,6 +81,19 @@ class VColumnRef final : public VExpr { } } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto node = clone_texpr_node(); + TColumnRef column_ref; + column_ref.__set_column_id(_column_id); + column_ref.__set_column_name(_column_name); + node.__set_column_ref(column_ref); + auto cloned = VColumnRef::create_shared(node); + cloned->set_gap(_gap.load()); + *cloned_expr = std::move(cloned); + return Status::OK(); + } + std::string debug_string() const override { std::stringstream out; out << "VColumnRef(slot_id: " << _column_id << ",column_name: " << _column_name diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h index e4965567d63fdb..8bf0839f274d1a 100644 --- a/be/src/exprs/vcompound_pred.h +++ b/be/src/exprs/vcompound_pred.h @@ -61,6 +61,11 @@ class VCompoundPred : public VectorizedFnCall { #endif const std::string& expr_name() const override { return _expr_name; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = VCompoundPred::create_shared(clone_texpr_node()); + return Status::OK(); + } bool can_evaluate_zonemap_filter() const override { switch (_op) { @@ -108,6 +113,87 @@ class VCompoundPred : public VectorizedFnCall { } } + bool can_evaluate_dictionary_filter() const override { + switch (_op) { + case TExprOpcode::COMPOUND_AND: + return std::ranges::any_of(_children, [](const VExprSPtr& child) { + return child->can_evaluate_dictionary_filter(); + }); + case TExprOpcode::COMPOUND_OR: + return !_children.empty() && std::ranges::all_of(_children, [](const VExprSPtr& child) { + return child->can_evaluate_dictionary_filter(); + }); + default: + return false; + } + } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + switch (_op) { + case TExprOpcode::COMPOUND_AND: + for (const auto& child : _children) { + if (!child->can_evaluate_dictionary_filter()) { + continue; + } + if (child->evaluate_dictionary_filter(ctx) == ZoneMapFilterResult::kNoMatch) { + return ZoneMapFilterResult::kNoMatch; + } + } + return ZoneMapFilterResult::kMayMatch; + case TExprOpcode::COMPOUND_OR: + for (const auto& child : _children) { + DORIS_CHECK(child->can_evaluate_dictionary_filter()); + if (child->evaluate_dictionary_filter(ctx) != ZoneMapFilterResult::kNoMatch) { + return ZoneMapFilterResult::kMayMatch; + } + } + return ZoneMapFilterResult::kNoMatch; + default: + return ZoneMapFilterResult::kUnsupported; + } + } + + bool can_evaluate_bloom_filter() const override { + switch (_op) { + case TExprOpcode::COMPOUND_AND: + return std::ranges::any_of(_children, [](const VExprSPtr& child) { + return child->can_evaluate_bloom_filter(); + }); + case TExprOpcode::COMPOUND_OR: + return !_children.empty() && std::ranges::all_of(_children, [](const VExprSPtr& child) { + return child->can_evaluate_bloom_filter(); + }); + default: + return false; + } + } + + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { + switch (_op) { + case TExprOpcode::COMPOUND_AND: + for (const auto& child : _children) { + if (!child->can_evaluate_bloom_filter()) { + continue; + } + if (child->evaluate_bloom_filter(ctx) == ZoneMapFilterResult::kNoMatch) { + return ZoneMapFilterResult::kNoMatch; + } + } + return ZoneMapFilterResult::kMayMatch; + case TExprOpcode::COMPOUND_OR: + for (const auto& child : _children) { + DORIS_CHECK(child->can_evaluate_bloom_filter()); + if (child->evaluate_bloom_filter(ctx) != ZoneMapFilterResult::kNoMatch) { + return ZoneMapFilterResult::kMayMatch; + } + } + return ZoneMapFilterResult::kNoMatch; + default: + return ZoneMapFilterResult::kUnsupported; + } + } + Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override { segment_v2::InvertedIndexResultBitmap res; bool all_pass = true; diff --git a/be/src/exprs/vcondition_expr.h b/be/src/exprs/vcondition_expr.h index 956ed746348a8d..a6c7bd68a2c89d 100644 --- a/be/src/exprs/vcondition_expr.h +++ b/be/src/exprs/vcondition_expr.h @@ -65,6 +65,11 @@ class VectorizedIfExpr : public VConditionExpr { size_t count, ColumnPtr& result_column) const override; const std::string& expr_name() const override { return IF_NAME; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = VectorizedIfExpr::create_shared(clone_texpr_node()); + return Status::OK(); + } inline static const std::string IF_NAME = "if"; protected: @@ -123,6 +128,11 @@ class VectorizedIfNullExpr : public VectorizedIfExpr { public: VectorizedIfNullExpr(const TExprNode& node) : VectorizedIfExpr(node) {} const std::string& expr_name() const override { return IF_NULL_NAME; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = VectorizedIfNullExpr::create_shared(clone_texpr_node()); + return Status::OK(); + } inline static const std::string IF_NULL_NAME = "ifnull"; Status execute_column(VExprContext* context, const Block* block, Selector* selector, @@ -137,6 +147,11 @@ class VectorizedCoalesceExpr : public VConditionExpr { size_t count, ColumnPtr& result_column) const override; VectorizedCoalesceExpr(const TExprNode& node) : VConditionExpr(node) {} const std::string& expr_name() const override { return NAME; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = VectorizedCoalesceExpr::create_shared(clone_texpr_node()); + return Status::OK(); + } inline static const std::string NAME = "coalesce"; }; diff --git a/be/src/exprs/vdirect_in_predicate.h b/be/src/exprs/vdirect_in_predicate.h index a9237cba652271..8e47122e24f11a 100644 --- a/be/src/exprs/vdirect_in_predicate.h +++ b/be/src/exprs/vdirect_in_predicate.h @@ -47,7 +47,7 @@ class VDirectInPredicate final : public VExpr { // materialization and slot-IN rewrite that would otherwise rebuild child-typed literals from // dictionary codes. VDirectInPredicate(const TExprNode& node, const std::shared_ptr& filter, - bool hybrid_set_values_match_child_type) + bool hybrid_set_values_match_child_type = true) : VExpr(node), _filter(filter), _hybrid_set_values_match_child_type(hybrid_set_values_match_child_type), @@ -99,6 +99,13 @@ class VDirectInPredicate final : public VExpr { std::dynamic_pointer_cast(get_child(0)) != nullptr; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = VDirectInPredicate::create_shared(clone_texpr_node(), _filter, + _hybrid_set_values_match_child_type); + return Status::OK(); + } + bool get_slot_in_expr(VExprSPtr& new_root) const { if (!_hybrid_set_values_match_child_type) { return false; diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index df616ebc2e22bc..621d183fc84a05 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -25,6 +25,7 @@ #include #include +#include #include "common/config.h" #include "common/exception.h" @@ -82,7 +83,9 @@ const static std::set DISTANCE_FUNCS = {L2DistanceApproximate::name const static std::set OPS_FOR_ANN_RANGE_SEARCH = { TExprOpcode::GE, TExprOpcode::LE, TExprOpcode::LE, TExprOpcode::GT, TExprOpcode::LT}; -VectorizedFnCall::VectorizedFnCall(const TExprNode& node) : VExpr(node) {} +VectorizedFnCall::VectorizedFnCall(const TExprNode& node) : VExpr(node) { + _function_name = _fn.name.function_name; +} Status VectorizedFnCall::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { @@ -219,6 +222,26 @@ bool VectorizedFnCall::can_evaluate_zonemap_filter() const { _function->can_evaluate_zonemap_filter(_children); } +ZoneMapFilterResult VectorizedFnCall::evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const { + return _function->evaluate_dictionary_filter(ctx, _children); +} + +bool VectorizedFnCall::can_evaluate_dictionary_filter() const { + return _function != nullptr && !_function->is_blockable() && + _function->can_evaluate_dictionary_filter(_children); +} + +ZoneMapFilterResult VectorizedFnCall::evaluate_bloom_filter( + const BloomFilterEvalContext& ctx) const { + return _function->evaluate_bloom_filter(ctx, _children); +} + +bool VectorizedFnCall::can_evaluate_bloom_filter() const { + return _function != nullptr && !_function->is_blockable() && + _function->can_evaluate_bloom_filter(_children); +} + Status VectorizedFnCall::_do_execute(VExprContext* context, const Block* block, Selector* selector, size_t count, ColumnPtr& result_column, ColumnPtr* arg_column) const { @@ -356,6 +379,18 @@ bool VectorizedFnCall::can_push_down_to_index() const { return _function->can_push_down_to_index(); } +bool VectorizedFnCall::is_deterministic() const { + static const std::set NON_DETERMINISTIC_FUNCTIONS = { + "random", "rand", "random_bytes", "uuid", "uuid_numeric"}; + return !NON_DETERMINISTIC_FUNCTIONS.contains(_function_name) && VExpr::is_deterministic(); +} + +bool VectorizedFnCall::is_safe_to_execute_on_selected_rows() const { + static const std::set ERROR_PRESERVING_FUNCTIONS = {"assert_true"}; + return !ERROR_PRESERVING_FUNCTIONS.contains(_function_name) && + VExpr::is_safe_to_execute_on_selected_rows(); +} + bool VectorizedFnCall::equals(const VExpr& other) { const auto* other_ptr = dynamic_cast(&other); if (!other_ptr) { diff --git a/be/src/exprs/vectorized_fn_call.h b/be/src/exprs/vectorized_fn_call.h index f423e7f6470ee7..765113f0b1cafc 100644 --- a/be/src/exprs/vectorized_fn_call.h +++ b/be/src/exprs/vectorized_fn_call.h @@ -61,6 +61,10 @@ class VectorizedFnCall : public VExpr { Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; bool can_evaluate_zonemap_filter() const override; + ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext& ctx) const override; + bool can_evaluate_dictionary_filter() const override; + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override; + bool can_evaluate_bloom_filter() const override; Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; Status open(RuntimeState* state, VExprContext* context, FunctionContext::FunctionStateScope scope) override; @@ -74,6 +78,8 @@ class VectorizedFnCall : public VExpr { std::any_of(_children.begin(), _children.end(), [](VExprSPtr child) { return child->is_blockable(); }); } + bool is_deterministic() const override; + bool is_safe_to_execute_on_selected_rows() const override; bool is_constant() const override { if (!_function->is_use_default_implementation_for_constants() || // udf function with no argument, can't sure it's must return const column @@ -102,6 +108,12 @@ class VectorizedFnCall : public VExpr { segment_v2::AnnRangeSearchRuntime& runtime, bool& suitable_for_ann_index) override; + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(*this); + return Status::OK(); + } + protected: FunctionBasePtr _function; std::string _expr_name; diff --git a/be/src/exprs/vexpr.cpp b/be/src/exprs/vexpr.cpp index d5686f8ca533b6..494ebcf04d1c34 100644 --- a/be/src/exprs/vexpr.cpp +++ b/be/src/exprs/vexpr.cpp @@ -83,6 +83,14 @@ ZoneMapFilterResult VExpr::evaluate_zonemap_filter(const ZoneMapEvalContext& ctx return unsupported_zonemap_filter(ctx); } +ZoneMapFilterResult VExpr::evaluate_dictionary_filter(const DictionaryEvalContext& ctx) const { + return ZoneMapFilterResult::kUnsupported; +} + +ZoneMapFilterResult VExpr::evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const { + return ZoneMapFilterResult::kUnsupported; +} + // NOLINTBEGIN(readability-function-cognitive-complexity) // NOLINTBEGIN(readability-function-size) TExprNode create_texpr_node_from(const void* data, const PrimitiveType& type, int precision, @@ -385,6 +393,51 @@ VExpr::VExpr(DataTypePtr type, bool is_slotref) } } +TExprNode VExpr::clone_texpr_node() const { + TExprNode node; + node.__set_node_type(_node_type); + node.__set_opcode(_opcode); + node.__set_type(create_type_desc(remove_nullable(_data_type)->get_primitive_type(), + static_cast(_data_type->get_precision()), + static_cast(_data_type->get_scale()))); + node.__set_is_nullable(_data_type->is_nullable()); + node.__set_num_children(get_num_children()); + node.__set_fn(_fn); + return node; +} + +Status VExpr::clone_node(VExprSPtr* cloned_expr) const { + DORIS_CHECK(cloned_expr != nullptr); + return Status::NotSupported("Cannot clone expression {} for file-local rewrite", expr_name()); +} + +Status VExpr::deep_clone(VExprSPtr* cloned_expr, + const VExprCloneNodeOverride& clone_node_override) const { + DORIS_CHECK(cloned_expr != nullptr); + + VExprSPtr cloned; + if (clone_node_override) { + RETURN_IF_ERROR(clone_node_override(*this, &cloned)); + } + if (cloned == nullptr) { + RETURN_IF_ERROR(clone_node(&cloned)); + } + DORIS_CHECK(cloned != nullptr); + + VExprSPtrs cloned_children; + cloned_children.reserve(_children.size()); + for (const auto& child : _children) { + DORIS_CHECK(child != nullptr); + VExprSPtr cloned_child; + RETURN_IF_ERROR(child->deep_clone(&cloned_child, clone_node_override)); + cloned_children.push_back(std::move(cloned_child)); + } + cloned->set_children(std::move(cloned_children)); + cloned->reset_prepare_state(); + *cloned_expr = std::move(cloned); + return Status::OK(); +} + Status VExpr::prepare(RuntimeState* state, const RowDescriptor& row_desc, VExprContext* context) { ++context->_depth_num; if (context->_depth_num > config::max_depth_of_expr_tree) { @@ -414,6 +467,15 @@ Status VExpr::open(RuntimeState* state, VExprContext* context, return Status::OK(); } +void VExpr::reset_prepare_state() { + _prepared = false; + _prepare_finished = false; + _open_finished = false; + for (auto& child : _children) { + child->reset_prepare_state(); + } +} + void VExpr::close(VExprContext* context, FunctionContext::FunctionStateScope scope) { for (auto& i : _children) { i->close(context, scope); @@ -764,8 +826,9 @@ Status VExpr::get_const_col(VExprContext* context, return Status::OK(); } - if (_constant_col != nullptr) { - DCHECK(column_wrapper != nullptr); + if (_constant_col != nullptr && column_wrapper == nullptr) { + return Status::OK(); + } else if (_constant_col != nullptr) { *column_wrapper = _constant_col; return Status::OK(); } diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index 6796058fe402e4..e7f9851b61a44e 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,7 @@ #include "core/value/large_int_value.h" #include "core/value/timestamptz_value.h" #include "exprs/aggregate/aggregate_function.h" +#include "exprs/expr_zonemap_filter.h" #include "exprs/function/cast/cast_to_string.h" #include "exprs/function/function.h" #include "exprs/function_context.h" @@ -80,6 +82,7 @@ struct AnnRangeSearchRuntime; // the relatioinship between threads and classes. using Selector = IColumn::Selector; +using VExprCloneNodeOverride = std::function; struct AnnRangeSearchEvaluationResult { // Indicates whether the expr row_bitmap has been updated. @@ -157,7 +160,16 @@ class VExpr { // which rows in the block should be evaluated. // If expr is executing constant expressions, then block should be nullptr. virtual Status execute_column(VExprContext* context, const Block* block, Selector* selector, - size_t count, ColumnPtr& result_column) const = 0; + size_t count, ColumnPtr& result_column) const { + return execute_column_impl(context, block, selector, count, result_column); + } + + virtual Status execute_column_impl(VExprContext* context, const Block* block, + const Selector* selector, size_t count, + ColumnPtr& result_column) const { + return execute_column(context, block, const_cast(selector), count, + result_column); + } // Currently, due to fe planning issues, for slot-ref expressions the type of the returned Column may not match data_type. // Therefore we need a function like this to return the actual type produced by execution. @@ -173,6 +185,17 @@ class VExpr { [](VExprSPtr child) { return child->is_blockable(); }); } + [[nodiscard]] virtual bool is_deterministic() const { + return std::ranges::all_of( + _children, [](const VExprSPtr& child) { return child->is_deterministic(); }); + } + + [[nodiscard]] virtual bool is_safe_to_execute_on_selected_rows() const { + return is_deterministic() && std::ranges::all_of(_children, [](const VExprSPtr& child) { + return child->is_safe_to_execute_on_selected_rows(); + }); + } + // execute current expr with inverted index to filter block. Given a roaring bitmap of match rows virtual Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) { return Status::OK(); @@ -180,6 +203,10 @@ class VExpr { virtual ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const; virtual bool can_evaluate_zonemap_filter() const { return false; } + virtual ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext& ctx) const; + virtual bool can_evaluate_dictionary_filter() const { return false; } + virtual ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const; + virtual bool can_evaluate_bloom_filter() const { return false; } // Get analyzer key for inverted index queries (overridden by VMatchPredicate) [[nodiscard]] virtual const std::string& get_analyzer_key() const { @@ -211,11 +238,13 @@ class VExpr { const DataTypePtr& data_type() const { return _data_type; } - bool is_slot_ref() const { return _node_type == TExprNodeType::SLOT_REF; } + virtual bool is_slot_ref() const { return _node_type == TExprNodeType::SLOT_REF; } - bool is_virtual_slot_ref() const { return _node_type == TExprNodeType::VIRTUAL_SLOT_REF; } + virtual bool is_virtual_slot_ref() const { + return _node_type == TExprNodeType::VIRTUAL_SLOT_REF; + } - bool is_column_ref() const { return _node_type == TExprNodeType::COLUMN_REF; } + virtual bool is_column_ref() const { return _node_type == TExprNodeType::COLUMN_REF; } virtual bool is_literal() const { return false; } @@ -249,6 +278,10 @@ class VExpr { static bool contains_blockable_function(const VExprContextSPtrs& ctxs); + Status deep_clone(VExprSPtr* cloned_expr, + const VExprCloneNodeOverride& clone_node_override = {}) const; + virtual Status clone_node(VExprSPtr* cloned_expr) const; + bool is_nullable() const { return _data_type->is_nullable(); } PrimitiveType result_type() const { return _data_type->get_primitive_type(); } @@ -263,6 +296,7 @@ class VExpr { virtual const VExprSPtrs& children() const { return _children; } void set_children(const VExprSPtrs& children) { _children = children; } void set_children(VExprSPtrs&& children) { _children = std::move(children); } + void reset_prepare_state(); virtual std::string debug_string() const; static std::string debug_string(const VExprSPtrs& exprs); static std::string debug_string(const VExprContextSPtrs& ctxs); @@ -270,7 +304,7 @@ class VExpr { static ColumnPtr filter_column_with_selector(const ColumnPtr& origin_column, const Selector* selector, size_t count) { if (selector == nullptr) { - DCHECK_EQ(origin_column->size(), count); + DCHECK_EQ(origin_column->size(), count) << origin_column->get_name(); return origin_column; } DCHECK_EQ(count, selector->size()); @@ -363,6 +397,8 @@ class VExpr { virtual uint64_t get_digest(uint64_t seed) const; protected: + TExprNode clone_texpr_node() const; + /// Simple debug string that provides no expr subclass-specific information std::string debug_string(const std::string& expr_name) const { std::stringstream out; diff --git a/be/src/exprs/vexpr_context.cpp b/be/src/exprs/vexpr_context.cpp index ba031cf8ea9de1..69abe2f748bcc7 100644 --- a/be/src/exprs/vexpr_context.cpp +++ b/be/src/exprs/vexpr_context.cpp @@ -205,6 +205,38 @@ ZoneMapFilterResult VExprContext::evaluate_zonemap_filter(const VExprContextSPtr return ZoneMapFilterResult::kMayMatch; } +ZoneMapFilterResult VExprContext::evaluate_dictionary_filter(const VExprContextSPtrs& conjuncts, + const DictionaryEvalContext& ctx) { + for (const auto& conjunct : conjuncts) { + DORIS_CHECK(conjunct != nullptr); + const auto& root = conjunct->root(); + DORIS_CHECK(root != nullptr); + if (!root->can_evaluate_dictionary_filter()) { + continue; + } + if (root->evaluate_dictionary_filter(ctx) == ZoneMapFilterResult::kNoMatch) { + return ZoneMapFilterResult::kNoMatch; + } + } + return ZoneMapFilterResult::kMayMatch; +} + +ZoneMapFilterResult VExprContext::evaluate_bloom_filter(const VExprContextSPtrs& conjuncts, + const BloomFilterEvalContext& ctx) { + for (const auto& conjunct : conjuncts) { + DORIS_CHECK(conjunct != nullptr); + const auto& root = conjunct->root(); + DORIS_CHECK(root != nullptr); + if (!root->can_evaluate_bloom_filter()) { + continue; + } + if (root->evaluate_bloom_filter(ctx) == ZoneMapFilterResult::kNoMatch) { + return ZoneMapFilterResult::kNoMatch; + } + } + return ZoneMapFilterResult::kMayMatch; +} + bool VExprContext::all_expr_inverted_index_evaluated() { return _index_context->has_index_result_for_expr(_root.get()); } diff --git a/be/src/exprs/vexpr_context.h b/be/src/exprs/vexpr_context.h index 757593a9d2e105..358ab7cbc5c05a 100644 --- a/be/src/exprs/vexpr_context.h +++ b/be/src/exprs/vexpr_context.h @@ -32,6 +32,7 @@ #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" #include "exec/runtime_filter/runtime_filter_selectivity.h" +#include "exprs/expr_zonemap_filter.h" #include "exprs/function_context.h" #include "exprs/vexpr_fwd.h" #include "runtime/runtime_state.h" @@ -297,6 +298,10 @@ class VExprContext { [[nodiscard]] static ZoneMapFilterResult evaluate_zonemap_filter( const VExprContextSPtrs& conjuncts, const ZoneMapEvalContext& ctx); + [[nodiscard]] static ZoneMapFilterResult evaluate_dictionary_filter( + const VExprContextSPtrs& conjuncts, const DictionaryEvalContext& ctx); + [[nodiscard]] static ZoneMapFilterResult evaluate_bloom_filter( + const VExprContextSPtrs& conjuncts, const BloomFilterEvalContext& ctx); bool all_expr_inverted_index_evaluated(); diff --git a/be/src/exprs/vin_predicate.cpp b/be/src/exprs/vin_predicate.cpp index ce57421b97e5b6..6c70f781c7e799 100644 --- a/be/src/exprs/vin_predicate.cpp +++ b/be/src/exprs/vin_predicate.cpp @@ -164,6 +164,25 @@ bool VInPredicate::can_evaluate_zonemap_filter() const { return _zonemap_materialized && std::dynamic_pointer_cast(get_child(0)) != nullptr; } +ZoneMapFilterResult VInPredicate::evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const { + return expr_zonemap::eval_in_dictionary(ctx, get_child(0), _is_not_in, _seg_filter_values); +} + +bool VInPredicate::can_evaluate_dictionary_filter() const { + return _zonemap_materialized && !_is_not_in && + std::dynamic_pointer_cast(get_child(0)) != nullptr; +} + +ZoneMapFilterResult VInPredicate::evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const { + return expr_zonemap::eval_in_bloom_filter(ctx, get_child(0), _is_not_in, _seg_filter_values); +} + +bool VInPredicate::can_evaluate_bloom_filter() const { + return _zonemap_materialized && !_is_not_in && + std::dynamic_pointer_cast(get_child(0)) != nullptr; +} + Status VInPredicate::execute_column(VExprContext* context, const Block* block, Selector* selector, size_t count, ColumnPtr& result_column) const { if (is_const_and_have_executed()) { // const have execute in open function diff --git a/be/src/exprs/vin_predicate.h b/be/src/exprs/vin_predicate.h index 23d78e7e958c93..7f643fc00a9497 100644 --- a/be/src/exprs/vin_predicate.h +++ b/be/src/exprs/vin_predicate.h @@ -62,8 +62,21 @@ class VInPredicate MOCK_REMOVE(final) : public VExpr { Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; bool can_evaluate_zonemap_filter() const override; + ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext& ctx) const override; + bool can_evaluate_dictionary_filter() const override; + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override; + bool can_evaluate_bloom_filter() const override; uint64_t get_digest(uint64_t seed) const override { return 0; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + auto node = clone_texpr_node(); + TInPredicate in_predicate; + in_predicate.__set_is_not_in(_is_not_in); + node.__set_in_predicate(in_predicate); + *cloned_expr = VInPredicate::create_shared(node); + return Status::OK(); + } private: Status _materialize_for_zonemap_filter(VExprContext* context); diff --git a/be/src/exprs/vliteral.cpp b/be/src/exprs/vliteral.cpp index a05ec0881a5c93..048b2951fc433e 100644 --- a/be/src/exprs/vliteral.cpp +++ b/be/src/exprs/vliteral.cpp @@ -38,12 +38,6 @@ namespace doris { class VExprContext; -void VLiteral::init(const TExprNode& node) { - Field field; - field = _data_type->get_field(node); - _column_ptr = _data_type->create_column_const(1, field); -} - Status VLiteral::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); return Status::OK(); diff --git a/be/src/exprs/vliteral.h b/be/src/exprs/vliteral.h index 12df95593e87f8..678927c895c770 100644 --- a/be/src/exprs/vliteral.h +++ b/be/src/exprs/vliteral.h @@ -24,6 +24,7 @@ #include "common/status.h" #include "core/data_type/data_type.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/field.h" #include "exprs/vexpr.h" namespace doris { @@ -39,10 +40,19 @@ class VLiteral : public VExpr { VLiteral(const TExprNode& node, bool should_init = true) : VExpr(node), _expr_name(_data_type->get_name()) { if (should_init) { - init(node); + Field field; + field = _data_type->get_field(node); + _column_ptr = _data_type->create_column_const(1, field); } } + VLiteral(const DataTypePtr& type, const Field& field) : VExpr(type, false) { + _data_type = type; + _column_ptr = _data_type->create_column_const(1, field); + _node_type = TExprNodeType::LITERAL; + _expr_name = _data_type->get_name(); + } + #ifdef BE_TEST VLiteral() = default; MOCK_FUNCTION std::string value() const; @@ -67,13 +77,18 @@ class VLiteral : public VExpr { bool equals(const VExpr& other) override; uint64_t get_digest(uint64_t seed) const override; + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + Field field; + _column_ptr->get(0, field); + *cloned_expr = VLiteral::create_shared(_data_type, field); + return Status::OK(); + } protected: + VLiteral(const DataTypePtr& type) : VExpr(type, false) {} ColumnPtr _column_ptr; std::string _expr_name; - -private: - void init(const TExprNode& node); }; } // namespace doris diff --git a/be/src/exprs/vruntimefilter_wrapper.cpp b/be/src/exprs/vruntimefilter_wrapper.cpp index c090c8376c269a..d403769ddaea12 100644 --- a/be/src/exprs/vruntimefilter_wrapper.cpp +++ b/be/src/exprs/vruntimefilter_wrapper.cpp @@ -70,6 +70,17 @@ VRuntimeFilterWrapper::VRuntimeFilterWrapper(const TExprNode& node, VExprSPtr im DORIS_CHECK(_impl != nullptr); } +Status VRuntimeFilterWrapper::clone_node(VExprSPtr* cloned_expr) const { + DORIS_CHECK(cloned_expr != nullptr); + DORIS_CHECK(_impl != nullptr); + VExprSPtr cloned_impl; + RETURN_IF_ERROR(_impl->deep_clone(&cloned_impl)); + *cloned_expr = VRuntimeFilterWrapper::create_shared(clone_texpr_node(), std::move(cloned_impl), + _ignore_thredhold, _null_aware, _filter_id, + _sampling_frequency); + return Status::OK(); +} + Status VRuntimeFilterWrapper::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { RETURN_IF_ERROR_OR_PREPARED(_impl->prepare(state, desc, context)); diff --git a/be/src/exprs/vruntimefilter_wrapper.h b/be/src/exprs/vruntimefilter_wrapper.h index a3996944d76468..1d903832de353b 100644 --- a/be/src/exprs/vruntimefilter_wrapper.h +++ b/be/src/exprs/vruntimefilter_wrapper.h @@ -82,6 +82,8 @@ class VRuntimeFilterWrapper final : public VExpr { } VExprSPtr get_impl() const override { return _impl; } + void set_impl(VExprSPtr impl) { _impl = std::move(impl); } + Status clone_node(VExprSPtr* cloned_expr) const override; void attach_profile_counter(std::shared_ptr rf_input_rows, std::shared_ptr rf_filter_rows, @@ -102,6 +104,9 @@ class VRuntimeFilterWrapper final : public VExpr { } bool is_rf_wrapper() const override { return true; } + bool is_slot_ref() const override { return false; } + bool is_virtual_slot_ref() const override { return false; } + bool is_column_ref() const override { return false; } ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; bool can_evaluate_zonemap_filter() const override; @@ -139,4 +144,4 @@ class VRuntimeFilterWrapper final : public VExpr { using VRuntimeFilterPtr = std::shared_ptr; #include "common/compile_check_end.h" -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/vslot_ref.cpp b/be/src/exprs/vslot_ref.cpp index 7f6f47e40e77bb..35b9f5468cb63b 100644 --- a/be/src/exprs/vslot_ref.cpp +++ b/be/src/exprs/vslot_ref.cpp @@ -41,10 +41,28 @@ VSlotRef::VSlotRef(const doris::TExprNode& node) VSlotRef::VSlotRef(const SlotDescriptor* desc) : VExpr(desc->type(), true), _slot_id(desc->id()), _column_id(-1), _column_name(nullptr) {} +VSlotRef::VSlotRef(int slot_id, int column_id, int column_uniq_id, const DataTypePtr& type, + std::string column_name) + : VExpr(type, true), + _slot_id(slot_id), + _column_id(column_id), + _column_uniq_id(column_uniq_id), + _owned_column_name(std::move(column_name)), + _column_name(&_owned_column_name) {} + Status VSlotRef::prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc, VExprContext* context) { - RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); DCHECK_EQ(_children.size(), 0); + if (_prepared) { + return Status::OK(); + } + if (_column_id >= 0 && _column_name != nullptr) { + _prepared = true; + _prepare_finished = true; + return Status::OK(); + } + _prepared = true; + RETURN_IF_ERROR(VExpr::prepare(state, desc, context)); if (_slot_id == -1) { _prepare_finished = true; return Status::OK(); @@ -108,6 +126,27 @@ DataTypePtr VSlotRef::execute_type(const Block* block) const { return block->get_by_position(_column_id).type; } +Status VSlotRef::clone_node(VExprSPtr* cloned_expr) const { + DORIS_CHECK(cloned_expr != nullptr); + if (_column_id >= 0 && _column_name != nullptr) { + *cloned_expr = VSlotRef::create_shared(_slot_id, _column_id, _column_uniq_id, _data_type, + *_column_name); + return Status::OK(); + } + auto node = clone_texpr_node(); + TSlotRef slot_ref; + slot_ref.__set_slot_id(_slot_id); + node.__set_slot_ref(slot_ref); + node.__set_label(_column_label); + auto cloned = VSlotRef::create_shared(node); + auto* cloned_slot_ref = static_cast(cloned.get()); + cloned_slot_ref->_column_id = _column_id; + cloned_slot_ref->_column_uniq_id = _column_uniq_id; + cloned_slot_ref->_column_name = _column_name; + *cloned_expr = std::move(cloned); + return Status::OK(); +} + const std::string& VSlotRef::expr_name() const { return *_column_name; } diff --git a/be/src/exprs/vslot_ref.h b/be/src/exprs/vslot_ref.h index c442179a5ff6eb..e696e9f31aeb5e 100644 --- a/be/src/exprs/vslot_ref.h +++ b/be/src/exprs/vslot_ref.h @@ -31,12 +31,14 @@ class TExprNode; class Block; class VExprContext; -class VSlotRef MOCK_REMOVE(final) : public VExpr { +class VSlotRef : public VExpr { ENABLE_FACTORY_CREATOR(VSlotRef); public: VSlotRef(const TExprNode& node); VSlotRef(const SlotDescriptor* desc); + VSlotRef(int slot_id, int column_id, int column_uniq_id, const DataTypePtr& type, + std::string column_name); #ifdef BE_TEST VSlotRef() = default; void set_slot_id(int slot_id) { _slot_id = slot_id; } @@ -58,6 +60,7 @@ class VSlotRef MOCK_REMOVE(final) : public VExpr { int column_id() const { return _column_id; } MOCK_FUNCTION int slot_id() const { return _slot_id; } + int column_uniq_id() const { return _column_uniq_id; } bool equals(const VExpr& other) override; @@ -67,16 +70,24 @@ class VSlotRef MOCK_REMOVE(final) : public VExpr { column_ids.insert(_column_id); } - MOCK_FUNCTION const std::string& column_name() const { return *_column_name; } + virtual const std::string& column_name() const { return *_column_name; } uint64_t get_digest(uint64_t seed) const override; double execute_cost() const override { return 0.0; } + Status clone_node(VExprSPtr* cloned_expr) const override; + +protected: + VSlotRef(int slot_id, int column_id, int column_uniq_id) + : _slot_id(slot_id), _column_id(column_id), _column_uniq_id(column_uniq_id) { + _node_type = TExprNodeType::SLOT_REF; + } private: int _slot_id; int _column_id; int _column_uniq_id = -1; + std::string _owned_column_name; const std::string* _column_name = nullptr; const std::string _column_label; }; diff --git a/be/src/exprs/vtopn_pred.h b/be/src/exprs/vtopn_pred.h index f49d3409c3118a..b481c0f5e7726d 100644 --- a/be/src/exprs/vtopn_pred.h +++ b/be/src/exprs/vtopn_pred.h @@ -64,6 +64,11 @@ class VTopNPred : public VExpr { } int source_node_id() const { return _source_node_id; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = VTopNPred::create_shared(clone_texpr_node(), _source_node_id, nullptr); + return Status::OK(); + } Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override { _predicate = &state->get_query_ctx()->get_runtime_predicate(_source_node_id); diff --git a/be/src/format/CMakeLists.txt b/be/src/format/CMakeLists.txt index 64e7b4f14fe05e..bc0325f3e0f252 100644 --- a/be/src/format/CMakeLists.txt +++ b/be/src/format/CMakeLists.txt @@ -22,6 +22,16 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/format") set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/format") file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp) +file(GLOB_RECURSE FORMAT_V2_SRC_FILES CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/../format_v2/*.cpp) +list(APPEND SRC_FILES ${FORMAT_V2_SRC_FILES}) + +# Lance reader requires Rust static library (BUILD_RUST_READERS=ON) +if (NOT BUILD_RUST_READERS) + file(GLOB_RECURSE LANCE_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/lance/*.cpp) + list(REMOVE_ITEM SRC_FILES ${LANCE_FILES}) +endif() + add_library(Format STATIC ${SRC_FILES}) pch_reuse(Format) diff --git a/be/src/format/file_reader/new_plain_text_line_reader.cpp b/be/src/format/file_reader/new_plain_text_line_reader.cpp index c5e14f6f1a94ec..a4df306bf49db5 100644 --- a/be/src/format/file_reader/new_plain_text_line_reader.cpp +++ b/be/src/format/file_reader/new_plain_text_line_reader.cpp @@ -47,6 +47,19 @@ namespace doris { #include "common/compile_check_begin.h" const uint8_t* EncloseCsvLineReaderCtx::read_line_impl(const uint8_t* start, const size_t length) { + if (_skip_utf8_bom && !_first_record_prefix_checked && _idx == 0) { + constexpr uint8_t UTF8_BOM[] = {0xEF, 0xBB, 0xBF}; + constexpr size_t UTF8_BOM_SIZE = sizeof(UTF8_BOM); + const size_t prefix_size = std::min(length, UTF8_BOM_SIZE); + if (std::memcmp(start, UTF8_BOM, prefix_size) != 0) { + _first_record_prefix_checked = true; + } else if (length < UTF8_BOM_SIZE) { + return nullptr; + } else { + _idx = UTF8_BOM_SIZE; + _first_record_prefix_checked = true; + } + } // Avoid part bytes of the multi-char column separator have already been parsed, // causing parse column separator error. if (_state.curr_state == ReaderState::NORMAL || diff --git a/be/src/format/file_reader/new_plain_text_line_reader.h b/be/src/format/file_reader/new_plain_text_line_reader.h index 05a14423231cee..3f18f28c673ad1 100644 --- a/be/src/format/file_reader/new_plain_text_line_reader.h +++ b/be/src/format/file_reader/new_plain_text_line_reader.h @@ -157,10 +157,12 @@ class EncloseCsvLineReaderCtx final : public BaseTextLineReaderContext - T* get(const KType& key, const std::function create_func) { + T* get(const KType& key, const std::function create_func, bool* cache_hit = nullptr) { std::lock_guard lock(_lock); auto it = _storage.find(key); if (it != _storage.end()) { + if (cache_hit != nullptr) { + *cache_hit = true; + } return reinterpret_cast(it->second); } else { + if (cache_hit != nullptr) { + *cache_hit = false; + } T* rawPtr = create_func(); if (rawPtr != nullptr) { _delete_fn[key] = [](void* obj) { delete reinterpret_cast(obj); }; @@ -130,8 +136,9 @@ class ShardedKVCache { } template - T* get(const std::string& key, const std::function create_func) { - return _shards[_get_idx(key)]->get(key, create_func); + T* get(const std::string& key, const std::function create_func, + bool* cache_hit = nullptr) { + return _shards[_get_idx(key)]->get(key, create_func, cache_hit); } private: diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md new file mode 100644 index 00000000000000..377a26ce530db8 --- /dev/null +++ b/be/src/format_v2/AGENTS.md @@ -0,0 +1,183 @@ +# Format V2 — Review Guide + +Use this guide when reviewing changes under `be/src/format_v2/`. Apply the repository-level +instructions as well; this file adds format-v2-specific review expectations. + +## Review Objective + +- Report actionable correctness, data-corruption, crash, resource-lifetime, and performance + regressions. Do not report style-only issues already enforced by the repository tooling. +- Trace the complete affected path instead of reviewing a changed function in isolation. The usual + path crosses `TableReader`, `TableColumnMapper`, schema projection/materialization, and a concrete + file or table reader. +- Verify claims against callers, implementations, and tests. Do not report a hypothetical failure + unless a reachable input or state demonstrates it. + +## Architecture and Interface Contracts + +- Use the [FileScannerV2 design document](../../../docs/file-scanner-v2-design.md) as the + architectural reference. Preserve the one-way responsibility chain: Scanner manages query + integration and Split progression, `TableReader` manages table semantics, and `FileReader` + interprets physical files. Layer boundaries take priority over incidental code reuse. +- `TableReader` owns table-level projection and column order, partition/default/virtual columns, + table predicates and delete semantics, per-Split state, reader orchestration, and final table-block + materialization. It may consume file schema and file-local blocks through stable contracts, but it + must not depend on a concrete format reader's metadata structures, decoding implementation, or + physical nested layout. +- `FileReader` owns physical schema discovery, file metadata, encoding and decoding, physical + pruning, lazy reads, and production of file-local blocks. It must not know query-global column + positions, table output order, partition/default/virtual-column construction, table-format + semantics, Scanner scheduling, or Split-source policy. +- `TableColumnMapper` is the only semantic bridge between table/global and file/local column + domains. It translates table projection and predicates plus file schema into `FileScanRequest`, + mapping/finalize metadata, constants, and localized expressions. It must not open or read files, + advance Splits, own reader lifecycle, or depend on concrete `TableReader`/`FileReader` + implementations. +- Coupling between these layers is allowed only through stable, format-neutral contracts such as + `ColumnDefinition`, `FileScanRequest`, mapper results, capability/status objects, and file-local + blocks. Flag new concrete-class includes, downcasts, reverse callbacks, shared mutable state, or + direct inspection of another layer's implementation details. +- Do not bypass `TableColumnMapper`: `TableReader` must not independently reproduce file-local + column matching or position logic, and `FileReader` must not independently resolve table schema, + defaults, partitions, virtual columns, or final table types. There must be one authoritative + mapping for projection, predicate localization, and final materialization. +- Keep identity namespaces explicit at every boundary. Query expressions and table output use + global identities; file requests and file blocks use local identities. A file-local ordinal, + field ID, physical child position, or format wrapper node must never leak upward as a table/global + identity. +- Localized predicates and delete information may be executed by a file reader only after the + mapper/table layer has converted them into a file-local contract. The file reader may optimize + execution but must not reinterpret or invent the table-level semantics. +- Format-specific capability or metadata needed by an upper layer should be exposed as the smallest + neutral capability/result contract. Do not add Parquet/ORC/JNI-specific conditionals to generic + table semantics when the decision belongs in a reader, factory, or capability interface. +- When reviewing an interface change, identify the owner layer, document input/output and lifecycle + invariants, inspect every caller and implementation, and verify that adding another file format or + table format would not require changes in unrelated layers. Require boundary-focused tests that + exercise mapping and materialization independently from physical decoding where possible. + +## Reader Lifecycle and Contracts + +- Preserve the reader lifecycle and state transitions across initialization, schema discovery, + opening, block production, EOF, split advancement, and close. +- Check that empty blocks, EOF, cancellation, early returns, and errors cannot skip required cleanup + or leave stale per-file/per-split state for the next reader. +- Keep `current_rows`, block row counts, selection vectors, row positions, and `eos` consistent on + every path, including fully filtered blocks and aggregate-pushdown paths. +- Check ownership and lifetime of file readers, column readers, blocks, columns, expression + contexts, callbacks, and objects referenced through raw pointers or views. + +## Schema Mapping and Materialization + +- Keep table/global identities and positions distinct from file/local identities and positions. + Review uses of `GlobalIndex`, `LocalColumnId`, `LocalIndex`, `ConstantIndex`, and nested child IDs + for accidental namespace or ordinal mixing. +- Verify mapping by field ID, name, and position against the intended table format. Missing columns, + partition columns, defaults, and virtual columns must be materialized with the correct type, + nullability, and row count. +- For schema evolution, check field additions, removals, renames, reordering, type changes, and + nullable/non-nullable transitions. +- For `STRUCT`, `ARRAY`, and `MAP`, verify recursive projection and reconstruction, child ordering, + file-local IDs, offsets, null maps, and empty collections. Remember that semantic Doris trees and + physical file-format trees may have different shapes. +- Check that casts and defaults preserve Doris semantics for overflow, precision/scale, timezone, + decimal, date/time, string, and nullable values. + +## Filtering, Deletes, and Pushdown + +- Predicate columns and lazily materialized non-predicate columns must refer to exactly the same + rows after filtering. Review selection-vector reuse, skipped row groups/pages, and row-position + accounting together. +- A pushed-down predicate, statistic, dictionary filter, bloom filter, or aggregate must be + semantically equivalent to evaluating it after materialization. Unsupported or unsafe cases must + follow the designed fallback or return an explicit error; they must not silently change results. +- Review equality deletes, position deletes, table-format predicates, and generated row-location + columns for ordering, null semantics, type conversion, file identity, and absolute row position. +- For Iceberg, Hive, Hudi, Paimon, Remote Doris, and JNI-backed readers, verify that the table-level + wrapper preserves the underlying file reader's schema, filtering, split, and EOF contracts. + +## Format-Specific Boundaries + +- Confirm file-format dispatch and capability checks match the actual implementation. New behavior + must not accidentally route unsupported formats or table modes into a reader that cannot handle + them. +- For Parquet and ORC, review physical-to-semantic schema conversion, nested levels/offsets, + statistics validity, page or stripe pruning, and corrupt/truncated input handling. +- For CSV, text, and JSON, review record boundaries, escaping/quoting, malformed rows, encoding, + column count, and partial-buffer behavior across reads. +- For JNI readers, review local/global reference lifetime, exception propagation, type conversion, + thread attachment assumptions, and cleanup on partial initialization. + +## Detailed FileReader Review Guides + +- Before reviewing any FileReader implementation, index, predicate path, cache, or virtual column, + read and apply the common checklist in + [FileScannerV2 Code Review Guide](../../../docs/file-scanner-v2-code-review-guide.md). +- For Parquet changes, also apply the guide's Parquet checklist and read + [FileScannerV2 Parquet Scan Design](../../../docs/file-scanner-v2-parquet-scan-design.md). +- For ORC changes, also apply the guide's ORC SARG and index checklist. +- These detailed guides are mandatory review instructions for their scope, not optional background + reading. Report any conflict between an implementation and the documented layer contract. + +## External Compatibility + +- Treat the external table-format specification and the behavior of supported external writers as + compatibility inputs. Do not assume Doris-generated fixtures or an existing Doris implementation + are authoritative when they conflict with the external contract. +- Do not require the external representation to behave like Doris internal storage. Verify the + complete translation from external semantics, through the format-v2 adapter, to the observable + Doris query result. Any intentional semantic difference must be documented and tested. +- Identify the compatibility matrix affected by a change: lake format and version, physical file + format and version, producing engine/writer, feature flags, encoding, compression codec, and + metadata version. Avoid fixes that only work for one writer's representation. +- Preserve backward compatibility with files and metadata produced by supported older versions. + For newer or unknown versions and features, follow the external specification's compatibility + rules; do not guess or silently reinterpret metadata. +- Review snapshot selection, time travel, manifest and partition evolution, schema and field IDs, + name and case matching, file identity, path normalization, and partition value decoding according + to the relevant lake-format semantics. +- Review writer-dependent physical representations, including Parquet logical annotations and + legacy encodings, ORC type attributes, timestamps and timezones, decimals, signedness, CHAR + padding, nested LIST/MAP layouts, null counts, NaN values, statistics, page/stripe indexes, and + optional or missing metadata. +- Capability detection and dispatch must happen before relying on a feature. Unsupported table + modes, metadata features, encodings, or semantic conversions must use the explicitly designed + fallback or return a clear error; they must never produce plausible but incorrect rows. +- Predicate, delete, statistics, and aggregate pushdown must return the same observable result as + reading and evaluating the external data without that optimization, including NULL, NaN, + timezone, collation/case, overflow, and precision edge cases. +- Check that a compatibility fix for one combination does not change existing behavior for other + lake formats, file formats, writers, or versions sharing the same abstraction. +- Require interoperability coverage using artifacts produced by representative external systems + such as Spark, Hive, Flink, or Trino when applicable. Prefer differential tests against a + non-pushdown path or the source system's expected result; do not rely only on files synthesized by + Doris test code. +- Each compatibility finding should state the affected external system or specification, versions + or writer variants, reachable input, Doris result, and expected result. + +## Performance and Observability + +- Treat per-row allocation, expression cloning, virtual dispatch, repeated schema work, unnecessary + column copies, and loss of lazy reads or pruning in hot paths as potential regressions. +- Check I/O ranges, caching, decompression, and batch sizing for accidental read amplification or + unbounded memory growth. +- Preserve profile counters and timers when control flow changes so filtered rows, bytes, reader + creation, and pushdown behavior remain diagnosable. + +## Tests + +- Require focused BE unit tests under `be/test/format_v2/`, following the source subdirectory when + possible. Add regression coverage when correctness depends on the FE-to-BE request or external + table integration. +- Include the relevant edge cases: empty input, all rows filtered, multiple blocks/splits/files, + EOF with and without output rows, nulls, missing/default columns, reordered or nested fields, and + malformed input. +- For bug fixes, require a test that fails for the original reachable path and validates the result, + row count, or explicit error after the fix. + +## Review Output + +- List findings first, ordered by severity. Each finding must identify the file and line, the + reachable execution path, and the concrete incorrect outcome. +- Distinguish verified defects from open questions. If no actionable defect is found, say so and + mention any important coverage or testing gap that remains. diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h new file mode 100644 index 00000000000000..ac510caaf07ac4 --- /dev/null +++ b/be/src/format_v2/column_data.h @@ -0,0 +1,416 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/consts.h" +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "exprs/vexpr_fwd.h" + +namespace doris::format { + +// File-local top-level column id. +// +// Scope: +// - Only valid inside one physical file schema returned by FileReader::get_schema(). +// - For Parquet, this is the top-level field ordinal in the new reader schema. +// - The synthetic row-position column also uses this type, with a reserved negative id. +// +// Do not use this for table/global column unique ids, block positions, nested child ids, or +// slot ids. Nested child ids are carried by LocalColumnIndex::index below. +class LocalColumnId { +public: + constexpr LocalColumnId() = default; + explicit constexpr LocalColumnId(int32_t id) : _id(id) {} + + static constexpr LocalColumnId invalid() { return LocalColumnId(); } + + constexpr int32_t value() const { return _id; } + constexpr bool is_valid() const { return _id >= 0; } + + constexpr bool operator==(const LocalColumnId& other) const { return _id == other._id; } + constexpr bool operator!=(const LocalColumnId& other) const { return !(*this == other); } + constexpr bool operator<(const LocalColumnId& other) const { return _id < other._id; } + +private: + int32_t _id = -1; +}; + +// Position of a file-local column in the Block produced by one FileScanRequest. +// +// This is assigned by TableColumnMapper/TableReader after predicate/non-predicate columns are +// deduplicated. It is not a file schema id and it is not stable across requests. Use value() only +// at the boundary where an existing Block or expression API still expects a size_t/int position. +class LocalIndex { +public: + constexpr LocalIndex() = default; + explicit constexpr LocalIndex(size_t index) : _index(index) {} + + constexpr size_t value() const { return _index; } + constexpr bool operator==(const LocalIndex& other) const { return _index == other._index; } + constexpr bool operator<(const LocalIndex& other) const { return _index < other._index; } + +private: + size_t _index = 0; +}; + +// Position of a table/global output column in the final Block returned by TableReader. +// +// This type is reserved for boundaries that need to refer to caller-visible column order. It must +// not be used to index a file-local Block, because schema evolution and lazy materialization can +// make file-local order different from table output order. +class GlobalIndex { +public: + constexpr GlobalIndex() = default; + explicit constexpr GlobalIndex(size_t index) : _index(index) {} + + constexpr size_t value() const { return _index; } + constexpr bool operator==(const GlobalIndex& other) const { return _index == other._index; } + constexpr bool operator<(const GlobalIndex& other) const { return _index < other._index; } + +private: + size_t _index = 0; +}; + +// Index of a split-local constant/default value used to materialize columns that are not read from +// the physical file, such as partition columns, added columns with default values, and virtual +// table-format columns. +// +// It is separate from LocalIndex because constants do not occupy a position in the file reader +// output block unless an expression explicitly materializes them. +class ConstantIndex { +public: + constexpr ConstantIndex() = default; + explicit constexpr ConstantIndex(size_t index) : _index(index) {} + + constexpr size_t value() const { return _index; } + constexpr bool operator==(const ConstantIndex& other) const { return _index == other._index; } + constexpr bool operator<(const ConstantIndex& other) const { return _index < other._index; } + +private: + size_t _index = 0; +}; + +inline std::ostream& operator<<(std::ostream& os, const LocalColumnId& id) { + return os << id.value(); +} + +inline std::ostream& operator<<(std::ostream& os, const LocalIndex& index) { + return os << index.value(); +} + +inline std::ostream& operator<<(std::ostream& os, const GlobalIndex& index) { + return os << index.value(); +} + +inline std::ostream& operator<<(std::ostream& os, const ConstantIndex& index) { + return os << index.value(); +} + +// A split/file-local constant value used to materialize a table/global column without reading a +// physical file column. +// +// Common producers are partition values, schema-evolution default expressions, generated columns +// and table-format virtual columns. The entry is keyed by ConstantIndex in ConstantMap; global_index +// keeps the link back to the caller-visible output column. +struct ConstantEntry { + GlobalIndex global_index; + VExprContextSPtr expr; + DataTypePtr type; +}; + +// Per mapping/split collection of constants. +// +// ConstantIndex only has meaning within this container. Keeping constants separate from LocalIndex +// makes it explicit that these values do not occupy positions in the file reader output Block. +class ConstantMap { +public: + ConstantIndex add(ConstantEntry entry) { + const auto index = ConstantIndex(_entries.size()); + _entries.push_back(std::move(entry)); + return index; + } + + const ConstantEntry& get(ConstantIndex index) const { + DORIS_CHECK(index.value() < _entries.size()); + return _entries[index.value()]; + } + + void clear() { _entries.clear(); } + bool empty() const { return _entries.empty(); } + size_t size() const { return _entries.size(); } + + const std::vector& entries() const { return _entries; } + +private: + std::vector _entries; +}; + +// Target of a localized filter. +// +// A filter can either reference a file-local Block position or a constant entry. Unset entries mean +// the filter cannot be evaluated below the table-reader finalize stage. +struct FilterEntry { + enum class Kind { + UNSET, + LOCAL, + CONSTANT, + }; + + static FilterEntry local(LocalIndex index) { + return {.kind = Kind::LOCAL, .index = index.value()}; + } + + static FilterEntry constant(ConstantIndex index) { + return {.kind = Kind::CONSTANT, .index = index.value()}; + } + + bool is_set() const { return kind != Kind::UNSET; } + bool is_local() const { return kind == Kind::LOCAL; } + bool is_constant() const { return kind == Kind::CONSTANT; } + + LocalIndex local_index() const { + DORIS_CHECK(is_local()); + return LocalIndex(index); + } + + ConstantIndex constant_index() const { + DORIS_CHECK(is_constant()); + return ConstantIndex(index); + } + + Kind kind = Kind::UNSET; + size_t index = 0; +}; + +enum ColumnType { + DATA_COLUMN = 0, // normal data column + ROW_NUMBER = 1, // row number in a file + GLOBAL_ROWID = 2, // global unique row id across files, used by TopN filter +}; + +struct GlobalRowIdContext { + uint8_t version = 0; + int64_t backend_id = 0; + uint32_t file_id = 0; +}; + +// Column schema definition shared by table/global projection and file-local schema matching. +// +// ColumnDefinition intentionally carries schema identity only. FE column unique ids are translated +// to GlobalIndex at the FileScannerV2 boundary and must not appear in table/file reader APIs. +struct ColumnDefinition { + // Typed identifier value used to match a column against another schema. + // + // - TYPE_NULL: no explicit identifier. BY_NAME falls back to ColumnDefinition::name. + // - TYPE_INT: interpreted by TableColumnMapperOptions::mode as a field id or file position. + // - TYPE_STRING: explicit name identifier. + // + // This is not the id that FileReader uses to read data. For example, a Parquet column can be + // matched by its optional Parquet field_id, while the reader still addresses it by a file-local + // ordinal. + Field identifier; + // Reader-local id of this node inside the file schema returned by FileReader::get_schema(). + // Top-level fields use the root column ordinal and nested fields use the child ordinal under + // their parent. -1 means unset; special virtual file columns may use other negative ids. + // Table/global ColumnDefinition values can leave this as -1 because they are not read directly + // by a FileReader. + int32_t local_id = -1; + // Logical table column name. This is also the matching name for by-name file formats. + std::string name; + // Historical or external names for the same logical field. Table formats such as Iceberg can + // use this to resolve partition path keys after column rename. + std::vector name_mapping {}; + DataTypePtr type; + // Semantic nested children for this schema node. + // + // Table/global columns carry projected table children. File-local schemas returned by + // FileReader::get_schema() also expose semantic children, not physical reader wrappers. For + // example, MAP children are key/value and ARRAY children contain only the element field. + std::vector children {}; + // Expression used to materialize missing/default/generated values when the column is not read + // directly from the file. + VExprContextSPtr default_expr = nullptr; + // Table-format initial default normalized for transport from FE. Binary-like values use Base64 + // and set initial_default_value_is_base64 because they can map to STRING/CHAR or VARBINARY. + // Unlike default_expr, this metadata is also available for hidden delete-predicate columns + // that are absent from the query projection. + std::optional initial_default_value = std::nullopt; + bool initial_default_value_is_base64 = false; + // Partition columns are constants from split metadata and should not be matched against file + // schema unless table-format logic explicitly asks for it. + bool is_partition_key = false; + // File-local column kind. For table/global columns this remains DATA_COLUMN. + ColumnType column_type = ColumnType::DATA_COLUMN; + + bool has_identifier() const { return !identifier.is_null(); } + bool has_identifier_field_id() const { return identifier.get_type() == TYPE_INT; } + bool has_identifier_name() const { return identifier.get_type() == TYPE_STRING; } + + // DuckDB-style helper for BY_FIELD_ID matching. The mapper binds the matching mode once, so a + // TYPE_INT identifier is interpreted as a field id only by the field-id matcher. + int32_t get_identifier_field_id() const { + DORIS_CHECK(has_identifier_field_id()); + return identifier.get(); + } + // DuckDB-style helper for BY_NAME matching. When no explicit string identifier is present, the + // logical column name is the identifier. + const std::string& get_identifier_name() const { + if (identifier.is_null()) { + return name; + } + DORIS_CHECK(has_identifier_name()); + return identifier.get(); + } + // Helper for BY_INDEX matching. BY_INDEX reuses the TYPE_INT identifier as the table-side file + // position, matching DuckDB's typed identifier plus mapper-mode interpretation. + int32_t get_identifier_position() const { + DORIS_CHECK(has_identifier_field_id()); + return identifier.get(); + } + + // Helper for reader-local projection and scan requests. + int32_t file_local_id() const { + if (local_id != -1) { + return local_id; + } + return get_identifier_field_id(); + } + + std::string debug_string() const; +}; + +static constexpr int ROW_POSITION_COLUMN_ID = -10001; +static constexpr const char* ROW_POSITION_COLUMN_NAME = "__file_row_position"; +static constexpr int GLOBAL_ROWID_COLUMN_ID = -10002; + +inline ColumnDefinition row_position_column_definition() { + ColumnDefinition field; + field.identifier = Field::create_field(ROW_POSITION_COLUMN_ID); + field.local_id = ROW_POSITION_COLUMN_ID; + field.name = ROW_POSITION_COLUMN_NAME; + field.type = std::make_shared(); + field.column_type = ColumnType::ROW_NUMBER; + return field; +} + +inline ColumnDefinition global_rowid_column_definition() { + ColumnDefinition field; + field.identifier = Field::create_field(BeConsts::GLOBAL_ROWID_COL); + field.local_id = GLOBAL_ROWID_COLUMN_ID; + field.name = BeConsts::GLOBAL_ROWID_COL; + field.type = std::make_shared(); + field.column_type = ColumnType::GLOBAL_ROWID; + return field; +} + +// Recursive file-local projection path. +// +// For a root entry in FileScanRequest::{predicate_columns, non_predicate_columns}, index is the +// top-level file column id and column_id() is valid. For children, index is the file-local child id +// under the parent node. This is the reader schema local id, not an Iceberg/Parquet field id, not a +// table child id, and not a child output ordinal. +// +// project_all_children=true means the whole subtree under this node is needed. When false, children +// lists the selected child paths. File readers can use this to avoid constructing readers for +// unprojected nested children. +struct LocalColumnIndex { + int32_t index = -1; + bool project_all_children = true; + std::vector children {}; + + static LocalColumnIndex top_level(LocalColumnId column_id) { + return {.index = column_id.value()}; + } + + static LocalColumnIndex local(int32_t local_id) { return {.index = local_id}; } + + static LocalColumnIndex partial_local(int32_t local_id) { + return {.index = local_id, .project_all_children = false}; + } + + LocalColumnId column_id() const { return LocalColumnId(index); } + int32_t local_id() const { return index; } + std::string debug_string() const; +}; + +inline bool is_full_projection(const LocalColumnIndex* projection) { + return projection == nullptr || projection->project_all_children; +} + +inline bool is_partial_projection(const LocalColumnIndex* projection) { + return projection != nullptr && !projection->project_all_children; +} + +inline const LocalColumnIndex* find_child_projection(const LocalColumnIndex* projection, + int32_t local_id) { + if (is_full_projection(projection)) { + return nullptr; + } + const auto child_it = std::find_if( + projection->children.begin(), projection->children.end(), + [&](const LocalColumnIndex& child) { return child.local_id() == local_id; }); + return child_it == projection->children.end() ? nullptr : &*child_it; +} + +inline bool is_child_projected(const LocalColumnIndex* projection, int32_t local_id) { + return is_full_projection(projection) || find_child_projection(projection, local_id) != nullptr; +} + +// Merge two projection trees that point to the same file-local node. +// +// A full projection dominates a partial projection. Two partial projections are merged by child id +// and recursively union their child paths. The caller must only merge projections for the same +// root/child node. +inline Status merge_local_column_index(LocalColumnIndex* target, const LocalColumnIndex& source) { + DORIS_CHECK(target != nullptr); + DORIS_CHECK(target->index == source.index); + if (target->project_all_children) { + return Status::OK(); + } + if (source.project_all_children) { + target->project_all_children = true; + target->children.clear(); + return Status::OK(); + } + for (const auto& source_child : source.children) { + auto target_child_it = std::find_if( + target->children.begin(), target->children.end(), + [&](const LocalColumnIndex& child) { return child.index == source_child.index; }); + if (target_child_it == target->children.end()) { + target->children.push_back(source_child); + continue; + } + RETURN_IF_ERROR(merge_local_column_index(&*target_child_it, source_child)); + } + return Status::OK(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp new file mode 100644 index 00000000000000..11064c04984e97 --- /dev/null +++ b/be/src/format_v2/column_mapper.cpp @@ -0,0 +1,2161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/column_mapper.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/consts.h" +#include "common/exception.h" +#include "common/status.h" +#include "core/data_type/convert_field_to_type.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/primitive_type.h" +#include "exprs/short_circuit_evaluation_expr.h" +#include "exprs/vcase_expr.h" +#include "exprs/vcast_expr.h" +#include "exprs/vcondition_expr.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr_context.h" +#include "exprs/vin_predicate.h" +#include "exprs/vliteral.h" +#include "exprs/vruntimefilter_wrapper.h" +#include "format_v2/column_mapper_nested.h" +#include "format_v2/expr/cast.h" +#include "format_v2/file_reader.h" +#include "format_v2/schema_projection.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/Exprs_types.h" + +namespace doris::format { + +namespace { + +std::string mapping_mode_to_string(TableColumnMappingMode mode) { + switch (mode) { + case TableColumnMappingMode::BY_FIELD_ID: + return "BY_FIELD_ID"; + case TableColumnMappingMode::BY_NAME: + return "BY_NAME"; + case TableColumnMappingMode::BY_INDEX: + return "BY_INDEX"; + } + return "UNKNOWN"; +} + +bool column_has_name(const ColumnDefinition& column, const std::string& name) { + if (to_lower(column.name) == to_lower(name)) { + return true; + } + if (column.has_identifier_name() && to_lower(column.get_identifier_name()) == to_lower(name)) { + return true; + } + return std::ranges::any_of(column.name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(name); + }); +} + +bool column_names_match(const ColumnDefinition& lhs, const ColumnDefinition& rhs) { + if (column_has_name(rhs, lhs.name)) { + return true; + } + if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { + return true; + } + return std::ranges::any_of(lhs.name_mapping, [&](const std::string& alias) { + return column_has_name(rhs, alias); + }); +} + +class ColumnMatcher { +public: + virtual ~ColumnMatcher() = default; + virtual const ColumnDefinition* find( + const ColumnDefinition& table_column, + const std::vector& file_schema) const = 0; +}; + +class FieldIdMatcher final : public ColumnMatcher { +public: + const ColumnDefinition* find(const ColumnDefinition& table_column, + const std::vector& file_schema) const override { + if (!table_column.has_identifier_field_id()) { + return nullptr; + } + const auto field_id = table_column.get_identifier_field_id(); + const auto field_it = std::ranges::find_if(file_schema, [&](const ColumnDefinition& field) { + return field.has_identifier_field_id() && field.get_identifier_field_id() == field_id; + }); + return field_it == file_schema.end() ? nullptr : &*field_it; + } +}; + +class NameMatcher final : public ColumnMatcher { +public: + const ColumnDefinition* find(const ColumnDefinition& table_column, + const std::vector& file_schema) const override { + const auto field_it = std::ranges::find_if(file_schema, [&](const ColumnDefinition& field) { + return column_names_match(table_column, field); + }); + return field_it == file_schema.end() ? nullptr : &*field_it; + } +}; + +class PositionMatcher final : public ColumnMatcher { +public: + const ColumnDefinition* find(const ColumnDefinition& table_column, + const std::vector& file_schema) const override { + if (!table_column.has_identifier_field_id()) { + return nullptr; + } + const auto position = table_column.get_identifier_position(); + if (position < 0 || static_cast(position) >= file_schema.size()) { + return nullptr; + } + return &file_schema[static_cast(position)]; + } +}; + +const ColumnMatcher& matcher_for_mode(TableColumnMappingMode mode) { + static const FieldIdMatcher field_id_matcher; + static const NameMatcher name_matcher; + static const PositionMatcher position_matcher; + switch (mode) { + case TableColumnMappingMode::BY_FIELD_ID: + return field_id_matcher; + case TableColumnMappingMode::BY_NAME: + return name_matcher; + case TableColumnMappingMode::BY_INDEX: + return position_matcher; + } + return field_id_matcher; +} + +std::string virtual_column_type_to_string(TableVirtualColumnType type) { + switch (type) { + case TableVirtualColumnType::INVALID: + return "INVALID"; + case TableVirtualColumnType::ROW_ID: + return "ROW_ID"; + case TableVirtualColumnType::LAST_UPDATED_SEQUENCE_NUMBER: + return "LAST_UPDATED_SEQUENCE_NUMBER"; + case TableVirtualColumnType::ICEBERG_ROWID: + return "ICEBERG_ROWID"; + } + return "UNKNOWN"; +} + +std::string filter_conversion_type_to_string(FilterConversionType type) { + switch (type) { + case FilterConversionType::COPY_DIRECTLY: + return "COPY_DIRECTLY"; + case FilterConversionType::CAST_FILTER: + return "CAST_FILTER"; + case FilterConversionType::READER_EXPRESSION: + return "READER_EXPRESSION"; + case FilterConversionType::FINALIZE_ONLY: + return "FINALIZE_ONLY"; + case FilterConversionType::CONSTANT: + return "CONSTANT"; + } + return "UNKNOWN"; +} + +std::string data_type_debug_string(const DataTypePtr& type) { + return type == nullptr ? "null" : type->get_name(); +} + +std::string field_debug_string(const Field& field) { + std::ostringstream out; + out << "Field{type=" << type_to_string(field.get_type()) << ", value="; + switch (field.get_type()) { + case TYPE_NULL: + out << "null"; + break; + case TYPE_INT: + out << field.get(); + break; + case TYPE_BIGINT: + out << field.get(); + break; + case TYPE_STRING: + out << field.get(); + break; + default: + out << field.to_debug_string(0); + break; + } + out << "}"; + return out.str(); +} + +template +std::string join_debug_strings(const std::vector& values, Formatter formatter) { + std::ostringstream out; + out << "["; + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) { + out << ", "; + } + out << formatter(values[i]); + } + out << "]"; + return out.str(); +} + +} // namespace + +const ColumnDefinition* find_column_by_name(const ColumnDefinition& table_column, + const std::vector& file_schema) { + return matcher_for_mode(TableColumnMappingMode::BY_NAME).find(table_column, file_schema); +} + +const Field* find_partition_value(const ColumnDefinition& table_column, + const std::map& partition_values) { + const auto find_by_name = [&](const std::string& name) -> const Field* { + const auto value_it = partition_values.find(name); + return value_it == partition_values.end() ? nullptr : &value_it->second; + }; + if (const auto* value = find_by_name(table_column.name); value != nullptr) { + return value; + } + if (table_column.has_identifier_name()) { + if (const auto* value = find_by_name(table_column.get_identifier_name()); + value != nullptr) { + return value; + } + } + for (const auto& alias : table_column.name_mapping) { + if (const auto* value = find_by_name(alias); value != nullptr) { + return value; + } + } + return nullptr; +} + +struct FileSlotRewriteInfo { + size_t block_position = 0; + DataTypePtr file_type; + DataTypePtr table_type; + std::string file_column_name; +}; + +struct RewriteContext { + RuntimeState* runtime_state = nullptr; + std::vector created_exprs {}; + + void add_created_expr(VExprSPtr expr) { created_exprs.push_back(std::move(expr)); } + + Status prepare_created_exprs(VExprContext* context) const { + DORIS_CHECK(context != nullptr); + RowDescriptor row_desc; + for (const auto& expr : created_exprs) { + if (dynamic_cast(expr.get()) != nullptr && runtime_state == nullptr) { + return Status::InvalidArgument( + "RuntimeState is required to prepare rewritten cast expression {}", + expr->expr_name()); + } + RETURN_IF_ERROR(expr->prepare(runtime_state, row_desc, context)); + } + return Status::OK(); + } +}; + +static VExprSPtr create_file_slot_ref(const VSlotRef& slot_ref, + const FileSlotRewriteInfo& rewrite_info, + RewriteContext* rewrite_context) { + auto ref = + VSlotRef::create_shared(slot_ref.slot_id(), cast_set(rewrite_info.block_position), + -1, rewrite_info.file_type, rewrite_info.file_column_name); + rewrite_context->add_created_expr(ref); + return ref; +} + +static bool is_cast_expr(const VExprSPtr& expr) { + return dynamic_cast(expr.get()) != nullptr; +} + +static bool is_binary_comparison_predicate(const VExprSPtr& expr) { + // BINARY_PRED and NULL_AWARE_BINARY_PRED are comparison-only node kinds. Nereids does not + // always populate the legacy opcode after resolving the comparison through TFunction, so + // requiring expr->op() to be set leaves an otherwise ordinary slot-literal predicate with + // mismatched implicit-coercion types. Row evaluation still works through the resolved + // function, but Parquet zonemap evaluation conservatively rejects the mismatched types. + return expr != nullptr && expr->get_num_children() == 2 && + (expr->node_type() == TExprNodeType::BINARY_PRED || + expr->node_type() == TExprNodeType::NULL_AWARE_BINARY_PRED); +} + +std::string TableColumnMapperOptions::debug_string() const { + std::ostringstream out; + out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) << "}"; + return out.str(); +} + +std::string ColumnDefinition::debug_string() const { + std::ostringstream out; + out << "ColumnDefinition{name=" << name << ", identifier=" << field_debug_string(identifier) + << ", name_mapping=" + << join_debug_strings(name_mapping, [](const std::string& name) { return name; }) + << ", local_id=" << local_id << ", type=" << data_type_debug_string(type) << ", children=" + << join_debug_strings(children, + [](const ColumnDefinition& child) { return child.debug_string(); }) + << ", has_default_expr=" << (default_expr != nullptr) + << ", is_partition_key=" << is_partition_key << "}"; + return out.str(); +} + +std::string LocalColumnIndex::debug_string() const { + std::ostringstream out; + out << "LocalColumnIndex{index=" << index << ", project_all_children=" << project_all_children + << ", children=" + << join_debug_strings(children, + [](const LocalColumnIndex& child) { return child.debug_string(); }) + << "}"; + return out.str(); +} + +std::string ColumnMapping::debug_string() const { + std::ostringstream out; + out << "ColumnMapping{global_index=" << global_index + << ", table_column_name=" << table_column_name << ", file_local_id="; + if (file_local_id.has_value()) { + out << *file_local_id; + } else { + out << "null"; + } + out << ", constant_index="; + if (constant_index.has_value()) { + out << *constant_index; + } else { + out << "null"; + } + out << ", file_column_name=" << file_column_name + << ", original_file_type=" << data_type_debug_string(original_file_type) + << ", original_file_children=" + << join_debug_strings(original_file_children, + [](const ColumnDefinition& child) { return child.debug_string(); }) + << ", file_type=" << data_type_debug_string(file_type) + << ", table_type=" << data_type_debug_string(table_type) + << ", has_projection=" << (projection != nullptr) << ", child_mappings=" + << join_debug_strings(child_mappings, + [](const ColumnMapping& child) { return child.debug_string(); }) + << ", is_trivial=" << is_trivial << ", is_constant=" << constant_index.has_value() + << ", filter_conversion=" << filter_conversion_type_to_string(filter_conversion) + << ", virtual_column_type=" << virtual_column_type_to_string(virtual_column_type) + << ", has_default_expr=" << (default_expr != nullptr) << "}"; + return out.str(); +} + +std::string TableColumnMapper::debug_string() const { + std::ostringstream out; + out << "TableColumnMapper{options=" << _options.debug_string() << ", mappings=" + << join_debug_strings(_mappings, + [](const ColumnMapping& mapping) { return mapping.debug_string(); }) + << ", hidden_mappings=" + << join_debug_strings(_hidden_mappings, + [](const ColumnMapping& mapping) { return mapping.debug_string(); }) + << ", constant_count=" << _constant_map.size() << "}"; + return out.str(); +} + +static const FileSlotRewriteInfo* find_slot_rewrite_info( + const VExprSPtr& expr, + const std::map& global_to_file_slot, + const VSlotRef** slot_ref) { + if (expr == nullptr) { + return nullptr; + } + VExprSPtr slot_expr = expr; + const bool input_is_cast = is_cast_expr(expr) && expr->get_num_children() == 1; + if (is_cast_expr(expr) && expr->get_num_children() == 1) { + slot_expr = expr->children()[0]; + } + if (!slot_expr->is_slot_ref()) { + return nullptr; + } + const auto* candidate_slot_ref = assert_cast(slot_expr.get()); + const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*candidate_slot_ref)); + if (rewrite_it == global_to_file_slot.end()) { + return nullptr; + } + if (input_is_cast && !expr->data_type()->equals(*rewrite_it->second.table_type)) { + return nullptr; + } + if (slot_ref != nullptr) { + *slot_ref = candidate_slot_ref; + } + return &rewrite_it->second; +} + +static bool filter_conversion_has_local_source(FilterConversionType conversion) { + switch (conversion) { + case FilterConversionType::COPY_DIRECTLY: + case FilterConversionType::CAST_FILTER: + case FilterConversionType::READER_EXPRESSION: + return true; + case FilterConversionType::FINALIZE_ONLY: + case FilterConversionType::CONSTANT: + return false; + } + return false; +} + +static bool table_filter_has_only_local_entries( + const TableFilter& table_filter, const std::map& filter_entries) { + for (const auto global_index : table_filter.global_indices) { + const auto entry_it = filter_entries.find(global_index); + if (entry_it == filter_entries.end() || !entry_it->second.is_local()) { + return false; + } + } + return true; +} + +static bool is_lossless_file_to_table_numeric_cast(const DataTypePtr& file_type, + const DataTypePtr& table_type); + +static Field literal_field_from_expr(const VExpr& literal_expr) { + DORIS_CHECK(literal_expr.is_literal()); + const auto* literal = dynamic_cast(&literal_expr); + DORIS_CHECK(literal != nullptr); + Field field; + literal->get_column_ptr()->get(0, field); + return field; +} + +static VExprSPtr unwrap_literal_for_file_cast(const VExprSPtr& expr, const DataTypePtr& table_type, + RewriteContext* rewrite_context) { + if (expr == nullptr) { + return nullptr; + } + + VExprSPtr literal; + if (expr->is_literal()) { + literal = expr; + } else if (is_cast_expr(expr) && expr->get_num_children() == 1 && + expr->children()[0]->is_literal() && expr->data_type()->equals(*table_type)) { + literal = expr->children()[0]; + } else { + return nullptr; + } + + if (literal->data_type()->equals(*table_type)) { + return literal; + } + // Nereids may leave a narrow numeric literal directly under a comparison and rely on the + // comparison's implicit type coercion, for example `INT slot = TINYINT 1`. Materialize that + // coercion before localizing the predicate so metadata readers still see slot-literal form. + if (!is_lossless_file_to_table_numeric_cast(literal->data_type(), table_type)) { + return nullptr; + } + + Field table_field; + try { + convert_field_to_type(literal_field_from_expr(*literal), *table_type, &table_field, + literal->data_type().get()); + } catch (const Exception&) { + return nullptr; + } + if (table_field.is_null() || + table_field.get_type() != remove_nullable(table_type)->get_primitive_type()) { + return nullptr; + } + auto normalized_literal = VLiteral::create_shared(table_type, std::move(table_field)); + rewrite_context->add_created_expr(normalized_literal); + return normalized_literal; +} + +// Table filter localization clones an already-prepared table expr and then rewrites it to file +// slots. Only split-local literals and BE cast nodes need table-reader-specific clone behavior; +// plain slot refs and literals use their own VExpr::clone_node(). +static Status clone_table_expr_node(const VExpr& expr, VExprSPtr* cloned_expr) { + DORIS_CHECK(cloned_expr != nullptr); + if (const auto* split_literal = dynamic_cast(&expr)) { + *cloned_expr = std::make_shared( + split_literal->data_type(), literal_field_from_expr(expr), + split_literal->original_type(), split_literal->original_field()); + } else if (const auto* vcast_expr = dynamic_cast(&expr); + vcast_expr != nullptr && vcast_expr->node_type() == TExprNodeType::CAST_EXPR) { + *cloned_expr = Cast::create_shared(vcast_expr->data_type()); + } + return Status::OK(); +} + +Status clone_table_expr_tree(const VExprSPtr& expr, VExprSPtr* cloned_expr) { + DORIS_CHECK(cloned_expr != nullptr); + if (expr == nullptr) { + *cloned_expr = nullptr; + return Status::OK(); + } + return expr->deep_clone(cloned_expr, clone_table_expr_node); +} + +static VExprSPtr original_table_literal(const VExprSPtr& literal_expr, + RewriteContext* rewrite_context = nullptr) { + DORIS_CHECK(literal_expr != nullptr); + DORIS_CHECK(literal_expr->is_literal()); + const auto* rewritten_literal = dynamic_cast(literal_expr.get()); + if (rewritten_literal == nullptr) { + return literal_expr; + } + auto literal = VLiteral::create_shared(rewritten_literal->original_type(), + rewritten_literal->original_field()); + if (rewrite_context != nullptr) { + rewrite_context->add_created_expr(literal); + } + return literal; +} + +static ColumnDefinition hidden_column_from_slot_ref(const VSlotRef& slot_ref) { + ColumnDefinition column; + column.name = slot_ref.column_name(); + column.identifier = Field::create_field(column.name); + column.type = slot_ref.data_type(); + return column; +} + +static void collect_top_level_slot_columns(const VExprSPtr& expr, + std::map* columns) { + DORIS_CHECK(columns != nullptr); + if (expr == nullptr) { + return; + } + if (expr->is_slot_ref()) { + const auto* slot_ref = assert_cast(expr.get()); + columns->try_emplace(slot_ref_global_index(*slot_ref), + hidden_column_from_slot_ref(*slot_ref)); + return; + } + for (const auto& child : expr->children()) { + collect_top_level_slot_columns(child, columns); + } +} + +static std::optional signed_integer_width(PrimitiveType type) { + switch (type) { + case TYPE_TINYINT: + return 8; + case TYPE_SMALLINT: + return 16; + case TYPE_INT: + return 32; + case TYPE_BIGINT: + return 64; + case TYPE_LARGEINT: + return 128; + default: + return std::nullopt; + } +} + +static std::optional floating_width(PrimitiveType type) { + switch (type) { + case TYPE_FLOAT: + return 32; + case TYPE_DOUBLE: + return 64; + default: + return std::nullopt; + } +} + +static std::optional floating_exact_integer_width(PrimitiveType type) { + switch (type) { + case TYPE_FLOAT: + return 24; + case TYPE_DOUBLE: + return 53; + default: + return std::nullopt; + } +} + +static bool is_lossless_file_to_table_numeric_cast(const DataTypePtr& file_type, + const DataTypePtr& table_type) { + const auto file_nested_type = remove_nullable(file_type); + const auto table_nested_type = remove_nullable(table_type); + if (file_nested_type->equals(*table_nested_type)) { + return true; + } + + const auto file_primitive_type = file_nested_type->get_primitive_type(); + const auto table_primitive_type = table_nested_type->get_primitive_type(); + if (const auto file_width = signed_integer_width(file_primitive_type)) { + if (const auto table_width = signed_integer_width(table_primitive_type)) { + return *table_width >= *file_width; + } + if (const auto table_width = floating_exact_integer_width(table_primitive_type)) { + return *table_width >= *file_width; + } + return false; + } + if (const auto file_width = floating_width(file_primitive_type)) { + const auto table_width = floating_width(table_primitive_type); + return table_width.has_value() && *table_width >= *file_width; + } + return false; +} + +static VExprSPtr rewrite_literal_to_file_type(const VExprSPtr& literal_expr, + const FileSlotRewriteInfo& rewrite_info, + RewriteContext* rewrite_context) { + DORIS_CHECK(literal_expr != nullptr); + DORIS_CHECK(literal_expr->is_literal()); + const auto original_literal = original_table_literal(literal_expr, rewrite_context); + const Field original_field = literal_field(original_literal); + if (rewrite_info.file_type->equals(*original_literal->data_type())) { + return original_literal; + } + // A literal round trip alone cannot prove that file-local evaluation is safe: the file slot + // itself may lose information when materialized as the table type. For example, DOUBLE 1.5 + // becomes BIGINT 1, so table predicate `value = 1` is true while file predicate + // `value = 1.0` is false. Complex Field equality also does not compare nested contents. + // Restrict localization to scalar numeric casts that preserve every file value; unsupported + // and complex casts keep the table predicate and evaluate after materialization. + if (!is_lossless_file_to_table_numeric_cast(rewrite_info.file_type, + original_literal->data_type())) { + return nullptr; + } + Field file_field; + try { + convert_field_to_type(original_field, *rewrite_info.file_type, &file_field, + original_literal->data_type().get()); + } catch (const Exception&) { + return nullptr; + } + if (file_field.is_null()) { + return nullptr; + } + if (file_field.get_type() != remove_nullable(rewrite_info.file_type)->get_primitive_type()) { + return nullptr; + } + Field round_trip_field; + try { + convert_field_to_type(file_field, *original_literal->data_type(), &round_trip_field, + rewrite_info.file_type.get()); + } catch (const Exception&) { + return nullptr; + } + // The file-to-table type check protects every possible file value. This round trip separately + // proves that the specific predicate boundary is exactly representable in the file type. + if (round_trip_field != original_field) { + return nullptr; + } + auto literal = std::make_shared( + rewrite_info.file_type, file_field, original_literal->data_type(), original_field); + rewrite_context->add_created_expr(literal); + return literal; +} + +static bool rewrite_binary_slot_literal_predicate( + const VExprSPtr& expr, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context) { + if (!is_binary_comparison_predicate(expr)) { + return false; + } + auto children = expr->children(); + const VSlotRef* slot_ref = nullptr; + const FileSlotRewriteInfo* rewrite_info = + find_slot_rewrite_info(children[0], global_to_file_slot, &slot_ref); + int slot_child_idx = 0; + int literal_child_idx = 1; + if (rewrite_info == nullptr) { + rewrite_info = find_slot_rewrite_info(children[1], global_to_file_slot, &slot_ref); + slot_child_idx = 1; + literal_child_idx = 0; + } + if (rewrite_info == nullptr || slot_ref == nullptr) { + return false; + } + auto literal_expr = unwrap_literal_for_file_cast(children[literal_child_idx], + rewrite_info->table_type, rewrite_context); + if (literal_expr == nullptr) { + return false; + } + + auto rewritten_literal = + rewrite_literal_to_file_type(literal_expr, *rewrite_info, rewrite_context); + if (rewritten_literal == nullptr) { + children[literal_child_idx] = original_table_literal(literal_expr, rewrite_context); + expr->set_children(std::move(children)); + return false; + } + + children[slot_child_idx] = create_file_slot_ref(*slot_ref, *rewrite_info, rewrite_context); + children[literal_child_idx] = std::move(rewritten_literal); + expr->set_children(std::move(children)); + return true; +} + +static bool rewrite_in_slot_literal_predicate( + const VExprSPtr& expr, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context) { + if (expr->node_type() != TExprNodeType::IN_PRED || expr->get_num_children() < 2) { + return false; + } + auto children = expr->children(); + const VSlotRef* slot_ref = nullptr; + const FileSlotRewriteInfo* rewrite_info = + find_slot_rewrite_info(children[0], global_to_file_slot, &slot_ref); + if (rewrite_info == nullptr || slot_ref == nullptr) { + return false; + } + + VExprSPtrs rewritten_literals; + rewritten_literals.reserve(children.size() - 1); + for (size_t child_idx = 1; child_idx < children.size(); ++child_idx) { + auto literal_expr = unwrap_literal_for_file_cast(children[child_idx], + rewrite_info->table_type, rewrite_context); + if (literal_expr == nullptr) { + return false; + } + auto rewritten_literal = + rewrite_literal_to_file_type(literal_expr, *rewrite_info, rewrite_context); + if (rewritten_literal == nullptr) { + for (size_t restore_idx = 1; restore_idx < children.size(); ++restore_idx) { + auto restore_literal = unwrap_literal_for_file_cast( + children[restore_idx], rewrite_info->table_type, rewrite_context); + if (restore_literal != nullptr) { + children[restore_idx] = + original_table_literal(restore_literal, rewrite_context); + } + } + expr->set_children(std::move(children)); + return false; + } + rewritten_literals.push_back(std::move(rewritten_literal)); + } + + children[0] = create_file_slot_ref(*slot_ref, *rewrite_info, rewrite_context); + for (size_t literal_idx = 0; literal_idx < rewritten_literals.size(); ++literal_idx) { + children[literal_idx + 1] = std::move(rewritten_literals[literal_idx]); + } + expr->set_children(std::move(children)); + return true; +} + +static VExprSPtr create_file_struct_child_name_literal(const std::string& file_child_name, + RewriteContext* rewrite_context) { + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(file_child_name)); + rewrite_context->add_created_expr(literal); + return literal; +} + +static bool needs_complex_file_slot_cast(const DataTypePtr& file_type, + const DataTypePtr& table_type) { + if (file_type == nullptr || table_type == nullptr || file_type->equals(*table_type)) { + return false; + } + const auto file_nested_type = remove_nullable(file_type); + const auto table_nested_type = remove_nullable(table_type); + if (file_nested_type->equals(*table_nested_type)) { + return false; + } + return is_complex_type(file_nested_type->get_primitive_type()) || + is_complex_type(table_nested_type->get_primitive_type()); +} + +static bool collect_struct_element_chain(const VExprSPtr& expr, std::vector* chain) { + DORIS_CHECK(chain != nullptr); + if (!is_struct_element_expr(expr)) { + return false; + } + const auto& parent = expr->children()[0]; + if (is_struct_element_expr(parent)) { + if (!collect_struct_element_chain(parent, chain)) { + return false; + } + } else if (!parent->is_slot_ref()) { + // Only support file-local rewrite for struct child chains rooted directly at a top-level + // slot, for example `element_at(s, 'a')` or `element_at(element_at(s, 'a'), 'b')`. + // + // Do not localize computed complex parents such as + // `element_at(element_at(map_values(m), 1), 'full_name')`. The intermediate map/array + // result has already been reshaped by scan projection and may have a different child order + // from the table expression. Partially rewriting that expression against the file block can + // silently evaluate the wrong struct child and filter out valid rows. Those predicates must + // remain as table-level conjuncts and be evaluated after TableReader materialization. + return false; + } + chain->push_back(expr); + return true; +} + +static bool rewrite_struct_element_path_to_file_expr( + const VExprSPtr& expr, const std::vector& mappings, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context) { + ResolvedNestedStructPath resolved; + if (!resolve_nested_struct_expr_for_file(expr, mappings, &resolved)) { + return false; + } + + std::vector struct_element_chain; + if (!collect_struct_element_chain(expr, &struct_element_chain) || + struct_element_chain.size() != resolved.file_child_names.size() || + struct_element_chain.size() != resolved.file_child_types.size()) { + return false; + } + + auto root_children = struct_element_chain.front()->children(); + if (!root_children[0]->is_slot_ref()) { + return false; + } + const auto* slot_ref = assert_cast(root_children[0].get()); + const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); + if (rewrite_it == global_to_file_slot.end()) { + return false; + } + + // File-local conjuncts are prepared against the file-reader Block, so both the root slot and + // every struct selector must be expressed in file schema terms. For a renamed Iceberg field, + // keeping the table selector would prepare `element_at(file_struct, 'renamed')` and + // fail before any rows are read. Rewrite the whole chain while ColumnMapping still preserves + // the table-to-file relationship. Example: + // table filter: element_at(element_at(s, 'renamed_parent'), 'renamed_leaf') + // old file: s> + // file filter: element_at(element_at(s, 'parent'), 'leaf') + root_children[0] = create_file_slot_ref(*slot_ref, rewrite_it->second, rewrite_context); + struct_element_chain.front()->set_children(std::move(root_children)); + for (size_t idx = 0; idx < struct_element_chain.size(); ++idx) { + auto children = struct_element_chain[idx]->children(); + children[1] = create_file_struct_child_name_literal(resolved.file_child_names[idx], + rewrite_context); + struct_element_chain[idx]->set_children(std::move(children)); + // The selector name and the expression return type must be moved to file schema together. + // Example: + // table filter: element_at(element_at(s, 'new_a'), 'new_aa') = 50 + // old file: s.new_a STRUCT + // file filter: element_at(element_at(s, 'new_a'), 'aa') = 50 + // + // If the inner element_at keeps the table return type STRUCT, preparing the + // outer element_at(..., 'aa') fails before scanning because `aa` is not a table field. + struct_element_chain[idx]->data_type() = resolved.file_child_types[idx]; + } + return true; +} + +static VExprSPtr rewrite_table_expr_to_file_expr( + const VExprSPtr& expr, + const std::map& global_to_file_slot, + const std::vector& filter_mappings, RewriteContext* rewrite_context, + bool* can_localize) { + if (expr == nullptr) { + return nullptr; + } + DORIS_CHECK(rewrite_context != nullptr); + DORIS_CHECK(can_localize != nullptr); + if (auto* runtime_filter = dynamic_cast(expr.get()); + runtime_filter != nullptr) { + auto impl = runtime_filter->get_impl(); + if (impl == nullptr) { + *can_localize = false; + return expr; + } + auto localized_impl = rewrite_table_expr_to_file_expr( + impl, global_to_file_slot, filter_mappings, rewrite_context, can_localize); + if (!*can_localize) { + return expr; + } + runtime_filter->set_impl(std::move(localized_impl)); + return expr; + } + if (rewrite_binary_slot_literal_predicate(expr, global_to_file_slot, rewrite_context)) { + return expr; + } + if (rewrite_in_slot_literal_predicate(expr, global_to_file_slot, rewrite_context)) { + return expr; + } + if (is_struct_element_expr(expr)) { + if (!rewrite_struct_element_path_to_file_expr(expr, filter_mappings, global_to_file_slot, + rewrite_context)) { + // The scanner still evaluates the original table-level conjunct after TableReader + // finalizes the output block. Skipping an unlocalizable file conjunct is therefore + // safer than preparing a partially rewritten expression against the wrong struct + // layout. In particular, do not generate file-local conjuncts for computed complex + // parents such as `element_at(element_at(map_values(m), 1), 'field')`; only direct + // slot-rooted struct chains are supported here. + *can_localize = false; + } + return expr; + } + if (expr->is_slot_ref()) { + const auto* slot_ref = assert_cast(expr.get()); + const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); + if (rewrite_it != global_to_file_slot.end()) { + const auto& rewrite_info = rewrite_it->second; + auto file_slot = create_file_slot_ref(*slot_ref, rewrite_info, rewrite_context); + if (rewrite_info.file_type->equals(*rewrite_info.table_type)) { + return file_slot; + } + if (needs_complex_file_slot_cast(rewrite_info.file_type, rewrite_info.table_type)) { + // Generic file-local expressions cannot safely cast an evolved complex file slot + // back to the table type. Example: + // + // table filter: ARRAY_CONTAINS(MAP_KEYS(m), 'person5') + // old file: m MAP> + // table: m MAP> + // + // Although MAP_KEYS only reads the key column, wrapping the file slot as + // `CAST(file_m AS table_m)` forces the value struct cast first and fails because + // the old and new value structs have different fields. Keep such filters at the + // table level, where TableReader materializes the evolved complex value before + // Scanner evaluates the original conjunct. Direct slot-rooted struct child paths + // are handled by rewrite_struct_element_path_to_file_expr() above. + *can_localize = false; + return expr; + } + auto cast_expr = Cast::create_shared(rewrite_info.table_type); + cast_expr->add_child(std::move(file_slot)); + rewrite_context->add_created_expr(cast_expr); + return cast_expr; + } + return expr; + } + // The input is a split-local cloned tree. A previous split-local clone may already have + // inserted Cast(slot). Keep that rewrite idempotent: rewrite the cast child from table slot to + // the current split's file slot, and drop the cast when the current split no longer needs it. + if (is_cast_expr(expr) && expr->get_num_children() == 1) { + const auto& child = expr->children()[0]; + if (child->is_slot_ref()) { + const auto* slot_ref = assert_cast(child.get()); + const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); + if (rewrite_it != global_to_file_slot.end() && + expr->data_type()->equals(*rewrite_it->second.table_type)) { + auto rewritten_child = + create_file_slot_ref(*slot_ref, rewrite_it->second, rewrite_context); + if (rewrite_it->second.file_type->equals(*rewrite_it->second.table_type)) { + return rewritten_child; + } + if (needs_complex_file_slot_cast(rewrite_it->second.file_type, + rewrite_it->second.table_type)) { + *can_localize = false; + return expr; + } + expr->set_children({std::move(rewritten_child)}); + return expr; + } + } + } + + VExprSPtrs rewritten_children; + rewritten_children.reserve(expr->children().size()); + for (const auto& child : expr->children()) { + rewritten_children.push_back(rewrite_table_expr_to_file_expr( + child, global_to_file_slot, filter_mappings, rewrite_context, can_localize)); + } + expr->set_children(std::move(rewritten_children)); + return expr; +} + +static constexpr const char* ROW_LINEAGE_ROW_ID = "_row_id"; +static constexpr const char* ROW_LINEAGE_LAST_UPDATED_SEQ_NUMBER = "_last_updated_sequence_number"; +static constexpr int32_t ROW_LINEAGE_ROW_ID_FIELD_ID = 2147483540; +static constexpr int32_t ROW_LINEAGE_LAST_UPDATED_SEQ_NUMBER_FIELD_ID = 2147483539; + +static TableVirtualColumnType row_lineage_virtual_column_type(const std::string& column_name) { + if (column_name == ROW_LINEAGE_ROW_ID) { + return TableVirtualColumnType::ROW_ID; + } + if (column_name == ROW_LINEAGE_LAST_UPDATED_SEQ_NUMBER) { + return TableVirtualColumnType::LAST_UPDATED_SEQUENCE_NUMBER; + } + return TableVirtualColumnType::INVALID; +} + +static TableVirtualColumnType row_lineage_virtual_column_type_by_field_id( + const ColumnDefinition& column) { + if (!column.has_identifier_field_id()) { + return TableVirtualColumnType::INVALID; + } + switch (column.get_identifier_field_id()) { + case ROW_LINEAGE_ROW_ID_FIELD_ID: + return TableVirtualColumnType::ROW_ID; + case ROW_LINEAGE_LAST_UPDATED_SEQ_NUMBER_FIELD_ID: + return TableVirtualColumnType::LAST_UPDATED_SEQUENCE_NUMBER; + default: + return TableVirtualColumnType::INVALID; + } +} + +static TableVirtualColumnType row_lineage_virtual_column_type(const ColumnDefinition& column, + TableColumnMappingMode mode) { + switch (mode) { + case TableColumnMappingMode::BY_FIELD_ID: + return row_lineage_virtual_column_type_by_field_id(column); + case TableColumnMappingMode::BY_NAME: + case TableColumnMappingMode::BY_INDEX: + return row_lineage_virtual_column_type(column.name); + } + return TableVirtualColumnType::INVALID; +} + +// Returns true when the current file type is not the exact nested type the scan should expose. +// This is about building the projected file-side type/projection, not about whether TableReader +// later needs to rematerialize the complex value back to table layout. +static bool needs_projected_file_type_rebuild(const ColumnMapping& mapping) { + if (!is_complex_type(mapping.file_type->get_primitive_type())) { + return false; + } + if (mapping.child_mappings.empty()) { + return false; + } + DORIS_CHECK(mapping.file_type != nullptr); + DORIS_CHECK(mapping.table_type != nullptr); + if (remove_nullable(mapping.file_type)->get_primitive_type() != + remove_nullable(mapping.table_type)->get_primitive_type()) { + return true; + } + if (!mapping.table_type->equals(*mapping.file_type)) { + return true; + } + for (const auto& child_mapping : mapping.child_mappings) { + // Rename-only child mappings do not change the file-side projected shape. If field-id + // matching maps table child `renamed_b` to file child `b`, the file reader can still expose + // the original file type as long as child count/order/types are unchanged. + if (!child_mapping.file_local_id.has_value() || + needs_projected_file_type_rebuild(child_mapping)) { + return true; + } + } + return false; +} + +static std::optional file_child_ordinal_in_scan_type(const ColumnMapping& mapping, + const ColumnMapping& child_mapping) { + if (!child_mapping.file_local_id.has_value()) { + return std::nullopt; + } + const auto& file_children = !mapping.projected_file_children.empty() + ? mapping.projected_file_children + : mapping.original_file_children; + const auto child_it = std::ranges::find_if(file_children, [&](const ColumnDefinition& child) { + return child.file_local_id() == *child_mapping.file_local_id; + }); + if (child_it == file_children.end()) { + return std::nullopt; + } + return static_cast(std::distance(file_children.begin(), child_it)); +} + +static bool needs_complex_rematerialize(const ColumnMapping& mapping) { + if (mapping.child_mappings.empty()) { + return false; + } + if (mapping.table_type == nullptr || mapping.file_type == nullptr || + !mapping.table_type->equals(*mapping.file_type)) { + return true; + } + for (size_t table_child_idx = 0; table_child_idx < mapping.child_mappings.size(); + ++table_child_idx) { + const auto& child_mapping = mapping.child_mappings[table_child_idx]; + const auto file_child_idx = file_child_ordinal_in_scan_type(mapping, child_mapping); + if (!file_child_idx.has_value() || *file_child_idx != table_child_idx || + needs_complex_rematerialize(child_mapping) || + (child_mapping.table_type != nullptr && child_mapping.file_type != nullptr && + !child_mapping.table_type->equals(*child_mapping.file_type))) { + return true; + } + } + return false; +} + +static bool mapping_can_use_file_column_directly(const ColumnMapping& mapping) { + if (mapping.table_type == nullptr || mapping.file_type == nullptr) { + return false; + } + const auto table_type = remove_nullable(mapping.table_type); + const auto file_type = remove_nullable(mapping.file_type); + const bool same_timestamptz_with_different_scale = + table_type->get_primitive_type() == TYPE_TIMESTAMPTZ && + file_type->get_primitive_type() == TYPE_TIMESTAMPTZ; + if (!mapping.table_type->equals(*mapping.file_type) && !same_timestamptz_with_different_scale) { + return false; + } + return !needs_complex_rematerialize(mapping); +} + +static bool type_contains_varbinary(const DataTypePtr& type) { + DORIS_CHECK(type != nullptr); + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_VARBINARY: + return true; + case TYPE_ARRAY: + return type_contains_varbinary( + assert_cast(*nested_type).get_nested_type()); + case TYPE_MAP: { + const auto& map_type = assert_cast(*nested_type); + return type_contains_varbinary(map_type.get_key_type()) || + type_contains_varbinary(map_type.get_value_type()); + } + case TYPE_STRUCT: + return std::ranges::any_of( + assert_cast(*nested_type).get_elements(), + [](const DataTypePtr& child_type) { return type_contains_varbinary(child_type); }); + default: + return false; + } +} + +static FilterConversionType direct_filter_conversion(const ColumnMapping& mapping) { + DORIS_CHECK(mapping.table_type != nullptr); + DORIS_CHECK(mapping.file_type != nullptr); + // FileScanOperator deliberately keeps VARBINARY predicates above external readers. Their + // physical binary representations are not uniformly supported by reader-side expression and + // metadata filtering, so localizing a late runtime filter here can incorrectly reject rows. + // Apply the same rule to a complex root because generic array/map/struct expressions rewrite + // the root slot and can otherwise expose a nested VARBINARY child to the reader. + if (type_contains_varbinary(mapping.table_type)) { + return FilterConversionType::FINALIZE_ONLY; + } + const auto table_type = remove_nullable(mapping.table_type); + const auto file_type = remove_nullable(mapping.file_type); + // TIMESTAMPTZ scale mismatch is intentionally materialized as pass-through: a SQL cast rounds + // fractional seconds. A file-local cast would therefore filter different instants from the + // scanner-level predicate evaluated on the pass-through value. + if (table_type->get_primitive_type() == TYPE_TIMESTAMPTZ && + file_type->get_primitive_type() == TYPE_TIMESTAMPTZ && + !mapping.table_type->equals(*mapping.file_type)) { + return FilterConversionType::FINALIZE_ONLY; + } + return mapping.is_trivial ? FilterConversionType::COPY_DIRECTLY + : FilterConversionType::CAST_FILTER; +} + +static FilterConversionType projected_filter_conversion(const ColumnMapping& mapping) { + const auto conversion = direct_filter_conversion(mapping); + return !mapping.is_trivial && conversion != FilterConversionType::FINALIZE_ONLY + ? FilterConversionType::READER_EXPRESSION + : conversion; +} + +static const ColumnDefinition* find_file_child_for_mapping(const ColumnDefinition& table_child, + const ColumnDefinition& file_parent, + TableColumnMappingMode mode, + size_t table_child_idx, + bool allow_ordinal_fallback) { + const auto file_parent_type = remove_nullable(file_parent.type)->get_primitive_type(); + switch (file_parent_type) { + case TYPE_ARRAY: + DORIS_CHECK(file_parent.children.size() == 1); + return &file_parent.children[0]; + case TYPE_MAP: + DORIS_CHECK(file_parent.children.size() == 2); + if (table_child.name == "key") { + return &file_parent.children[0]; + } + if (table_child.name == "value") { + return &file_parent.children[1]; + } + if (table_child.local_id == 0 || table_child.local_id == 1) { + return &file_parent.children[table_child.local_id]; + } + return nullptr; + default: + // Hive BY_INDEX is a top-level column matching rule. Once a complex root is selected by + // file position, nested struct children follow Hive reader's historical name matching + // semantics; their integer identifiers can be field ids, not file positions. + const auto nested_mode = + mode == TableColumnMappingMode::BY_INDEX ? TableColumnMappingMode::BY_NAME : mode; + if (const auto* file_child = + matcher_for_mode(nested_mode).find(table_child, file_parent.children); + file_child != nullptr) { + return file_child; + } + if (allow_ordinal_fallback && mode == TableColumnMappingMode::BY_FIELD_ID && + !table_child.has_identifier_field_id()) { + // Synthetic children are derived from the table DataType when nested ColumnDefinition + // metadata has been pruned away. They do not carry Iceberg field ids, so try a name + // match before falling back to ordinal order. Example: + // table value type: Struct(age, full_name, gender) + // old file value: Struct(name, age) + // Name matching keeps `age -> age`; the later unused-child fallback can then map the + // renamed `full_name -> name` instead of consuming `age` twice. + if (const auto* file_child = NameMatcher().find(table_child, file_parent.children); + file_child != nullptr) { + return file_child; + } + } + // Some callers only carry the full complex DataType for a projected table column, without + // expanded nested ColumnDefinitions. In that case we can still preserve full materialization + // by walking table/file struct fields by ordinal. This is a fallback only: explicit + // ColumnDefinition children keep using the requested table-format matching rule, which is + // required for precise schema evolution. + if (allow_ordinal_fallback && table_child_idx < file_parent.children.size()) { + return &file_parent.children[table_child_idx]; + } + return nullptr; + } +} + +static ColumnDefinition synthetic_child_definition(const std::string& name, DataTypePtr type, + int32_t local_id) { + ColumnDefinition child; + child.identifier = Field::create_field(name); + child.local_id = local_id; + child.name = name; + child.type = std::move(type); + return child; +} + +static std::vector synthesize_complex_children_from_type( + const DataTypePtr& type) { + std::vector children; + if (type == nullptr) { + return children; + } + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_ARRAY: { + const auto* array_type = assert_cast(nested_type.get()); + children.push_back(synthetic_child_definition("element", array_type->get_nested_type(), 0)); + break; + } + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + children.push_back(synthetic_child_definition("key", map_type->get_key_type(), 0)); + children.push_back(synthetic_child_definition("value", map_type->get_value_type(), 1)); + break; + } + case TYPE_STRUCT: { + const auto* struct_type = assert_cast(nested_type.get()); + children.reserve(struct_type->get_elements().size()); + for (size_t idx = 0; idx < struct_type->get_elements().size(); ++idx) { + children.push_back(synthetic_child_definition(struct_type->get_element_name(idx), + struct_type->get_element(idx), + cast_set(idx))); + } + break; + } + default: + break; + } + return children; +} + +static void align_struct_child_types_with_parent(const DataTypePtr& parent_type, + std::vector& children) { + const auto nested_parent_type = remove_nullable(parent_type); + DORIS_CHECK(nested_parent_type->get_primitive_type() == TYPE_STRUCT); + const auto type_children = synthesize_complex_children_from_type(parent_type); + for (auto& child : children) { + const auto type_child = std::ranges::find_if( + type_children, [&](const auto& candidate) { return candidate.name == child.name; }); + DORIS_CHECK(type_child != type_children.end()) + << "Complex child '" << child.name + << "' is absent from its parent table type: " << parent_type->get_name(); + // The parent DataType is the authoritative output contract. Nested schema descriptors can + // omit child nullability even though the parent struct still declares Nullable(String). + // For example, the Iceberg full-schema-change case maps nullable `location` to `city`, but + // its child descriptor carries String. Keeping String here makes rematerialization strip + // the child's null map and creates Struct(String) under a Struct(Nullable(String)) type. + child.type = type_child->type; + } +} + +static bool has_table_child_named(const std::vector& children, + std::string_view name) { + return std::ranges::any_of(children, [&](const ColumnDefinition& child) { + return std::string_view(child.name) == name; + }); +} + +static void complete_required_complex_children_from_type(const DataTypePtr& type, + std::vector& children) { + if (type == nullptr) { + return; + } + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + // MAP key/value are structural children, not independently materializable table fields. + // A key-only projection can still be attached to a whole-map output root, for example: + // SELECT * FROM t WHERE ARRAY_CONTAINS(MAP_KEYS(new_map_column), 'person5') + // + // In that shape the scanner keeps the value stream readable, but the table projection can + // carry only the key child. Add the missing value child so recursive mapping can evolve the + // value type instead of letting TableReader cast old/new value structs directly. + if (has_table_child_named(children, "key") && !has_table_child_named(children, "value")) { + children.push_back(synthetic_child_definition("value", map_type->get_value_type(), 1)); + } + break; + } + case TYPE_ARRAY: + // ARRAY has only one required structural child (`element`), so a non-empty projection is + // already rooted at the element path. + break; + case TYPE_STRUCT: + // STRUCT children are real fields and must remain prunable. Completing missing struct + // fields here would turn `SELECT s.a` into a full-struct read and undo nested projection. + break; + default: + break; + } +} + +struct PreparedTableChildren { + std::vector children; + bool synthesized_from_type = false; +}; + +static PreparedTableChildren prepare_table_children_for_mapping( + const ColumnDefinition& table_column, const DataTypePtr& file_type) { + PreparedTableChildren prepared {.children = table_column.children}; + const auto nested_table_type = remove_nullable(table_column.type); + + // Some scan paths, especially SELECT *, only carry the complete complex DataType for a table + // column and leave ColumnDefinition::children empty. Synthesize the hierarchy so recursive + // mapping can evolve nested fields instead of falling back to an invalid whole-column cast. + prepared.synthesized_from_type = prepared.children.empty() && + is_complex_type(nested_table_type->get_primitive_type()) && + !table_column.type->equals(*file_type); + if (prepared.synthesized_from_type) { + prepared.children = synthesize_complex_children_from_type(table_column.type); + } else if (!prepared.children.empty() && !table_column.type->equals(*file_type)) { + complete_required_complex_children_from_type(table_column.type, prepared.children); + } + + if (!prepared.children.empty() && nested_table_type->get_primitive_type() == TYPE_STRUCT) { + // Struct children are table fields, so the parent Struct type is authoritative for their + // nullability. ARRAY and MAP children are format-level structural wrappers and keep the + // descriptor types used by their recursive mappings. + align_struct_child_types_with_parent(table_column.type, prepared.children); + } + return prepared; +} + +static Status validate_file_schema_children(const ColumnDefinition& file_field) { + if (file_field.type == nullptr) { + return Status::InternalError("File column '{}' has null type", file_field.name); + } + const auto nested_type = remove_nullable(file_field.type); + size_t expected_children = 0; + bool complex_with_fixed_children = true; + switch (nested_type->get_primitive_type()) { + case TYPE_ARRAY: + expected_children = 1; + break; + case TYPE_MAP: + expected_children = 2; + break; + case TYPE_STRUCT: + expected_children = + assert_cast(nested_type.get())->get_elements().size(); + break; + default: + complex_with_fixed_children = false; + break; + } + if (!complex_with_fixed_children || file_field.children.size() == expected_children) { + return Status::OK(); + } + return Status::InternalError( + "Malformed complex file schema for column '{}': type={}, expected_children={}, " + "actual_children={}", + file_field.name, file_field.type->get_name(), expected_children, + file_field.children.size()); +} + +static bool has_projected_file_children(const ColumnMapping& mapping) { + if (mapping.original_file_children.empty() || mapping.projected_file_children.empty()) { + return false; + } + if (mapping.original_file_children.size() != mapping.projected_file_children.size()) { + return true; + } + for (size_t idx = 0; idx < mapping.original_file_children.size(); ++idx) { + if (mapping.original_file_children[idx].file_local_id() != + mapping.projected_file_children[idx].file_local_id()) { + return true; + } + } + return false; +} + +static bool needs_nested_file_projection(const ColumnMapping& mapping) { + if (has_projected_file_children(mapping)) { + // Return True if the projected child column is missing / re-ordered + return true; + } + return std::ranges::any_of(mapping.child_mappings, [](const ColumnMapping& child_mapping) { + return needs_nested_file_projection(child_mapping); + }); +} + +static Status build_complex_projection(const ColumnMapping& mapping, LocalColumnIndex* projection); + +// Build the projected file children/type according to the pruned complex projection. For example, +// if we have a struct column `s` with children `id` and `name`, and the projection only keeps +// `s.name`, then the file reader should expose `STRUCT`. +static Status rebuild_projected_file_children_and_type( + const DataTypePtr& file_type, const std::vector& original_file_children, + const std::vector& child_mappings, + std::vector* projected_file_children, DataTypePtr* projected_type) { + DORIS_CHECK(file_type != nullptr); + DORIS_CHECK(projected_file_children != nullptr); + DORIS_CHECK(projected_type != nullptr); + ColumnDefinition field; + field.type = file_type; + field.children = original_file_children; + LocalColumnIndex projection = LocalColumnIndex::partial_local(-1); + projection.children.reserve(child_mappings.size()); + for (const auto* child_mapping : present_child_mappings_in_file_order(child_mappings)) { + DORIS_CHECK(child_mapping->file_local_id.has_value()); + LocalColumnIndex child_projection; + RETURN_IF_ERROR(build_complex_projection(*child_mapping, &child_projection)); + projection.children.push_back(std::move(child_projection)); + } + + ColumnDefinition projected_field; + RETURN_IF_ERROR(project_column_definition(field, projection, &projected_field)); + *projected_file_children = std::move(projected_field.children); + *projected_type = std::move(projected_field.type); + return Status::OK(); +} + +// Build the complex column projection according to the ColumnMapping which is re-ordered by the +// file-schema's order. +// +// For MAP, a partial projection represents value-subtree pruning only. The key child is not a +// projected output shape; file readers still read full keys to construct ColumnMap offsets and keep +// key semantics unchanged. If a caller tries to project only/prune the key child, the common schema +// projection helper rejects it. +static Status build_complex_projection(const ColumnMapping& mapping, LocalColumnIndex* projection) { + if (projection == nullptr) { + return Status::InvalidArgument("projection is null"); + } + DORIS_CHECK(mapping.file_local_id.has_value()); + *projection = LocalColumnIndex::local(*mapping.file_local_id); + projection->project_all_children = mapping.child_mappings.empty(); + projection->children.clear(); + const auto present_children = present_child_mappings_in_file_order(mapping.child_mappings); + if (!projection->project_all_children && present_children.empty()) { + // All requested table children under this complex node are missing/default-only. The file + // reader cannot expose an empty complex projection, but TableReader can still rematerialize + // the table shape from a full file subtree and fill the missing children with defaults. + projection->project_all_children = true; + return Status::OK(); + } + for (const auto* child_mapping : present_children) { + LocalColumnIndex child_projection; + RETURN_IF_ERROR(build_complex_projection(*child_mapping, &child_projection)); + projection->children.push_back(std::move(child_projection)); + } + if (!projection->project_all_children && projection->children.empty()) { + return Status::NotSupported("Projection for complex column {} contains no file children", + mapping.file_column_name); + } + return Status::OK(); +} + +using FilterProjectionMap = std::map; + +// Update the mapping's file type according to the projection, and determine whether the projection +// is trivial (i.e. the projected file type is the same as the table type, so no need to +// rematerialize the complex value back to table layout after reading from file). +static Status apply_projection_to_mapping_file_type(const LocalColumnIndex& projection, + ColumnMapping* mapping) { + DORIS_CHECK(mapping != nullptr); + if (mapping->original_file_type == nullptr) { + mapping->original_file_type = mapping->file_type; + } + if (mapping->original_file_type == nullptr || + !is_complex_type(remove_nullable(mapping->original_file_type)->get_primitive_type())) { + return Status::OK(); + } + ColumnDefinition field; + field.type = mapping->original_file_type; + field.children = mapping->original_file_children; + ColumnDefinition projected_field; + RETURN_IF_ERROR(project_column_definition(field, projection, &projected_field)); + mapping->file_type = std::move(projected_field.type); + mapping->projected_file_children = std::move(projected_field.children); + mapping->is_trivial = mapping_can_use_file_column_directly(*mapping); + return Status::OK(); +} + +static Status merge_filter_projection(const FilterProjectionMap* filter_projections, + LocalColumnIndex* projection) { + DORIS_CHECK(projection != nullptr); + if (filter_projections == nullptr) { + return Status::OK(); + } + const auto filter_projection_it = filter_projections->find(projection->column_id()); + if (filter_projection_it == filter_projections->end()) { + return Status::OK(); + } + // Merge predicate-only nested paths into the root projection that is about to be scanned. + // Example: `SELECT s.a WHERE s.b > 1` first builds the output projection `s -> a` from + // ColumnMapping, while build_nested_struct_filter_projection_map() records `s -> b`. This merge + // produces one file scan projection `s -> a,b`. + RETURN_IF_ERROR(merge_local_column_index(projection, filter_projection_it->second)); + return Status::OK(); +} + +static bool table_root_is_map(const ColumnMapping& mapping) { + if (mapping.table_type == nullptr) { + return false; + } + return remove_nullable(mapping.table_type)->get_primitive_type() == TYPE_MAP; +} + +static Status add_scan_column(FileScanRequest* file_request, ColumnMapping* mapping, + bool is_predicate_column, bool force_full_complex_scan_projection, + const FilterProjectionMap* filter_projections = nullptr) { + const auto file_column_id = LocalColumnId(mapping->file_local_id.value()); + LocalColumnIndex projection = LocalColumnIndex::top_level(file_column_id); + // Columnar readers can turn a complex mapping into a nested file projection, but + // row-oriented readers must scan the full top-level complex field because all children are + // encoded in the same text cell. + if (!force_full_complex_scan_projection && needs_nested_file_projection(*mapping)) { + RETURN_IF_ERROR(build_complex_projection(*mapping, &projection)); + } + if (is_predicate_column && !force_full_complex_scan_projection) { + DCHECK(filter_projections != nullptr); + // If a projected complex root is also used by a predicate, rebuild the predicate scan + // projection from the output mapping before merging predicate-only children. For + // `SELECT s.a WHERE s.b > 1`, build_complex_projection() produces `s -> a` and + // merge_filter_projection() adds `s -> b`, so the predicate column reads both children. + RETURN_IF_ERROR(merge_filter_projection(filter_projections, &projection)); + } + FileScanRequestBuilder builder(file_request); + if (is_predicate_column) { + return builder.add_predicate_column(std::move(projection)); + } + return builder.add_non_predicate_column(std::move(projection)); +} + +static const LocalColumnIndex* find_scan_projection( + const std::vector& scan_columns, LocalColumnId file_column_id) { + const auto projection_it = + std::ranges::find_if(scan_columns, [&](const LocalColumnIndex& projection) { + return projection.column_id() == file_column_id; + }); + return projection_it == scan_columns.end() ? nullptr : &*projection_it; +} + +// Apply the final scan projection of one root file column back to its ColumnMapping. This updates +// mapping.file_type/projected_file_children from the original file schema to the exact shape that +// FileReader will return. +// +// Example: for `SELECT s.a WHERE s.b > 1`, add_scan_column() keeps only one predicate scan +// projection `s -> a,b`. Applying that projection changes the mapping's file type from the full +// file struct `s` to the projected file struct `s`, so later filter rewrite and +// TableReader final materialization use the same column shape as the file-local block. +static Status apply_scan_projection_to_mapping_file_type(const FileScanRequest& file_request, + ColumnMapping* mapping) { + DORIS_CHECK(mapping != nullptr); + DORIS_CHECK(mapping->file_local_id.has_value()); + const auto file_column_id = LocalColumnId(*mapping->file_local_id); + // Predicate columns are the actual scan projection when a column is used by row-level filters: + // add_scan_column() removes the duplicate non-predicate projection in that case. + const auto* projection = find_scan_projection(file_request.predicate_columns, file_column_id); + if (projection == nullptr) { + projection = find_scan_projection(file_request.non_predicate_columns, file_column_id); + } + DORIS_CHECK(projection != nullptr); + return apply_projection_to_mapping_file_type(*projection, mapping); +} + +// Build extra scan projections required only by row-level filters on nested struct children. +// +// Example: for `SELECT s.a FROM t WHERE s.b.c > 1`, the output projection may only contain `s.a`, +// but the file reader must also read `s.b.c` to evaluate the predicate. This function collects the +// table-side filter path, resolves it through ColumnMapping first, and records the corresponding +// file-side projection in filter_projections. This keeps renamed fields consistent between the scan +// projection and row-level conjunct rewrite. Example: +// table filter path: s -> renamed_b -> c +// old file path: s -> b -> c +// recorded path: s -> b -> c +// When add_scan_column() adds the same root as a predicate column, it rebuilds that root from the +// output mapping, merges this filter-only projection into it, and removes the duplicate +// non-predicate root entry. +static Status build_nested_struct_filter_projection_map( + const std::vector& table_filters, const std::vector& mappings, + FilterProjectionMap* filter_projections) { + DORIS_CHECK(filter_projections != nullptr); + filter_projections->clear(); + for (const auto& table_filter : table_filters) { + if (table_filter.conjunct == nullptr) { + continue; + } + // Collect all nested struct paths in the table filter. For example, for + // `s.id > 5 AND element_at(s, 'renamed_name') = 'abc'`, collect the table paths + // `s -> id` and `s -> renamed_name`, then resolve each one to its file-side projection. + std::vector paths; + collect_nested_struct_paths(table_filter.conjunct->root(), &paths); + for (const auto& path : paths) { + auto mapping_it = std::ranges::find_if(mappings, [&](const ColumnMapping& mapping) { + return mapping.global_index == path.root_global_index; + }); + if (mapping_it == mappings.end() || !mapping_it->file_local_id.has_value() || + path.selectors.empty()) { + continue; + } + + ResolvedNestedStructPath resolved; + LocalColumnIndex root_projection; + if (!resolve_nested_struct_path_for_file(path, mappings, &resolved)) { + if (!table_root_is_map(*mapping_it)) { + continue; + } + // Direct map value filters such as `m.value.a > 1` need the value leaf for row + // evaluation even when the query only projects another value child. This is only a + // scan projection fallback; complex map/array expressions are still not rewritten + // into file-local conjuncts. + LocalColumnIndex child_projection; + RETURN_IF_ERROR(build_file_child_projection_from_schema( + mapping_it->original_file_children, path.selectors, &child_projection)); + if (child_projection.local_id() < 0) { + continue; + } + root_projection = LocalColumnIndex::partial_local(*mapping_it->file_local_id); + root_projection.children.push_back(std::move(child_projection)); + } else { + root_projection = std::move(resolved.file_projection); + } + auto filter_projection_it = filter_projections->find(root_projection.column_id()); + if (filter_projection_it == filter_projections->end()) { + filter_projections->emplace(root_projection.column_id(), + std::move(root_projection)); + continue; + } + RETURN_IF_ERROR( + merge_local_column_index(&filter_projection_it->second, root_projection)); + } + } + return Status::OK(); +} + +static void rebuild_projection(ColumnMapping* mapping, LocalIndex block_position) { + DORIS_CHECK(mapping->file_local_id.has_value()); + if (mapping->is_trivial || needs_complex_rematerialize(*mapping)) { + mapping->projection = VExprContext::create_shared(VSlotRef::create_shared( + cast_set(block_position.value()), cast_set(block_position.value()), -1, + mapping->file_type, mapping->file_column_name)); + return; + } + + auto expr = Cast::create_shared(mapping->table_type); + expr->add_child(VSlotRef::create_shared(cast_set(block_position.value()), + cast_set(block_position.value()), -1, + mapping->file_type, mapping->file_column_name)); + mapping->projection = VExprContext::create_shared(expr); +} + +// Build file slot rewrite info from the localized filter targets. Only local targets can enter +// file-reader expressions; constant and unset targets stay above the file reader. +static std::map build_file_slot_rewrite_map( + const std::vector& mappings, + const std::map& filter_entries) { + std::map global_to_file_slot; + for (const auto& mapping : mappings) { + const auto entry_it = filter_entries.find(mapping.global_index); + if (entry_it == filter_entries.end() || !entry_it->second.is_local()) { + continue; + } + DORIS_CHECK(mapping.file_local_id.has_value()); + global_to_file_slot.emplace( + mapping.global_index, + FileSlotRewriteInfo {.block_position = entry_it->second.local_index().value(), + .file_type = mapping.file_type, + .table_type = mapping.table_type, + .file_column_name = mapping.file_column_name}); + } + return global_to_file_slot; +} + +Status TableColumnMapper::_create_by_index_mapping(const ColumnDefinition& table_column, + const std::vector& file_schema, + ColumnMapping* mapping) { + DORIS_CHECK(mapping != nullptr); + DORIS_CHECK(!table_column.is_partition_key); + + // Key contract: in BY_INDEX mode, `ColumnDefinition::identifier` TYPE_INT is interpreted as the + // 0-based position of this column inside `file_schema`. FE writes the physical file position + // of each non-partition projected column into that identifier. This interpretation allows: + // - sparse projection: read only a subset of file columns (for example only `_col2` + // and `_col4`); + // - column reordering: table column order differs from file column order; + // - no many-to-one mapping: FE must guarantee that each file position is referenced by at + // most one table column. + const auto file_index = table_column.get_identifier_position(); + + // Case A: file_index is in range, so build a direct positional mapping. + // The file column name (for example `_col0`) is intentionally ignored here. + if (file_index >= 0 && static_cast(file_index) < file_schema.size()) { + return _create_direct_mapping(table_column, file_schema[static_cast(file_index)], + mapping); + } + + // Case B: file_index is out of range, which means the file does not contain this column. + // Route it through the missing-column path used by schema evolution. + if (table_column.default_expr != nullptr) { + _set_constant_mapping(mapping, table_column.default_expr); + return Status::OK(); + } + // Keep the mapping empty (`file_local_id` remains `nullopt`) and let the upper finalize + // stage fill NULL/default values. + return Status::OK(); +} + +void TableColumnMapper::_set_constant_mapping(ColumnMapping* mapping, VExprContextSPtr expr) { + DORIS_CHECK(mapping != nullptr); + DORIS_CHECK(expr != nullptr); + mapping->default_expr = std::move(expr); + mapping->constant_index = _constant_map.add(ConstantEntry { + .global_index = mapping->global_index, + .expr = mapping->default_expr, + .type = mapping->table_type, + }); + mapping->filter_conversion = FilterConversionType::CONSTANT; +} + +Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& table_column, + GlobalIndex global_index, + ColumnMapping* mapping) { + DORIS_CHECK(mapping != nullptr); + *mapping = ColumnMapping {}; + mapping->global_index = global_index; + mapping->table_column_name = table_column.name; + mapping->table_type = table_column.type; + const auto row_lineage_type = row_lineage_virtual_column_type(table_column, _options.mode); + if (const auto* partition_value = find_partition_value(table_column, _partition_values); + table_column.is_partition_key && partition_value != nullptr) { + // Partition values are split constants and must take precedence over defaults. + _set_constant_mapping(mapping, VExprContext::create_shared(VLiteral::create_shared( + mapping->table_type, *partition_value))); + } else if (_options.mode == TableColumnMappingMode::BY_INDEX && + !table_column.is_partition_key && table_column.has_identifier_field_id()) { + // BY_INDEX interprets ColumnDefinition::identifier as physical file position. + RETURN_IF_ERROR(_create_by_index_mapping(table_column, _file_schema, mapping)); + } else if (const auto* file_field = _find_file_field(table_column, _file_schema)) { + // Normal physical file column mapping. + RETURN_IF_ERROR(_create_direct_mapping(table_column, *file_field, mapping)); + if (row_lineage_type != TableVirtualColumnType::INVALID) { + // Iceberg v3 rewritten files may physically contain row lineage metadata fields. + // File non-null values must be preserved, while file NULLs still inherit from data file + // metadata in IcebergTableReader. Therefore the mapping has a real file source plus a + // virtual post-materialization step, and filters must wait for finalize output. + mapping->virtual_column_type = row_lineage_type; + mapping->filter_conversion = FilterConversionType::FINALIZE_ONLY; + } + } else if (row_lineage_type != TableVirtualColumnType::INVALID) { + // Iceberg row lineage metadata fields are optional in data files. Missing fields are exposed + // as all-NULL table columns first; IcebergTableReader fills inherited values only when the + // split carries first_row_id / last_updated_sequence_number metadata. + // FE may attach a default_expr to these hidden metadata columns, but the Iceberg v3 + // inheritance rule must take precedence over the generic missing-column default path. + mapping->virtual_column_type = row_lineage_type; + } else if (table_column.name == BeConsts::ICEBERG_ROWID_COL) { + // Doris internal Iceberg row locator is never a physical Iceberg data column. It is built + // from file path, row position and partition metadata for delete/update/merge. + mapping->virtual_column_type = TableVirtualColumnType::ICEBERG_ROWID; + } else if (table_column.default_expr != nullptr) { + // Missing schema-evolution column with an explicit default expression. + _set_constant_mapping(mapping, table_column.default_expr); + } else { + if (table_column.is_partition_key) { + return Status::InvalidArgument( + "Table column '{}' (global_index={}) does not have a matching partition value", + table_column.name, mapping->global_index.value()); + } + } + return Status::OK(); +} + +Status TableColumnMapper::_create_hidden_filter_mapping(const ColumnDefinition& table_column, + GlobalIndex global_index, + ColumnMapping* mapping) { + auto status = _create_mapping_for_column(table_column, global_index, mapping); + if (mapping->file_local_id.has_value() || mapping->constant_index.has_value() || + mapping->virtual_column_type != TableVirtualColumnType::INVALID) { + return Status::OK(); + } + if (_options.mode == TableColumnMappingMode::BY_NAME) { + return status; + } + + // Predicate-only slot refs carry the table name/type but do not carry the table-format field + // id used by BY_FIELD_ID or the file position used by BY_INDEX. Use a name fallback only for + // hidden filter localization; projected columns still obey the requested mapping mode. + const auto* file_field = + matcher_for_mode(TableColumnMappingMode::BY_NAME).find(table_column, _file_schema); + if (file_field == nullptr) { + return status; + } + ColumnMapping fallback_mapping; + fallback_mapping.global_index = global_index; + fallback_mapping.table_column_name = table_column.name; + fallback_mapping.table_type = table_column.type; + RETURN_IF_ERROR(_create_direct_mapping(table_column, *file_field, &fallback_mapping)); + *mapping = std::move(fallback_mapping); + return Status::OK(); +} + +Status TableColumnMapper::_build_hidden_filter_mappings( + const std::vector& table_filters) { + _hidden_mappings.clear(); + + std::map filter_columns; + for (const auto& table_filter : table_filters) { + if (table_filter.conjunct != nullptr) { + collect_top_level_slot_columns(table_filter.conjunct->root(), &filter_columns); + } + } + + for (const auto& [global_index, table_column] : filter_columns) { + if (_find_mapping(global_index) != nullptr) { + // Ignore columns that are already mapped by the projected columns + continue; + } + ColumnMapping mapping; + RETURN_IF_ERROR(_create_hidden_filter_mapping(table_column, global_index, &mapping)); + if (mapping.file_local_id.has_value() || mapping.constant_index.has_value() || + mapping.virtual_column_type != TableVirtualColumnType::INVALID) { + _hidden_mappings.push_back(std::move(mapping)); + } + } + return Status::OK(); +} + +Status TableColumnMapper::create_mapping(const std::vector& projected_columns, + const std::map& partition_values, + const std::vector& file_schema) { + clear(); + _partition_values = partition_values; + _file_schema = file_schema; + for (size_t column_idx = 0; column_idx < projected_columns.size(); ++column_idx) { + ColumnMapping mapping; + RETURN_IF_ERROR(_create_mapping_for_column(projected_columns[column_idx], + GlobalIndex(column_idx), &mapping)); + _mappings.push_back(std::move(mapping)); + } + return Status::OK(); +} + +std::vector TableColumnMapper::_filter_visible_mappings() const { + std::vector mappings; + mappings.reserve(_mappings.size() + _hidden_mappings.size()); + mappings.insert(mappings.end(), _mappings.begin(), _mappings.end()); + mappings.insert(mappings.end(), _hidden_mappings.begin(), _hidden_mappings.end()); + return mappings; +} + +Status TableColumnMapper::_build_filter_entries(const FileScanRequest& file_request) { + _filter_entries.clear(); + const auto mappings = _filter_visible_mappings(); + for (const auto& mapping : mappings) { + FilterEntry entry; + if (mapping.constant_index.has_value()) { + entry = FilterEntry::constant(*mapping.constant_index); + } else if (mapping.file_local_id.has_value() && + filter_conversion_has_local_source(mapping.filter_conversion)) { + const auto local_position_it = + file_request.local_positions.find(LocalColumnId(*mapping.file_local_id)); + if (local_position_it != file_request.local_positions.end()) { + entry = FilterEntry::local(local_position_it->second); + } + } + _filter_entries.emplace(mapping.global_index, entry); + } + return Status::OK(); +} + +Status TableColumnMapper::create_scan_request( + const std::vector& table_filters, + const std::vector& projected_columns, FileScanRequest* file_request, + RuntimeState* runtime_state) { + // FileReader evaluates expressions against a file-local block. This mapper owns the + // table-column to file-column conversion, so it also owns the file-local block positions. + file_request->predicate_columns.clear(); + file_request->non_predicate_columns.clear(); + file_request->local_positions.clear(); + file_request->conjuncts.clear(); + file_request->delete_conjuncts.clear(); + _filter_entries.clear(); + // 1. Build referenced non-predicate columns + for (size_t column_idx = 0; column_idx < projected_columns.size(); ++column_idx) { + const auto global_index = GlobalIndex(column_idx); + auto* mapping = _find_mapping(global_index); + if (mapping != nullptr && mapping->file_local_id.has_value()) { + // A file column can be read lazily as a non-predicate column only when it is not used + // by row-level expression filters. + bool used_by_filter = false; + for (const auto& table_filter : table_filters) { + const auto& global_indices = table_filter.global_indices; + if (std::find(global_indices.begin(), global_indices.end(), global_index) != + global_indices.end() && + filter_conversion_has_local_source(mapping->filter_conversion)) { + used_by_filter = true; + break; + } + } + if (!used_by_filter || !enable_lazy_materialization()) { + RETURN_IF_ERROR(add_scan_column(file_request, mapping, false, + force_full_complex_scan_projection())); + } + } + } + // 2. Build referenced predicate columns + // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. + RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); + RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state)); + // 3. Rebuild output projection expressions for projected columns. localize_filters() has + // already applied the final scan projection to mapping.file_type/projected_file_children before + // rewriting filter expressions. + for (auto& mapping : _mappings) { + if (!mapping.file_local_id.has_value()) { + continue; + } + auto position_it = + file_request->local_positions.find(LocalColumnId(*mapping.file_local_id)); + DORIS_CHECK(position_it != file_request->local_positions.end()) + << file_request->local_positions.size() << " " << *mapping.file_local_id << " " + << mapping.file_column_name; + rebuild_projection(&mapping, position_it->second); + } + return Status::OK(); +} + +ColumnMapping* TableColumnMapper::_find_mapping(GlobalIndex global_index) { + for (auto& mapping : _mappings) { + if (mapping.global_index == global_index) { + return &mapping; + } + } + return nullptr; +} + +ColumnMapping* TableColumnMapper::_find_filter_mapping(GlobalIndex global_index) { + if (auto* mapping = _find_mapping(global_index); mapping != nullptr) { + return mapping; + } + for (auto& mapping : _hidden_mappings) { + if (mapping.global_index == global_index) { + return &mapping; + } + } + return nullptr; +} + +Status TableColumnMapper::localize_filters(const std::vector& table_filters, + FileScanRequest* file_request, + RuntimeState* runtime_state) { + FilterProjectionMap filter_projections; + auto filter_mappings = _filter_visible_mappings(); + RETURN_IF_ERROR(build_nested_struct_filter_projection_map(table_filters, filter_mappings, + &filter_projections)); + for (const auto& table_filter : table_filters) { + for (const auto& global_index : table_filter.global_indices) { + auto* mapping = _find_filter_mapping(global_index); + if (mapping == nullptr || !mapping->file_local_id.has_value() || + !filter_conversion_has_local_source(mapping->filter_conversion)) { + continue; + } + RETURN_IF_ERROR(add_scan_column(file_request, mapping, enable_lazy_materialization(), + force_full_complex_scan_projection(), + &filter_projections)); + } + } + // Rebuild the file type for every scan-local mapping before expression rewrite. Predicate-only + // hidden mappings must see the same projected file type as the file reader will produce. + for (auto& mapping : _mappings) { + if (mapping.file_local_id.has_value() && + file_request->local_positions.contains(LocalColumnId(*mapping.file_local_id))) { + RETURN_IF_ERROR(apply_scan_projection_to_mapping_file_type(*file_request, &mapping)); + } + } + for (auto& mapping : _hidden_mappings) { + if (mapping.file_local_id.has_value() && + file_request->local_positions.contains(LocalColumnId(*mapping.file_local_id))) { + RETURN_IF_ERROR(apply_scan_projection_to_mapping_file_type(*file_request, &mapping)); + } + } + RETURN_IF_ERROR(_build_filter_entries(*file_request)); + + // Build the complete table-slot rewrite map after all predicate columns have been assigned. + // This keeps expression localization independent from filter iteration order. + filter_mappings = _filter_visible_mappings(); + const auto global_to_file_slot = build_file_slot_rewrite_map(filter_mappings, _filter_entries); + for (const auto& table_filter : table_filters) { + if (table_filter.conjunct != nullptr && + table_filter_has_only_local_entries(table_filter, _filter_entries)) { + RewriteContext rewrite_context {.runtime_state = runtime_state}; + VExprSPtr rewrite_root; + Status clone_status; + try { + clone_status = clone_table_expr_tree(table_filter.conjunct->root(), &rewrite_root); + } catch (const Exception& e) { + // Some table filters contain complex intermediate values, for example + // `element_at(MAP_VALUES(m)[1], 'age') > 30`. The current file-local rewrite only + // understands top-level slots and struct-element paths rooted at top-level slots; + // cloning such expressions can hit the generic TExpr complex-type limitation. + // Leave them above TableReader, where Scanner evaluates the original table-level + // conjunct after final materialization. +#ifndef NDEBUG + return Status::InternalError( + "Failed to clone table filter for file-local rewrite: {}, expr={}", + e.to_string(), table_filter.conjunct->root()->debug_string()); +#else + continue; +#endif + } catch (const std::exception& e) { +#ifndef NDEBUG + return Status::InternalError( + "Failed to clone table filter for file-local rewrite: {}, expr={}", + e.what(), table_filter.conjunct->root()->debug_string()); +#else + continue; +#endif + } + if (!clone_status.ok()) { +#ifndef NDEBUG + return Status::InternalError( + "Failed to clone table filter for file-local rewrite: {}, expr={}", + clone_status.to_string(), table_filter.conjunct->root()->debug_string()); +#else + continue; +#endif + } + bool can_localize = true; + auto localized_root = rewrite_table_expr_to_file_expr(rewrite_root, global_to_file_slot, + filter_mappings, &rewrite_context, + &can_localize); + if (!can_localize) { + continue; + } + auto localized_conjunct = VExprContext::create_shared(std::move(localized_root)); + RETURN_IF_ERROR(rewrite_context.prepare_created_exprs(localized_conjunct.get())); + file_request->conjuncts.push_back(std::move(localized_conjunct)); + } + } + return Status::OK(); +} + +const ColumnDefinition* TableColumnMapper::_find_file_field( + const ColumnDefinition& table_column, + const std::vector& file_schema) const { + if (table_column.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) { + const auto field_it = std::ranges::find_if(file_schema, [](const ColumnDefinition& field) { + return field.column_type == ColumnType::GLOBAL_ROWID; + }); + return field_it == file_schema.end() ? nullptr : &*field_it; + } + return matcher_for_mode(_options.mode).find(table_column, file_schema); +} + +Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_column, + const ColumnDefinition& file_field, + ColumnMapping* mapping) const { + DORIS_CHECK(mapping != nullptr); + DORIS_CHECK(file_field.local_id >= 0 || file_field.local_id == GLOBAL_ROWID_COLUMN_ID); + mapping->file_local_id = file_field.local_id; + mapping->table_column_name = table_column.name; + mapping->file_column_name = file_field.name; + mapping->original_file_type = file_field.type; + mapping->original_file_children = file_field.children; + mapping->projected_file_children = file_field.children; + mapping->file_type = file_field.type; + mapping->is_trivial = mapping_can_use_file_column_directly(*mapping); + mapping->filter_conversion = direct_filter_conversion(*mapping); + mapping->child_mappings.clear(); + + auto [table_children, synthesized_table_children] = + prepare_table_children_for_mapping(table_column, mapping->file_type); + + if (!table_children.empty()) { + if (!is_complex_type(remove_nullable(mapping->file_type)->get_primitive_type())) { + return Status::NotSupported( + "Cannot map complex table column '{}' to scalar parquet column '{}', table " + "type={}, file type={}", + table_column.name, file_field.name, mapping->table_type->get_name(), + mapping->file_type->get_name()); + } + RETURN_IF_ERROR(validate_file_schema_children(file_field)); + std::vector synthesized_used_file_child_ids; + for (size_t table_child_idx = 0; table_child_idx < table_children.size(); + ++table_child_idx) { + const auto& table_child = table_children[table_child_idx]; + const auto* file_child = + find_file_child_for_mapping(table_child, file_field, _options.mode, + table_child_idx, synthesized_table_children); + if (synthesized_table_children && file_child != nullptr) { + const auto file_child_id = file_child->file_local_id(); + if (std::ranges::find(synthesized_used_file_child_ids, file_child_id) != + synthesized_used_file_child_ids.end()) { + file_child = nullptr; + for (const auto& candidate : file_field.children) { + const auto candidate_id = candidate.file_local_id(); + if (std::ranges::find(synthesized_used_file_child_ids, candidate_id) == + synthesized_used_file_child_ids.end()) { + file_child = &candidate; + break; + } + } + } + if (file_child != nullptr) { + synthesized_used_file_child_ids.push_back(file_child->file_local_id()); + } + } + if (file_child == nullptr) { + ColumnMapping child_mapping; + child_mapping.table_column_name = table_child.name; + child_mapping.file_column_name = table_child.name; + child_mapping.table_type = table_child.type; + child_mapping.file_type = table_child.type; + child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; + mapping->child_mappings.push_back(std::move(child_mapping)); + continue; + } + ColumnMapping child_mapping; + child_mapping.table_column_name = table_child.name; + child_mapping.table_type = table_child.type; + RETURN_IF_ERROR(_create_direct_mapping(table_child, *file_child, &child_mapping)); + mapping->child_mappings.push_back(std::move(child_mapping)); + } + if (needs_projected_file_type_rebuild(*mapping)) { + // If complex projection prunes some children, we have to rebuild the projected file type to make sure the reader expression can find the correct child types by name. + RETURN_IF_ERROR(rebuild_projected_file_children_and_type( + mapping->file_type, mapping->original_file_children, mapping->child_mappings, + &mapping->projected_file_children, &mapping->file_type)); + DCHECK(mapping->table_type != nullptr); + mapping->is_trivial = mapping_can_use_file_column_directly(*mapping); + mapping->filter_conversion = projected_filter_conversion(*mapping); + } + } + return Status::OK(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h new file mode 100644 index 00000000000000..4f417a680aad60 --- /dev/null +++ b/be/src/format_v2/column_mapper.h @@ -0,0 +1,284 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/field.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/file_reader.h" + +namespace doris { +class RuntimeState; +} // namespace doris + +namespace doris::format { + +struct ColumnDefinition; +struct TableFilter; + +enum class TableColumnMappingMode { + // Match by ColumnDefinition::identifier TYPE_INT as field id. + BY_FIELD_ID, + // Match by ColumnDefinition::identifier TYPE_STRING, or logical name when identifier is null. + BY_NAME, + // Match top-level columns by file position. This mainly serves Hive1 ORC style files whose + // column names are placeholder values such as `_col0` / `_col1`, where position is the only + // reliable way to select the correct column. + BY_INDEX, +}; + +enum TableVirtualColumnType { + INVALID = 0, // not a virtual column + // Iceberg v3 row lineage metadata column `_row_id`. Physical non-null values + // are preserved; NULL or missing values inherit first_row_id + row_position. + ROW_ID = 1, + // Iceberg v3 row lineage metadata column `_last_updated_sequence_number`. + // Physical non-null values are preserved; NULL or missing values inherit the + // data file's last_updated_sequence_number. + LAST_UPDATED_SEQUENCE_NUMBER = 2, + // Doris internal Iceberg row locator column `__DORIS_ICEBERG_ROWID_COL__`. + // It is a struct used by delete/update/merge, not the Iceberg `_row_id`. + ICEBERG_ROWID = 3, +}; + +enum class FilterConversionType { + COPY_DIRECTLY, // filter can be copied directly from file layer without any change, e.g. column type and table type are the same and no complex nested projection is involved. + CAST_FILTER, // filter can be converted from file layer by adding a cast, e.g. column type is nullable but table type is not, or file column has a trivial nested projection but table column has a complex nested projection. + READER_EXPRESSION, + FINALIZE_ONLY, // filter cannot be converted to file layer and should be evaluated at table reader finalize phase, e.g. predicates on ICEBERG_ROW_ID column which is generated by IcebergReader. + CONSTANT, +}; + +// Nested global-to-local child mapping. The root index points either to a request-local slot or to +// a child id, depending on the owner. child_mapping keeps the recursive table-child to file-child +// relationship explicit instead of encoding it in ColumnMapping flags. +struct IndexMapping { + int32_t index = -1; + std::map> child_mapping; +}; + +// Recursive result produced after one table/global column is assigned to a file-local source. +struct ColumnMapResult { + std::optional local_column_id; + std::optional column_index; + std::optional mapping; +}; + +// Final mapping entry from one global result column to one file-local source. +struct ColumnMapEntry { + IndexMapping mapping; + DataTypePtr local_type; + DataTypePtr global_type; + FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; +}; + +// Collection of final result-column mappings produced for one file/split. +struct ResultColumnMapping { + std::map global_to_local; +}; + +// Mapping result from one table column to one file column. +// This is the main boundary object between table-level schema semantics and file-local schema +// semantics. +struct ColumnMapping { + // Position of the top-level projected column in the table/global output block. Table-level + // filters and column predicates refer to this index after FileScannerV2 translates FE ids at + // the scanner boundary. + GlobalIndex global_index; + std::string table_column_name; + // File-reader local id for the mapped node. + // + // For a root mapping it is convertible to LocalColumnId. For a nested mapping it is the + // LocalColumnIndex child id under the parent projection. This is deliberately separated from + // ColumnDefinition::identifier, which is the table-to-file matching key such as Parquet/Iceberg + // field_id or column name. + // + // Empty means the table column is constant, missing, partition-only, or virtual. + std::optional file_local_id; + std::string file_column_name; + // Full file type/children before nested projection pruning. Used to rebuild projected types + // and to localize nested filters that reference children not present in the output projection. + DataTypePtr original_file_type; + std::vector original_file_children; + // File children after applying the scan projection. The order follows the file-local semantic + // schema, not table child order. TableReader uses this to map table-output children back to the + // file-local block layout when projection, predicate-only children, and schema evolution mix. + std::vector projected_file_children; + // Split/file-local constant entry when this mapping is produced from partition/default/virtual + // expression instead of physical file data. + std::optional constant_index; + // Effective file type after applying casts/remaps/nested projection pruning. + DataTypePtr file_type; + // Target table/global type after final materialization. + DataTypePtr table_type; + + // Final projection expression used to convert file-local values into table/global values, such + // as casts, defaults, partition values, generated columns, or complex-column remaps. + VExprContextSPtr projection; + + // Mapping tree for nested table children. The order follows table output children, while file + // children can be pruned/reordered through each child mapping's file-reader local id. + std::vector child_mappings; + // True when file value can be used directly as table value without cast or child remap. + bool is_trivial = false; + // How filters referencing this table/global column can be converted below table-reader + // finalize. This is metadata for localize_filters() and future constant-filter evaluation. + FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; + TableVirtualColumnType virtual_column_type = TableVirtualColumnType::INVALID; + VExprContextSPtr default_expr; + + std::string debug_string() const; +}; + +struct TableColumnMapperOptions { + TableColumnMappingMode mode = TableColumnMappingMode::BY_FIELD_ID; + + std::string debug_string() const; +}; + +Status clone_table_expr_tree(const VExprSPtr& expr, VExprSPtr* cloned_expr); +const Field* find_partition_value(const ColumnDefinition& table_column, + const std::map& partition_values); +// Apply the same case-insensitive logical name, string identifier, and bidirectional alias rules +// used by TableColumnMapper's BY_NAME mode. +const ColumnDefinition* find_column_by_name(const ColumnDefinition& table_column, + const std::vector& file_schema); + +// Generic mapping layer from table schema to file schema. +// Iceberg uses BY_FIELD_ID. Plain by-name formats can reuse this component as well, so keep this +// abstraction table-format neutral instead of making it Iceberg-only. +class TableColumnMapper { +public: + explicit TableColumnMapper(TableColumnMapperOptions options = {}) + : _options(std::move(options)) {} + virtual ~TableColumnMapper() = default; + + // Build column mappings from table schema to file schema. + // The resulting ColumnMapping describes how each table column is produced from a file column, + // a constant, or an expression. Later projection, filter localization, and table-block + // finalization should all reuse the same mapping. + virtual Status create_mapping(const std::vector& projected_columns, + const std::map& partition_values, + const std::vector& file_schema); + + // Convert a table-level scan request into a file-local scan request. table_filters preserve + // row-level filtering semantics and are rewritten as file-local conjuncts. File-layer pruning + // such as ZoneMap, dictionary, and bloom filter derives from those localized VExpr conjuncts. + virtual Status create_scan_request(const std::vector& table_filters, + const std::vector& projected_columns, + FileScanRequest* file_request, + RuntimeState* runtime_state = nullptr); + + // Localize table-level filters to the file schema. + // Trivial mappings can copy structured predicates directly. Type changes may be localized with + // a safe cast. Expressions that cannot be pushed down safely should be handled by the + // table-level finalize/filter fallback. + virtual Status localize_filters(const std::vector& table_filters, + FileScanRequest* file_request, + RuntimeState* runtime_state = nullptr); + void clear() { + _mappings.clear(); + _hidden_mappings.clear(); + _constant_map.clear(); + _filter_entries.clear(); + _file_schema.clear(); + _partition_values.clear(); + } + const std::vector& mappings() const { return _mappings; } + const std::map& filter_entries() const { return _filter_entries; } + const ConstantMap& constant_map() const { return _constant_map; } + std::string debug_string() const; + +protected: + // Columnar readers such as Parquet can read predicate columns first, evaluate row filters, and + // lazily read the rest. Row-oriented readers such as CSV/Text materialize one row at a time and + // should keep all required columns in one scan list. + virtual bool enable_lazy_materialization() const { return true; } + // Row-oriented readers such as CSV/Text cannot physically read only a nested child from one + // delimited text field. They must scan the whole complex top-level field and let TableReader + // rematerialize the requested table child after row-level filters have run. + virtual bool force_full_complex_scan_projection() const { return false; } + + const ColumnDefinition* _find_file_field( + const ColumnDefinition& table_column, + const std::vector& file_schema) const; + Status _create_direct_mapping(const ColumnDefinition& table_column, + const ColumnDefinition& file_field, ColumnMapping* mapping) const; + + Status _create_by_index_mapping(const ColumnDefinition& table_column, + const std::vector& file_schema, + ColumnMapping* mapping); + Status _build_filter_entries(const FileScanRequest& file_request); + Status _build_result_column_mapping(const FileScanRequest& file_request); + + void _set_constant_mapping(ColumnMapping* mapping, VExprContextSPtr expr); + Status _create_mapping_for_column(const ColumnDefinition& table_column, + GlobalIndex global_index, ColumnMapping* mapping); + Status _create_hidden_filter_mapping(const ColumnDefinition& table_column, + GlobalIndex global_index, ColumnMapping* mapping); + Status _build_hidden_filter_mappings(const std::vector& table_filters); + std::vector _filter_visible_mappings() const; + + ColumnMapping* _find_mapping(GlobalIndex global_index); + ColumnMapping* _find_filter_mapping(GlobalIndex global_index); + + TableColumnMapperOptions _options; + // Column mapping for each projected column, in the same order as projected_columns. Each entry + // describes how to get one table/global column from file-local sources, and carries metadata + // for filter localization and result finalize. + std::vector _mappings; + // Predicate-only top-level columns are not output projection columns, so keep their mappings + // here. They are visible only to filter localization and file-reader predicate construction. + std::vector _hidden_mappings; + std::map _filter_entries; + ConstantMap _constant_map; + // Split-local schema state retained from create_mapping() so create_scan_request() can build + // hidden mappings for top-level filter slots that are absent from projected_columns. + std::vector _file_schema; + std::map _partition_values; +}; + +// Parquet consumes the full FileScanRequest shape: predicate columns for lazy materialization and +// file-local conjuncts for ZoneMap, dictionary, and bloom-filter pruning. +class ParquetColumnMapper final : public TableColumnMapper { +public: + using TableColumnMapper::TableColumnMapper; +}; + +// Mapper for readers that always materialize every required file column before filtering. The +// table-to-file schema mapping is still generic, but the FileScanRequest layout is simpler: +// predicate_columns are not populated. +class MaterializedColumnMapper final : public TableColumnMapper { +public: + using TableColumnMapper::TableColumnMapper; + +protected: + bool enable_lazy_materialization() const override { return false; } + bool force_full_complex_scan_projection() const override { return true; } +}; + +} // namespace doris::format diff --git a/be/src/format_v2/column_mapper_nested.cpp b/be/src/format_v2/column_mapper_nested.cpp new file mode 100644 index 00000000000000..b172c6ecc874d7 --- /dev/null +++ b/be/src/format_v2/column_mapper_nested.cpp @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/column_mapper_nested.h" + +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/primitive_type.h" +#include "exprs/vexpr.h" +#include "format_v2/expr/cast.h" +#include "gen_cpp/Exprs_types.h" + +namespace doris::format { + +namespace { + +static bool is_cast_expr(const VExprSPtr& expr) { + return dynamic_cast(expr.get()) != nullptr; +} + +static bool is_signed_integer_type(PrimitiveType type) { + switch (type) { + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_LARGEINT: + return true; + default: + return false; + } +} + +static int primitive_integer_width(PrimitiveType type) { + switch (type) { + case TYPE_TINYINT: + return 1; + case TYPE_SMALLINT: + return 2; + case TYPE_INT: + return 4; + case TYPE_BIGINT: + return 8; + case TYPE_LARGEINT: + return 16; + default: + return 0; + } +} + +static bool is_decimal_type(PrimitiveType type) { + switch (type) { + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMALV2: + case TYPE_DECIMAL128I: + case TYPE_DECIMAL256: + return true; + default: + return false; + } +} + +static bool is_order_preserving_safe_cast(const DataTypePtr& from_type, + const DataTypePtr& to_type) { + if (from_type == nullptr || to_type == nullptr) { + return false; + } + const auto from_nested_type = remove_nullable(from_type); + const auto to_nested_type = remove_nullable(to_type); + if (from_nested_type->equals(*to_nested_type)) { + return true; + } + + const auto from_primitive_type = from_nested_type->get_primitive_type(); + const auto to_primitive_type = to_nested_type->get_primitive_type(); + if (is_signed_integer_type(from_primitive_type) && is_signed_integer_type(to_primitive_type)) { + return primitive_integer_width(to_primitive_type) >= + primitive_integer_width(from_primitive_type); + } + if (from_primitive_type == TYPE_FLOAT && to_primitive_type == TYPE_DOUBLE) { + return true; + } + if (is_decimal_type(from_primitive_type) && is_decimal_type(to_primitive_type)) { + return from_nested_type->get_scale() == to_nested_type->get_scale() && + to_nested_type->get_precision() >= from_nested_type->get_precision(); + } + return false; +} + +static bool parse_struct_child_selector(const VExprSPtr& expr, StructChildSelector* selector) { + DORIS_CHECK(selector != nullptr); + if (expr == nullptr || !expr->is_literal()) { + return false; + } + const Field field = literal_field(expr); + switch (field.get_type()) { + case TYPE_STRING: + case TYPE_CHAR: + case TYPE_VARCHAR: + selector->by_name = true; + selector->name = std::string(field.as_string_view()); + return true; + case TYPE_BOOLEAN: + selector->by_name = false; + selector->ordinal = field.get() ? 1 : 0; + return selector->ordinal > 0; + case TYPE_TINYINT: + selector->by_name = false; + if (field.get() <= 0) { + return false; + } + selector->ordinal = cast_set(field.get()); + return true; + case TYPE_SMALLINT: + selector->by_name = false; + if (field.get() <= 0) { + return false; + } + selector->ordinal = cast_set(field.get()); + return true; + case TYPE_INT: + selector->by_name = false; + if (field.get() <= 0) { + return false; + } + selector->ordinal = cast_set(field.get()); + return true; + case TYPE_BIGINT: + selector->by_name = false; + if (field.get() <= 0) { + return false; + } + selector->ordinal = cast_set(field.get()); + return true; + default: + return false; + } +} + +static bool extract_nested_struct_path(const VExprSPtr& expr, NestedStructPath* path) { + DORIS_CHECK(path != nullptr); + if (!is_struct_element_expr(expr)) { + return false; + } + + // Process for element_at(struct, 'field') or element_at(struct, 1) expression. + StructChildSelector selector; + if (!parse_struct_child_selector(expr->children()[1], &selector)) { + return false; + } + + const auto& parent = expr->children()[0]; + if (parent->is_slot_ref()) { + const auto* slot_ref = assert_cast(parent.get()); + path->root_global_index = slot_ref_global_index(*slot_ref); + path->selectors.clear(); + path->selectors.push_back(std::move(selector)); + return true; + } + + // Process for element_at(element_at(struct, 'field'), 'field') or + // element_at(element_at(struct, 1), 1) expression. + if (!extract_nested_struct_path(parent, path)) { + return false; + } + path->selectors.push_back(std::move(selector)); + return true; +} + +static bool extract_nested_struct_path_for_pruning(const VExprSPtr& expr, NestedStructPath* path) { + DORIS_CHECK(path != nullptr); + // Simple `ELEMENT_AT` + if (extract_nested_struct_path(expr, path)) { + return true; + } + + // `ELEMENT_AT` with `CAST` + if (!is_cast_expr(expr) || expr->get_num_children() != 1) { + return false; + } + const auto& child = expr->children()[0]; + if (!is_order_preserving_safe_cast(child->data_type(), expr->data_type())) { + return false; + } + // A safe widening cast is null-preserving and keeps the comparison ordering of the nested + // primitive leaf, so file-layer pruning can target the original leaf statistics. The row-level + // filter still evaluates the original cast expression after read. + return extract_nested_struct_path_for_pruning(child, path); +} + +static const ColumnDefinition* resolve_file_child(const std::vector& children, + const StructChildSelector& selector) { + if (selector.by_name) { + const auto child_it = std::ranges::find_if(children, [&](const ColumnDefinition& child) { + return child.name == selector.name; + }); + return child_it == children.end() ? nullptr : &*child_it; + } + if (selector.ordinal == 0 || selector.ordinal > children.size()) { + return nullptr; + } + return &children[selector.ordinal - 1]; +} + +static const DataTypeStruct* struct_type_or_null(const DataTypePtr& type) { + if (type == nullptr) { + return nullptr; + } + const auto nested_type = remove_nullable(type); + if (nested_type->get_primitive_type() != TYPE_STRUCT) { + return nullptr; + } + return assert_cast(nested_type.get()); +} + +static std::optional struct_child_index(const ColumnMapping& mapping, + const StructChildSelector& selector) { + const auto* struct_type = struct_type_or_null(mapping.table_type); + if (struct_type == nullptr) { + return std::nullopt; + } + if (selector.by_name) { + const auto position = struct_type->try_get_position_by_name(selector.name); + if (!position.has_value()) { + return std::nullopt; + } + return cast_set(*position); + } + if (selector.ordinal == 0 || selector.ordinal > struct_type->get_elements().size()) { + return std::nullopt; + } + return cast_set(selector.ordinal - 1); +} + +// Get the global child index for a child mapping. If the mapping's table type is struct, resolve +// the child index by the child mapping's table column name; otherwise, use the fallback child index. +static int32_t child_mapping_global_index(const ColumnMapping& mapping, + const ColumnMapping& child_mapping, + size_t fallback_child_idx) { + const auto* struct_type = struct_type_or_null(mapping.table_type); + if (struct_type == nullptr) { + return cast_set(fallback_child_idx); + } + const auto position = struct_type->try_get_position_by_name(child_mapping.table_column_name); + DORIS_CHECK(position.has_value()) << "Cannot find child '" << child_mapping.table_column_name + << "' in table type " << mapping.table_type->get_name(); + return cast_set(*position); +} + +static const ColumnMapping* resolve_mapped_child(const ColumnMapping& mapping, + int32_t global_child_index) { + for (size_t child_idx = 0; child_idx < mapping.child_mappings.size(); ++child_idx) { + const auto& child_mapping = mapping.child_mappings[child_idx]; + if (child_mapping_global_index(mapping, child_mapping, child_idx) == global_child_index) { + return &child_mapping; + } + } + return nullptr; +} + +enum class NestedProjectionResolveResult { + RESOLVED, + NOT_REPRESENTED, + MISSING_FILE_CHILD, +}; + +// Resolve a table-side nested struct path through the existing ColumnMapping tree and build the +// corresponding file-local projection. For example, if table column `s` has children +// `{a, renamed_b}` and file column `s` has children `{a, b}`, the filter path +// `struct_element(s, 'renamed_b')` is resolved to the file projection `s -> b` by following the +// child mapping instead of matching the table child name against the file schema. Return +// MISSING_FILE_CHILD when ColumnMapping explicitly says a table child is absent from this file; in +// that case callers must not fall back to schema-name lookup, because Iceberg can drop a field and +// later add a different field with the same name. +static NestedProjectionResolveResult resolve_nested_projection_with_mapping( + const NestedStructPath& path, const std::vector& mappings, + LocalColumnIndex* root_projection) { + DORIS_CHECK(root_projection != nullptr); + *root_projection = {}; + if (path.selectors.empty()) { + return NestedProjectionResolveResult::NOT_REPRESENTED; + } + const auto mapping_it = std::ranges::find_if(mappings, [&](const ColumnMapping& mapping) { + return mapping.global_index == path.root_global_index; + }); + if (mapping_it == mappings.end() || !mapping_it->file_local_id.has_value()) { + return NestedProjectionResolveResult::NOT_REPRESENTED; + } + + *root_projection = LocalColumnIndex::partial_local(*mapping_it->file_local_id); + auto* current_projection = root_projection; + const auto* current_mapping = &*mapping_it; + + // Traverse the ColumnMapping tree according to the table-side struct selectors and emit the + // corresponding file-local child ids. A missing child mapping means this predicate-only path + // may need schema fallback; an existing child mapping without a file id means the table child + // is genuinely absent from this file and must stay above the file reader. + for (size_t selector_idx = 0; selector_idx < path.selectors.size(); ++selector_idx) { + const auto global_child_index = + struct_child_index(*current_mapping, path.selectors[selector_idx]); + if (!global_child_index.has_value()) { + *root_projection = {}; + return NestedProjectionResolveResult::NOT_REPRESENTED; + } + const auto* child_mapping = resolve_mapped_child(*current_mapping, *global_child_index); + if (child_mapping == nullptr) { + *root_projection = {}; + return NestedProjectionResolveResult::NOT_REPRESENTED; + } + if (!child_mapping->file_local_id.has_value()) { + *root_projection = {}; + return NestedProjectionResolveResult::MISSING_FILE_CHILD; + } + + auto child_projection = LocalColumnIndex::partial_local(*child_mapping->file_local_id); + child_projection.project_all_children = selector_idx + 1 == path.selectors.size(); + current_projection->children.push_back(std::move(child_projection)); + current_projection = ¤t_projection->children.back(); + current_mapping = child_mapping; + } + return NestedProjectionResolveResult::RESOLVED; +} + +static bool table_root_is_struct(const ColumnMapping& mapping) { + return struct_type_or_null(mapping.table_type) != nullptr; +} + +static const std::vector& scan_file_children(const ColumnMapping& mapping) { + return !mapping.projected_file_children.empty() ? mapping.projected_file_children + : mapping.original_file_children; +} + +static bool collect_file_child_names_from_projection(const std::vector& children, + const LocalColumnIndex& projection, + std::vector* file_child_names, + std::vector* file_child_types) { + DORIS_CHECK(file_child_names != nullptr); + DORIS_CHECK(file_child_types != nullptr); + const auto child_it = std::ranges::find_if(children, [&](const ColumnDefinition& child) { + return child.file_local_id() == projection.local_id(); + }); + if (child_it == children.end()) { + return false; + } + file_child_names->push_back(child_it->name); + file_child_types->push_back(child_it->type); + if (projection.children.empty()) { + return true; + } + if (projection.children.size() != 1) { + return false; + } + return collect_file_child_names_from_projection(child_it->children, projection.children[0], + file_child_names, file_child_types); +} + +} // namespace + +SplitLocalFileLiteral::SplitLocalFileLiteral(const DataTypePtr& file_type, const Field& file_field, + DataTypePtr original_type, Field original_field) + : VLiteral(file_type, file_field), + _original_type(std::move(original_type)), + _original_field(std::move(original_field)) {} + +GlobalIndex slot_ref_global_index(const VSlotRef& slot_ref) { + DORIS_CHECK(slot_ref.column_id() >= 0); + return GlobalIndex(cast_set(slot_ref.column_id())); +} + +bool is_struct_element_expr(const VExprSPtr& expr) { + if (expr == nullptr || expr->get_num_children() != 2) { + return false; + } + const auto& function_name = expr->fn().name.function_name; + if (function_name == "struct_element") { + return true; + } + if (function_name != "element_at") { + return false; + } + const auto& parent_type = expr->children()[0]->data_type(); + return parent_type != nullptr && + remove_nullable(parent_type)->get_primitive_type() == TYPE_STRUCT; +} + +Field literal_field(const VExprSPtr& literal_expr) { + DORIS_CHECK(literal_expr != nullptr); + DORIS_CHECK(literal_expr->is_literal()); + const auto* literal = dynamic_cast(literal_expr.get()); + DORIS_CHECK(literal != nullptr); + Field field; + literal->get_column_ptr()->get(0, field); + return field; +} + +bool resolve_nested_struct_path_for_file(const NestedStructPath& path, + const std::vector& mappings, + ResolvedNestedStructPath* resolved, + bool require_scan_projection) { + DORIS_CHECK(resolved != nullptr); + *resolved = {}; + const auto mapping_it = std::ranges::find_if(mappings, [&](const ColumnMapping& mapping) { + return mapping.global_index == path.root_global_index; + }); + if (mapping_it == mappings.end() || !mapping_it->file_local_id.has_value() || + path.selectors.empty()) { + return false; + } + + // Prefer ColumnMapping over schema-name lookup. This is the only path that can correctly + // localize renamed Iceberg fields: a table filter `element_at(s, 'renamed_b')` must become a + // file filter on physical child `b`, even if the old file type is `STRUCT`. + const auto mapping_result = + resolve_nested_projection_with_mapping(path, mappings, &resolved->file_projection); + if (mapping_result == NestedProjectionResolveResult::MISSING_FILE_CHILD) { + return false; + } + if (mapping_result == NestedProjectionResolveResult::NOT_REPRESENTED) { + if (!table_root_is_struct(*mapping_it)) { + return false; + } + LocalColumnIndex child_projection; + if (!build_file_child_projection_from_schema(mapping_it->original_file_children, + path.selectors, &child_projection) + .ok() || + child_projection.local_id() < 0) { + return false; + } + resolved->file_projection = LocalColumnIndex::partial_local(*mapping_it->file_local_id); + resolved->file_projection.children.push_back(std::move(child_projection)); + } + + if (resolved->file_projection.children.size() != 1) { + *resolved = {}; + return false; + } + // When rewriting the final localized element_at chain, it executes on the file column produced + // by this scan, so the intermediate return types must match the projected file shape, not the + // full historical file schema. Example: + // SELECT s.c WHERE element_at(element_at(s, 'b'), 'cc') LIKE 'NestedC%' + // reads only b.cc and c; the inner element_at(s, 'b') returns Struct(cc), not + // Struct(cc, new_dd). + // + // Earlier projection collection also calls this resolver before filter-only children have been + // merged into the scan projection. That phase only needs the file path, so it still resolves + // names/types from the original file schema. + const auto& child_source = require_scan_projection ? scan_file_children(*mapping_it) + : mapping_it->original_file_children; + if (!collect_file_child_names_from_projection( + child_source, resolved->file_projection.children[0], &resolved->file_child_names, + &resolved->file_child_types) || + resolved->file_child_names.size() != path.selectors.size() || + resolved->file_child_types.size() != path.selectors.size()) { + *resolved = {}; + return false; + } + return true; +} + +bool resolve_nested_struct_expr_for_file(const VExprSPtr& expr, + const std::vector& mappings, + ResolvedNestedStructPath* resolved) { + DORIS_CHECK(resolved != nullptr); + NestedStructPath path; + if (!extract_nested_struct_path(expr, &path)) { + *resolved = {}; + return false; + } + return resolve_nested_struct_path_for_file(path, mappings, resolved, true); +} + +// Collect nested struct leaf references that can be turned into file-reader projections. For +// example, from `s.a > 1 AND element_at(s, 'b') = 2`, this records two paths rooted at `s`: +// `s -> a` and `s -> b`. Non-struct expressions are traversed recursively, while a recognized +// struct path is emitted once so the caller can merge it into the scan projection for that +// top-level file column. +void collect_nested_struct_paths(const VExprSPtr& expr, std::vector* paths) { + DORIS_CHECK(paths != nullptr); + if (expr == nullptr) { + return; + } + NestedStructPath path; + if (extract_nested_struct_path_for_pruning(expr, &path)) { + paths->push_back(std::move(path)); + return; + } + for (const auto& child : expr->children()) { + collect_nested_struct_paths(child, paths); + } +} + +std::vector present_child_mappings_in_file_order( + const std::vector& child_mappings) { + std::vector result; + result.reserve(child_mappings.size()); + for (const auto& child_mapping : child_mappings) { + if (child_mapping.file_local_id.has_value()) { + result.push_back(&child_mapping); + } + } + std::ranges::sort(result, [](const ColumnMapping* lhs, const ColumnMapping* rhs) { + DORIS_CHECK(lhs->file_local_id.has_value()); + DORIS_CHECK(rhs->file_local_id.has_value()); + return *lhs->file_local_id < *rhs->file_local_id; + }); + return result; +} + +// Build the nested child projection under a top-level file column by walking file schema children +// directly. The returned projection does not include the root column id; callers attach it under a +// `LocalColumnIndex::partial_local(root_id)` when merging into the scan request. +Status build_file_child_projection_from_schema(const std::vector& children, + std::span selectors, + LocalColumnIndex* projection) { + DORIS_CHECK(projection != nullptr); + if (selectors.empty()) { + return Status::InvalidArgument("Nested struct selector path is empty"); + } + const auto* child = resolve_file_child(children, selectors.front()); + if (child == nullptr) { + return Status::OK(); + } + *projection = LocalColumnIndex::local(child->file_local_id()); + projection->project_all_children = selectors.size() == 1; + projection->children.clear(); + if (selectors.size() == 1) { + return Status::OK(); + } + if (child->children.empty() || + remove_nullable(child->type)->get_primitive_type() != TYPE_STRUCT) { + *projection = LocalColumnIndex {}; + return Status::OK(); + } + LocalColumnIndex child_projection; + RETURN_IF_ERROR(build_file_child_projection_from_schema(child->children, selectors.subspan(1), + &child_projection)); + if (child_projection.local_id() < 0) { + *projection = LocalColumnIndex {}; + return Status::OK(); + } + projection->children.push_back(std::move(child_projection)); + return Status::OK(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/column_mapper_nested.h b/be/src/format_v2/column_mapper_nested.h new file mode 100644 index 00000000000000..7b2e8cb1513cb4 --- /dev/null +++ b/be/src/format_v2/column_mapper_nested.h @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/field.h" +#include "exprs/vexpr_fwd.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/column_mapper.h" +#include "format_v2/file_reader.h" + +namespace doris::format { + +struct StructChildSelector { + bool by_name = true; + std::string name; + size_t ordinal = 0; +}; + +struct NestedStructPath { + GlobalIndex root_global_index; + std::vector selectors; +}; + +struct ResolvedNestedStructPath { + LocalColumnIndex file_projection; + std::vector file_child_names; + std::vector file_child_types; +}; + +// A split-local literal produced by slot-literal predicate localization. This wrapper keeps the +// original table literal so a cloned conjunct can be localized again for another split. +class SplitLocalFileLiteral final : public VLiteral { +public: + SplitLocalFileLiteral(const DataTypePtr& file_type, const Field& file_field, + DataTypePtr original_type, Field original_field); + + const DataTypePtr& original_type() const { return _original_type; } + const Field& original_field() const { return _original_field; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + Field file_field; + get_column_ptr()->get(0, file_field); + *cloned_expr = std::make_shared(_data_type, file_field, + _original_type, _original_field); + return Status::OK(); + } + +private: + DataTypePtr _original_type; + Field _original_field; +}; + +GlobalIndex slot_ref_global_index(const VSlotRef& slot_ref); +bool is_struct_element_expr(const VExprSPtr& expr); +Field literal_field(const VExprSPtr& literal_expr); + +bool resolve_nested_struct_path_for_file(const NestedStructPath& path, + const std::vector& mappings, + ResolvedNestedStructPath* resolved, + bool require_scan_projection = false); + +bool resolve_nested_struct_expr_for_file(const VExprSPtr& expr, + const std::vector& mappings, + ResolvedNestedStructPath* resolved); + +void collect_nested_struct_paths(const VExprSPtr& expr, std::vector* paths); + +std::vector present_child_mappings_in_file_order( + const std::vector& child_mappings); + +Status build_file_child_projection_from_schema(const std::vector& children, + std::span selectors, + LocalColumnIndex* projection); + +} // namespace doris::format diff --git a/be/src/format_v2/deletion_vector.h b/be/src/format_v2/deletion_vector.h new file mode 100644 index 00000000000000..2c89771b2e6b0e --- /dev/null +++ b/be/src/format_v2/deletion_vector.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "roaring/roaring64map.hh" + +namespace doris { + +// A deletion vector is already a bitmap on the wire. Keep decoded DVs compressed in the +// query-local cache instead of expanding every set bit into an int64_t. Position delete files use +// a different representation because their input is a stream of (file_path, row_position) rows. +using DeletionVector = roaring::Roaring64Map; + +} // namespace doris diff --git a/be/src/format_v2/deletion_vector_reader.cpp b/be/src/format_v2/deletion_vector_reader.cpp new file mode 100644 index 00000000000000..d70956de8b60fc --- /dev/null +++ b/be/src/format_v2/deletion_vector_reader.cpp @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/deletion_vector_reader.h" + +#include + +#include +#include + +#include "exec/common/endian.h" + +namespace doris::format { + +std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, + const TIcebergDeleteFileDesc& delete_file) { + return fmt::format("delete_dv_{}:{}{}:{}#{}#{}", data_file_path.size(), data_file_path, + delete_file.path.size(), delete_file.path, delete_file.content_offset, + delete_file.content_size_in_bytes); +} + +Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* rows_to_delete) { + if (buf == nullptr || rows_to_delete == nullptr) { + return Status::InvalidArgument("invalid deletion vector decode arguments"); + } + if (buffer_size < 12) { + return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); + } + + const auto total_length = BigEndian::Load32(buf); + if (total_length + 8 != buffer_size) { + return Status::DataQualityError("Deletion vector length mismatch, expected: {}, actual: {}", + total_length + 8, buffer_size); + } + + constexpr static char MAGIC_NUMBER[] = {'\xD1', '\xD3', '\x39', '\x64'}; + if (std::memcmp(buf + sizeof(total_length), MAGIC_NUMBER, 4) != 0) { + return Status::DataQualityError("Deletion vector magic number mismatch"); + } + + try { + *rows_to_delete |= roaring::Roaring64Map::readSafe(buf + 8, buffer_size - 12); + } catch (const std::runtime_error& e) { + return Status::DataQualityError("Decode roaring bitmap failed, {}", e.what()); + } + return Status::OK(); +} + +std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file) { + return fmt::format("paimon_dv_{}#{}#{}", deletion_file.path, deletion_file.offset, + deletion_file.length); +} + +Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* deletion_vector) { + if (deletion_vector == nullptr) { + return Status::InvalidArgument("deletion_vector must not be null"); + } + if (buffer_size < 8) [[unlikely]] { + return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); + } + const uint32_t actual_length = BigEndian::Load32(buf); + if (actual_length + 4 != buffer_size) [[unlikely]] { + return Status::RuntimeError( + "DeletionVector deserialize error: length not match, actual length: {}, expect " + "length: {}", + actual_length, buffer_size - 4); + } + constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'}; + if (std::memcmp(buf + sizeof(actual_length), PAIMON_BITMAP_MAGIC, 4) != 0) [[unlikely]] { + return Status::RuntimeError("DeletionVector deserialize error: invalid magic number {}", + BigEndian::Load32(buf + sizeof(actual_length))); + } + roaring::Roaring roaring_bitmap; + try { + roaring_bitmap = roaring::Roaring::readSafe(buf + 8, buffer_size - 8); + } catch (const std::runtime_error& e) { + return Status::RuntimeError( + "DeletionVector deserialize error: failed to deserialize roaring bitmap, {}", + e.what()); + } + *deletion_vector |= DeletionVector(std::move(roaring_bitmap)); + return Status::OK(); +} + +DeletionVectorReader::~DeletionVectorReader() { + _file_reader.reset(); + _merge_io_statistics(); +} + +void DeletionVectorReader::_init_io_context() { + if (_parent_io_ctx == nullptr) { + return; + } + _reader_io_ctx = *_parent_io_ctx; + _reader_io_ctx.file_cache_stats = &_file_cache_stats; + _reader_io_ctx.file_reader_stats = &_file_reader_stats; + _io_ctx = &_reader_io_ctx; +} + +void DeletionVectorReader::_merge_io_statistics() { + if (_statistics_merged || _parent_io_ctx == nullptr) { + return; + } + if (_parent_io_ctx->file_cache_stats != nullptr) { + _parent_io_ctx->file_cache_stats->merge_from(_file_cache_stats); + } + if (_parent_io_ctx->file_reader_stats != nullptr) { + _parent_io_ctx->file_reader_stats->read_calls += _file_reader_stats.read_calls; + _parent_io_ctx->file_reader_stats->read_bytes += _file_reader_stats.read_bytes; + _parent_io_ctx->file_reader_stats->read_time_ns += _file_reader_stats.read_time_ns; + _parent_io_ctx->file_reader_stats->read_rows += _file_reader_stats.read_rows; + } + _statistics_merged = true; +} + +Status DeletionVectorReader::open() { + if (_is_opened) [[unlikely]] { + return Status::OK(); + } + + _init_system_properties(); + _init_file_description(); + RETURN_IF_ERROR(_create_file_reader()); + + _file_size = _file_reader->size(); + _is_opened = true; + return Status::OK(); +} + +Status DeletionVectorReader::read_at(size_t offset, Slice result) { + if (UNLIKELY(_parent_io_ctx != nullptr && _parent_io_ctx->should_stop)) { + return Status::EndOfFile("stop read."); + } + if (_io_ctx != nullptr) { + _io_ctx->should_stop = _parent_io_ctx->should_stop; + } + size_t bytes_read = 0; + RETURN_IF_ERROR(_file_reader->read_at(offset, result, &bytes_read, _io_ctx)); + if (bytes_read != result.size) [[unlikely]] { + return Status::IOError("Failed to read fully at offset {}, expected {}, got {}", offset, + result.size, bytes_read); + } + return Status::OK(); +} + +Status DeletionVectorReader::_create_file_reader() { + if (UNLIKELY(_parent_io_ctx != nullptr && _parent_io_ctx->should_stop)) { + return Status::EndOfFile("stop read."); + } + + _file_description.mtime = _desc.modification_time; + io::FileReaderOptions reader_options = + FileFactory::get_reader_options(_state->query_options(), _file_description); + _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( + _profile, _system_properties, _file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, _io_ctx)); + return Status::OK(); +} + +void DeletionVectorReader::_init_file_description() { + _file_description.path = _desc.path; + _file_description.file_size = _desc.file_size; + _file_description.fs_name = _desc.fs_name; +} + +void DeletionVectorReader::_init_system_properties() { + _system_properties.system_type = _params.file_type; + _system_properties.properties = _params.properties; + _system_properties.hdfs_params = _params.hdfs_params; + if (_params.__isset.broker_addresses) { + _system_properties.broker_addresses.assign(_params.broker_addresses.begin(), + _params.broker_addresses.end()); + } +} + +} // namespace doris::format diff --git a/be/src/format_v2/deletion_vector_reader.h b/be/src/format_v2/deletion_vector_reader.h new file mode 100644 index 00000000000000..7ecdb6f1ab219f --- /dev/null +++ b/be/src/format_v2/deletion_vector_reader.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "format/generic_reader.h" +#include "format_v2/deletion_vector.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/file_factory.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader.h" +#include "io/io_common.h" +#include "util/profile_collector.h" +#include "util/slice.h" + +namespace doris::format { + +struct DeleteFileDesc { + enum class Format { + PAIMON, + ICEBERG, + }; + + std::string key = ""; + std::string path = ""; + std::string fs_name = ""; + int64_t start_offset = 0; + int64_t size = 0; + int64_t file_size = -1; + int64_t modification_time = 0; + Format format = Format::PAIMON; +}; + +std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, + const TIcebergDeleteFileDesc& delete_file); + +Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* rows_to_delete); + +std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file); + +Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* deletion_vector); + +class DeletionVectorReader { +public: + DeletionVectorReader(RuntimeState* state, RuntimeProfile* profile, + const TFileScanRangeParams& params, const DeleteFileDesc& desc, + io::IOContext* io_ctx) + : _state(state), + _profile(profile), + _params(params), + _desc(desc), + _parent_io_ctx(io_ctx) { + _init_io_context(); + } + ~DeletionVectorReader(); + + DeletionVectorReader(const DeletionVectorReader&) = delete; + DeletionVectorReader& operator=(const DeletionVectorReader&) = delete; + + Status open(); + Status read_at(size_t offset, Slice result); + + const io::FileCacheStatistics& file_cache_statistics() const { return _file_cache_stats; } + +private: + void _init_system_properties(); + void _init_file_description(); + void _init_io_context(); + void _merge_io_statistics(); + Status _create_file_reader(); + + RuntimeState* _state = nullptr; + RuntimeProfile* _profile = nullptr; + const TFileScanRangeParams& _params; + DeleteFileDesc _desc; + io::IOContext* _parent_io_ctx = nullptr; + io::IOContext _reader_io_ctx; + io::IOContext* _io_ctx = nullptr; + io::FileCacheStatistics _file_cache_stats; + io::FileReaderStats _file_reader_stats; + bool _statistics_merged = false; + + io::FileSystemProperties _system_properties; + io::FileDescription _file_description; + io::FileReaderSPtr _file_reader; + int64_t _file_size = 0; + bool _is_opened = false; +}; + +} // namespace doris::format diff --git a/be/src/format_v2/delimited_text/csv_reader.cpp b/be/src/format_v2/delimited_text/csv_reader.cpp new file mode 100644 index 00000000000000..bb6c55d30847b2 --- /dev/null +++ b/be/src/format_v2/delimited_text/csv_reader.cpp @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/delimited_text/csv_reader.h" + +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type_serde/data_type_string_serde.h" +#include "format/file_reader/new_plain_binary_line_reader.h" +#include "format/file_reader/new_plain_text_line_reader.h" +#include "gen_cpp/internal_service.pb.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" +#include "util/decompressor.h" +#include "util/utf8_check.h" + +namespace doris::format::csv { +namespace { + +bool starts_with_at(const Slice& line, size_t pos, const std::string& needle) { + return !needle.empty() && pos + needle.size() <= line.size && + std::memcmp(line.data + pos, needle.data(), needle.size()) == 0; +} + +bool is_csv_text_format(TFileFormatType::type format_type) { + switch (format_type) { + case TFileFormatType::FORMAT_CSV_PLAIN: + case TFileFormatType::FORMAT_CSV_GZ: + case TFileFormatType::FORMAT_CSV_BZ2: + case TFileFormatType::FORMAT_CSV_LZ4FRAME: + case TFileFormatType::FORMAT_CSV_LZ4BLOCK: + case TFileFormatType::FORMAT_CSV_LZOP: + case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK: + case TFileFormatType::FORMAT_CSV_DEFLATE: + return true; + default: + return false; + } +} + +} // namespace + +CsvReader::CsvReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileScanRangeParams* scan_params, + const std::vector& file_slot_descs, + TFileCompressType::type range_compress_type, + std::optional stream_load_id) + : DelimitedTextReader(system_properties, file_description, std::move(io_ctx), profile, + scan_params, file_slot_descs, range_compress_type, + std::move(stream_load_id), "CSV") {} + +CsvReader::~CsvReader() = default; + +Status CsvReader::_init_format_state() { + _file_format_type = _scan_params->format_type; + _file_compress_type = + _range_compress_type != TFileCompressType::UNKNOWN + ? _range_compress_type + : (_scan_params->__isset.compress_type ? _scan_params->compress_type + : TFileCompressType::UNKNOWN); + if (_file_compress_type == TFileCompressType::UNKNOWN && + _file_format_type == TFileFormatType::FORMAT_CSV_PLAIN) { + // FORMAT_CSV_PLAIN is an uncompressed byte stream even when FE does not fill + // compress_type. Non-first splits rely on this normalization; otherwise UNKNOWN would be + // rejected by the split-compressed-file guard in the shared reader base. + _file_compress_type = TFileCompressType::PLAIN; + } + + const auto& text_params = _scan_params->file_attributes.text_params; + _value_separator = text_params.column_separator; + _line_delimiter = text_params.line_delimiter; + if (text_params.__isset.enclose) { + _enclose = text_params.enclose; + } + if (text_params.__isset.escape) { + _escape = text_params.escape; + } + _trim_tailing_spaces = _runtime_state != nullptr && + _runtime_state->trim_tailing_spaces_for_external_table_query(); + _options.escape_char = _escape; + _options.quote_char = _enclose; + _options.collection_delim = + text_params.collection_delimiter.empty() ? ',' : text_params.collection_delimiter[0]; + _options.map_key_delim = + text_params.mapkv_delimiter.empty() ? ':' : text_params.mapkv_delimiter[0]; + if (text_params.__isset.null_format) { + _options.null_format = text_params.null_format.data(); + _options.null_len = text_params.null_format.length(); + } + if (_scan_params->file_attributes.__isset.trim_double_quotes) { + _trim_double_quotes = _scan_params->file_attributes.trim_double_quotes; + } + _options.converted_from_string = _trim_double_quotes; + if (_runtime_state != nullptr) { + _keep_cr = _runtime_state->query_options().keep_carriage_return; + } + if (text_params.__isset.empty_field_as_null) { + _empty_field_as_null = text_params.empty_field_as_null; + } + return Status::OK(); +} + +Status CsvReader::_create_decompressor() { + if (_file_compress_type != TFileCompressType::UNKNOWN) { + return Decompressor::create_decompressor(_file_compress_type, &_decompressor); + } + return Decompressor::create_decompressor(_file_format_type, &_decompressor); +} + +Status CsvReader::_create_line_reader() { + if (is_csv_text_format(_file_format_type)) { + std::shared_ptr text_line_reader_ctx; + if (_enclose == 0) { + text_line_reader_ctx = std::make_shared( + _line_delimiter, _line_delimiter.size(), _keep_cr); + } else { + const size_t col_sep_num = + _source_file_slot_descs.size() > 1 ? _source_file_slot_descs.size() - 1 : 0; + _enclose_reader_ctx = std::make_shared( + _line_delimiter, _line_delimiter.size(), _value_separator, + _value_separator.size(), col_sep_num, _enclose, _escape, _keep_cr, + _start_offset == 0); + text_line_reader_ctx = _enclose_reader_ctx; + } + _line_reader = NewPlainTextLineReader::create_unique( + _profile, _file_reader, _decompressor.get(), std::move(text_line_reader_ctx), _size, + _start_offset); + return Status::OK(); + } + if (_file_format_type == TFileFormatType::FORMAT_PROTO) { + _line_reader = NewPlainBinaryLineReader::create_unique(_file_reader); + return Status::OK(); + } + return Status::InternalError("Unknown CSV format type {}", _file_format_type); +} + +Status CsvReader::_validate_line(const Slice& line) { + if (_file_format_type != TFileFormatType::FORMAT_PROTO && _enable_text_validate_utf8 && + !validate_utf8(line.data, line.size)) { + return Status::InternalError("Only support csv data in utf8 codec"); + } + return Status::OK(); +} + +void CsvReader::_split_line(const Slice& line) { + _split_values.clear(); + if (_file_format_type == TFileFormatType::FORMAT_PROTO) { + auto** row_ptr = reinterpret_cast(line.data); + PDataRow* row = *row_ptr; + for (const PDataColumn& col : row->col()) { + _split_values.emplace_back(col.value()); + } + return; + } + + const auto append_value = [&](size_t value_start, size_t value_len) { + while (_trim_tailing_spaces && value_len > 0 && + line.data[value_start + value_len - 1] == ' ') { + --value_len; + } + if (_enclose != 0 && value_len > 1 && line.data[value_start] == _enclose && + line.data[value_start + value_len - 1] == _enclose) { + ++value_start; + value_len -= 2; + } + _split_values.emplace_back(line.data + value_start, value_len); + }; + + size_t value_start = 0; + if (_enclose_reader_ctx != nullptr) { + for (const size_t separator_position : _enclose_reader_ctx->column_sep_positions()) { + DORIS_CHECK_LE(value_start, separator_position); + DORIS_CHECK_LE(separator_position, line.size); + append_value(value_start, separator_position - value_start); + value_start = separator_position + _value_separator.size(); + } + } else { + for (size_t i = 0; i < line.size;) { + if (starts_with_at(line, i, _value_separator)) { + append_value(value_start, i - value_start); + i += _value_separator.size(); + value_start = i; + } else { + ++i; + } + } + } + DORIS_CHECK_LE(value_start, line.size); + append_value(value_start, line.size - value_start); +} + +Status CsvReader::_deserialize_one_cell(const RequestedColumn& column, IColumn* output, + Slice value) { + DORIS_CHECK(output != nullptr); + if (column.nullable_string_fast_path) { + auto& null_column = assert_cast(*output); + // String is the hottest CSV type. Avoid the generic nullable serde wrapper here: + // deserialize directly into the nested string column and append the null map bit ourselves. + if (_empty_field_as_null && value.size == 0) { + null_column.insert_data(nullptr, 0); + return Status::OK(); + } + // CSV keeps empty-field handling separate from null_format matching. An empty + // null_format must not turn every empty CSV field into NULL unless FE explicitly sets + // empty_field_as_null; OpenCSV-compatible tables expect empty fields to stay empty strings. + const bool quoted = _options.converted_from_string && value.trim_double_quotes(); + if (!quoted && _options.null_len > 0 && value.size == _options.null_len && + std::memcmp(value.data, _options.null_format, value.size) == 0) { + null_column.insert_data(nullptr, 0); + return Status::OK(); + } + static DataTypeStringSerDe string_serde(TYPE_STRING); + auto status = string_serde.deserialize_one_cell_from_csv(null_column.get_nested_column(), + value, _options); + if (!status.ok()) { + null_column.insert_data(nullptr, 0); + return Status::OK(); + } + null_column.get_null_map_data().push_back(0); + return Status::OK(); + } + return column.serde->deserialize_one_cell_from_csv(*output, value, _options); +} + +Slice CsvReader::_normalize_value(Slice value) const { + if (_empty_field_as_null && value.size == 0) { + return Slice(_options.null_format, _options.null_len); + } + return value; +} + +bool CsvReader::_can_split() const { + return (_file_compress_type == TFileCompressType::PLAIN) || + (_file_compress_type == TFileCompressType::UNKNOWN && + _file_format_type == TFileFormatType::FORMAT_CSV_PLAIN); +} + +void CsvReader::_on_bom_removed(size_t bom_size) { + if (_enclose_reader_ctx != nullptr) { + _enclose_reader_ctx->adjust_column_sep_positions(bom_size); + } +} + +} // namespace doris::format::csv diff --git a/be/src/format_v2/delimited_text/csv_reader.h b/be/src/format_v2/delimited_text/csv_reader.h new file mode 100644 index 00000000000000..dd1d77d5a34b9a --- /dev/null +++ b/be/src/format_v2/delimited_text/csv_reader.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "format_v2/delimited_text/delimited_text_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "util/slice.h" + +namespace doris { +class EncloseCsvLineReaderCtx; +class SlotDescriptor; +} // namespace doris + +namespace doris::format::csv { + +// FileScannerV2 CSV reader. +// +// CSV files do not carry a physical schema. FE provides the table slot descriptors plus +// TFileScanRangeParams::column_idxs, where each file slot maps to a CSV field ordinal. This reader +// exposes that information as a v2 file-local schema and implements CSV parsing directly in the v2 +// FileReader contract. +class CsvReader final : public ::doris::format::DelimitedTextReader { +public: + // `file_slot_descs` must contain only columns physically readable from the CSV payload. + // Partition/default/virtual columns are materialized by TableReader after this reader returns + // a file-local block. Keeping that boundary is important because CSV has no embedded schema + // from which those non-file columns could be derived. + CsvReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileScanRangeParams* scan_params, + const std::vector& file_slot_descs, + TFileCompressType::type range_compress_type = TFileCompressType::UNKNOWN, + std::optional stream_load_id = std::nullopt); + ~CsvReader() override; + +private: + Status _init_format_state() override; + Status _create_decompressor() override; + Status _create_line_reader() override; + Status _validate_line(const Slice& line) override; + void _split_line(const Slice& line) override; + Status _deserialize_one_cell(const RequestedColumn& column, IColumn* output, + Slice value) override; + Slice _normalize_value(Slice value) const override; + bool _can_split() const override; + void _on_bom_removed(size_t bom_size) override; + + TFileFormatType::type _file_format_type = TFileFormatType::FORMAT_CSV_PLAIN; + char _enclose = 0; + bool _trim_double_quotes = false; + bool _trim_tailing_spaces = false; + bool _empty_field_as_null = false; + bool _keep_cr = false; + std::shared_ptr _enclose_reader_ctx; +}; + +} // namespace doris::format::csv diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp b/be/src/format_v2/delimited_text/delimited_text_reader.cpp new file mode 100644 index 00000000000000..f19d12c75714b9 --- /dev/null +++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp @@ -0,0 +1,650 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/delimited_text/delimited_text_reader.h" + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "format/line_reader.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "io/file_factory.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" +#include "util/decompressor.h" +#include "util/string_util.h" + +namespace doris::format { +namespace { + +constexpr const char* DELIMITED_TEXT_PROFILE = "DelimitedTextReader"; + +void update_counter(RuntimeProfile::Counter* counter, int64_t value) { + if (counter != nullptr) { + COUNTER_UPDATE(counter, value); + } +} + +DataTypePtr nullable_type(DataTypePtr type) { + return type != nullptr && type->is_nullable() ? std::move(type) + : make_nullable(std::move(type)); +} + +DataTypePtr delimited_file_type_from_slot_type(const DataTypePtr& type) { + if (type == nullptr) { + return nullptr; + } + + const bool is_nullable = type->is_nullable(); + const auto nested_type = remove_nullable(type); + DataTypePtr file_type; + switch (nested_type->get_primitive_type()) { + case TYPE_CHAR: + case TYPE_VARCHAR: + // Delimited text files do not carry CHAR/VARCHAR length metadata. FE slot types describe + // the table target type, not a bounded physical file type. Expose bounded strings as + // unbounded STRING on the file side so TableReader can later enforce the table length. + // Example: a TEXT field "hangzhou" mapped to table CHAR(3) must be read as STRING and + // truncated to "han" during table materialization. + file_type = std::make_shared(); + break; + case TYPE_ARRAY: { + const auto* array_type = assert_cast(nested_type.get()); + file_type = std::make_shared( + delimited_file_type_from_slot_type(array_type->get_nested_type())); + break; + } + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + file_type = std::make_shared( + delimited_file_type_from_slot_type(map_type->get_key_type()), + delimited_file_type_from_slot_type(map_type->get_value_type())); + break; + } + case TYPE_STRUCT: { + const auto* struct_type = assert_cast(nested_type.get()); + DataTypes file_children; + file_children.reserve(struct_type->get_elements().size()); + for (const auto& child_type : struct_type->get_elements()) { + file_children.push_back(delimited_file_type_from_slot_type(child_type)); + } + file_type = + std::make_shared(file_children, struct_type->get_element_names()); + break; + } + default: + file_type = nested_type; + break; + } + + return is_nullable ? make_nullable(file_type) : file_type; +} + +ColumnDefinition synthetic_file_child(const std::string& name, DataTypePtr type, int32_t local_id); + +std::vector synthesize_file_children_from_type(const DataTypePtr& type) { + std::vector children; + if (type == nullptr) { + return children; + } + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_ARRAY: { + const auto* array_type = assert_cast(nested_type.get()); + children.push_back(synthetic_file_child("element", array_type->get_nested_type(), 0)); + break; + } + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + children.push_back(synthetic_file_child("key", map_type->get_key_type(), 0)); + children.push_back(synthetic_file_child("value", map_type->get_value_type(), 1)); + break; + } + case TYPE_STRUCT: { + const auto* struct_type = assert_cast(nested_type.get()); + children.reserve(struct_type->get_elements().size()); + for (size_t idx = 0; idx < struct_type->get_elements().size(); ++idx) { + children.push_back(synthetic_file_child(struct_type->get_element_name(idx), + struct_type->get_element(idx), + cast_set(idx))); + } + break; + } + default: + break; + } + return children; +} + +ColumnDefinition synthetic_file_child(const std::string& name, DataTypePtr type, int32_t local_id) { + ColumnDefinition child; + child.identifier = Field::create_field(name); + child.local_id = local_id; + child.name = name; + child.type = std::move(type); + child.children = synthesize_file_children_from_type(child.type); + return child; +} + +} // namespace + +DelimitedTextReader::DelimitedTextReader( + std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileScanRangeParams* scan_params, + const std::vector& file_slot_descs, + TFileCompressType::type range_compress_type, std::optional stream_load_id, + std::string reader_name) + : FileReader(system_properties, file_description, std::move(io_ctx), profile), + _scan_params(scan_params), + _source_file_slot_descs(file_slot_descs), + _range_compress_type(range_compress_type), + _stream_load_id(std::move(stream_load_id)), + _reader_name(std::move(reader_name)) {} + +DelimitedTextReader::~DelimitedTextReader() { + static_cast(close()); +} + +void DelimitedTextReader::_init_profile() { + if (_profile == nullptr || _text_profile.raw_lines_read != nullptr) { + return; + } + + ADD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, 1); + _text_profile.open_file_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OpenFileTime", DELIMITED_TEXT_PROFILE, 1); + _text_profile.create_line_reader_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CreateLineReaderTime", DELIMITED_TEXT_PROFILE, 1); + _text_profile.read_line_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadLineTime", DELIMITED_TEXT_PROFILE, 1); + _text_profile.split_line_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SplitLineTime", DELIMITED_TEXT_PROFILE, 1); + _text_profile.deserialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DeserializeTime", DELIMITED_TEXT_PROFILE, 1); + _text_profile.conjunct_filter_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ConjunctFilterTime", DELIMITED_TEXT_PROFILE, 1); + _text_profile.delete_conjunct_filter_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "DeleteConjunctFilterTime", DELIMITED_TEXT_PROFILE, 1); + _text_profile.raw_lines_read = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "RawLinesRead", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _text_profile.rows_read_before_filter = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "RowsReadBeforeFilter", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _text_profile.rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "RowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _text_profile.rows_filtered_by_delete_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "RowsFilteredByDeleteConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _text_profile.rows_returned = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "RowsReturned", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _text_profile.empty_lines_read = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "EmptyLinesRead", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _text_profile.skipped_lines = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "SkippedLines", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _text_profile.cells_deserialized = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "CellsDeserialized", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); +} + +Status DelimitedTextReader::init(RuntimeState* state) { + _init_profile(); + _runtime_state = state; + if (_scan_params == nullptr) { + return Status::InvalidArgument("{} v2 reader requires scan params", _reader_name); + } + if (_file_description == nullptr) { + return Status::InvalidArgument("{} v2 reader requires file description", _reader_name); + } + if (!_scan_params->__isset.file_attributes || + !_scan_params->file_attributes.__isset.text_params) { + return Status::InvalidArgument("{} v2 reader requires text file attributes", _reader_name); + } + _enable_text_validate_utf8 = !_scan_params->file_attributes.__isset.enable_text_validate_utf8 || + _scan_params->file_attributes.enable_text_validate_utf8; + + RETURN_IF_ERROR(_init_format_state()); + + // Delimited text files have no physical column ids. FE sends `column_idxs` to describe how + // each physical file slot maps to a field ordinal in the text row. The local id exposed in the + // file schema is therefore the text-field ordinal, not the slot vector position. + _source_column_idxs.clear(); + if (_scan_params->__isset.column_idxs && !_scan_params->column_idxs.empty()) { + if (_scan_params->column_idxs.size() != _source_file_slot_descs.size()) { + return Status::InvalidArgument( + "{} v2 reader column_idxs size {} does not match file slot size {}", + _reader_name, _scan_params->column_idxs.size(), _source_file_slot_descs.size()); + } + _source_column_idxs.reserve(_scan_params->column_idxs.size()); + for (const auto column_idx : _scan_params->column_idxs) { + _source_column_idxs.push_back(column_idx); + } + } else { + _source_column_idxs.reserve(_source_file_slot_descs.size()); + for (size_t i = 0; i < _source_file_slot_descs.size(); ++i) { + _source_column_idxs.push_back(static_cast(i)); + } + } + + _source_serdes = create_data_type_serdes(_source_file_slot_descs); + _file_schema.clear(); + _file_schema.reserve(_source_file_slot_descs.size()); + for (size_t i = 0; i < _source_file_slot_descs.size(); ++i) { + const auto* slot = _source_file_slot_descs[i]; + DORIS_CHECK(slot != nullptr); + ColumnDefinition field; + field.identifier = Field::create_field(slot->col_name()); + field.local_id = _source_column_idxs[i]; + field.name = slot->col_name(); + field.type = nullable_type(delimited_file_type_from_slot_type(slot->get_data_type_ptr())); + // Delimited text stores a complex value in one top-level text field, but TableColumnMapper + // still needs semantic children to localize nested projections and predicates. Expose + // ARRAY element, MAP key/value, and STRUCT fields as file-schema children while keeping the + // top-level local id as the physical text field ordinal from column_idxs. + field.children = synthesize_file_children_from_type(field.type); + _file_schema.push_back(std::move(field)); + } + _eof = false; + return Status::OK(); +} + +Status DelimitedTextReader::get_schema(std::vector* file_schema) const { + if (file_schema == nullptr) { + return Status::InvalidArgument("{} v2 file_schema is null", _reader_name); + } + *file_schema = _file_schema; + return Status::OK(); +} + +std::unique_ptr DelimitedTextReader::create_column_mapper( + TableColumnMapperOptions options) const { + return std::make_unique(std::move(options)); +} + +Status DelimitedTextReader::open(std::shared_ptr request) { + RETURN_IF_ERROR(FileReader::open(std::move(request))); + DORIS_CHECK(_request != nullptr); + RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); + { + SCOPED_TIMER(_text_profile.open_file_time); + RETURN_IF_ERROR(_open_file()); + } + RETURN_IF_ERROR(_create_decompressor()); + { + SCOPED_TIMER(_text_profile.create_line_reader_time); + RETURN_IF_ERROR(_create_line_reader()); + } + _line_reader_eof = false; + _bom_removed = false; + _eof = false; + return Status::OK(); +} + +Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof) { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + if (_line_reader == nullptr) { + return Status::InternalError("{} v2 reader is not open", _reader_name); + } + + const auto batch_size = _runtime_state != nullptr ? _runtime_state->batch_size() : 4096; + const auto max_block_bytes = _runtime_state != nullptr + ? _runtime_state->preferred_block_size_bytes() + : std::numeric_limits::max(); + *rows = 0; + *eof = false; + + { + auto columns_guard = file_block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + // Delimited text readers are column-pruned but not lazy materialized: all file-local + // columns requested by TableReader are decoded before file-local conjuncts are evaluated. + while (*rows < batch_size && !_line_reader_eof && + Block::columns_byte_size(columns) < max_block_bytes) { + Slice line; + bool line_eof = false; + RETURN_IF_ERROR(_read_next_line(&line, &line_eof)); + if (line_eof) { + break; + } + RETURN_IF_ERROR(_fill_columns_from_line(line, &columns, rows)); + } + } + + const size_t rows_before_filter = *rows; + update_counter(_text_profile.rows_read_before_filter, rows_before_filter); + _record_scan_rows(cast_set(rows_before_filter)); + + MaterializedReaderFilterProfile filter_profile; + filter_profile.delete_conjunct_filter_time = _text_profile.delete_conjunct_filter_time; + filter_profile.conjunct_filter_time = _text_profile.conjunct_filter_time; + filter_profile.rows_filtered_by_delete_conjunct = + _text_profile.rows_filtered_by_delete_conjunct; + filter_profile.rows_filtered_by_conjunct = _text_profile.rows_filtered_by_conjunct; + RETURN_IF_ERROR(apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, + rows, &filter_profile)); + update_counter(_text_profile.rows_returned, *rows); + *eof = _line_reader_eof && *rows == 0; + _eof = *eof; + return Status::OK(); +} + +Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& request, + FileAggregateResult* result) { + DORIS_CHECK(result != nullptr); + if (request.agg_type != TPushAggOp::type::COUNT) { + return Status::NotSupported("{} v2 reader only supports COUNT aggregate pushdown", + _reader_name); + } + if (_line_reader == nullptr) { + return Status::InternalError("{} v2 reader is not open", _reader_name); + } + + int64_t count = 0; + while (!_line_reader_eof) { + Slice line; + bool line_eof = false; + RETURN_IF_ERROR(_read_next_line(&line, &line_eof)); + if (line_eof) { + break; + } + if (line.size == 0) { + update_counter(_text_profile.empty_lines_read, 1); + if (_empty_line_as_record() || + (_runtime_state != nullptr && _runtime_state->is_read_csv_empty_line_as_null())) { + ++count; + } + continue; + } + RETURN_IF_ERROR(_validate_line(line)); + ++count; + } + result->count = count; + result->columns.clear(); + update_counter(_text_profile.rows_read_before_filter, count); + update_counter(_text_profile.rows_returned, count); + _record_scan_rows(count); + _eof = true; + return Status::OK(); +} + +Status DelimitedTextReader::close() { + if (_line_reader != nullptr) { + _line_reader->close(); + _line_reader.reset(); + } + _decompressor.reset(); + _file_reader.reset(); + _tracing_file_reader.reset(); + _requested_columns.clear(); + return Status::OK(); +} + +bool DelimitedTextReader::_is_null_format(Slice value) const { + if (value.size != _options.null_len) { + return false; + } + if (_options.null_len == 0) { + return true; + } + return std::memcmp(value.data, _options.null_format, value.size) == 0; +} + +Status DelimitedTextReader::_build_requested_columns(const FileScanRequest& request, + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + + // `request.local_positions` is keyed by FileReader schema local id. For delimited text readers + // that local id is the field ordinal from column_idxs, so reverse-map it to the source slot + // descriptor before choosing the serde. + std::vector by_position(request.local_positions.size()); + for (const auto& [file_column_id, block_position] : request.local_positions) { + const auto source_it = std::find(_source_column_idxs.begin(), _source_column_idxs.end(), + file_column_id.value()); + if (source_it == _source_column_idxs.end()) { + return Status::InvalidArgument("{} v2 request references unknown local column id {}", + _reader_name, file_column_id.value()); + } + const auto source_index = std::distance(_source_column_idxs.begin(), source_it); + DORIS_CHECK(source_index >= 0 && + static_cast(source_index) < _source_file_slot_descs.size()); + if (block_position.value() >= by_position.size()) { + return Status::InvalidArgument("{} v2 request has invalid block position {}", + _reader_name, block_position.value()); + } + const auto* slot = _source_file_slot_descs[source_index]; + const auto type = slot->get_data_type_ptr(); + RequestedColumn requested_column; + requested_column.file_column_id = file_column_id; + requested_column.block_position = block_position; + requested_column.slot_desc = slot; + requested_column.serde = _source_serdes[source_index]; + requested_column.nullable_string_fast_path = + type->is_nullable() && is_string_type(type->get_primitive_type()); + by_position[block_position.value()] = std::move(requested_column); + } + + for (size_t i = 0; i < by_position.size(); ++i) { + if (!by_position[i].file_column_id.is_valid()) { + return Status::InvalidArgument("{} v2 request misses block position {}", _reader_name, + i); + } + } + *columns = std::move(by_position); + return Status::OK(); +} + +Status DelimitedTextReader::_open_file() { + _start_offset = _file_description->range_start_offset; + _size = _file_description->range_size; + // Some callers, especially stream-load/http_stream, do not know the total length up front. + // For a first split this is fine: NewPlainTextLineReader can read until the underlying reader + // returns EOF. For non-first splits we still need a concrete range so the pre-read/skip-one-line + // boundary logic does not read an unbounded stream. + if (_size <= 0 && _file_description->file_size >= 0) { + _size = _file_description->file_size - _start_offset; + } + if (_size < 0 && _start_offset > 0) { + return Status::InvalidArgument("{} v2 reader requires a valid split size for {}", + _reader_name, _file_description->path); + } + _skip_lines = 0; + if (_start_offset == 0) { + if (_scan_params->file_attributes.__isset.header_type && + !_scan_params->file_attributes.header_type.empty()) { + const auto header_type = to_lower(_scan_params->file_attributes.header_type); + if (header_type == BeConsts::CSV_WITH_NAMES) { + _skip_lines = 1; + } else if (header_type == BeConsts::CSV_WITH_NAMES_AND_TYPES) { + _skip_lines = 2; + } + } else if (_scan_params->file_attributes.__isset.skip_lines) { + _skip_lines = _scan_params->file_attributes.skip_lines; + } + } else { + if (!_can_split()) { + return Status::InternalError("For now we do not support split compressed file"); + } + // Non-first splits normally start in the middle of a record. Pre-read at most one line + // delimiter byte range, then skip one line in `_read_next_line()`, so the first returned + // row is always complete. Example with '\n': + // file bytes: "1,a\n2,b\n" + // split start: ^ + // pre-read: ^ + // skipped line: "a" + // returned row: "2,b" + const int64_t pre_read_len = + std::min(static_cast(_line_delimiter.size()), _start_offset); + _start_offset -= pre_read_len; + _size += pre_read_len; + _skip_lines = 1; + } + + if (_scan_params->file_type == TFileType::FILE_STREAM) { + if (!_stream_load_id.has_value()) { + return Status::InvalidArgument("{} v2 stream reader requires load id", _reader_name); + } + // Stream load/http_stream data lives in NewLoadStreamMgr rather than a filesystem. The + // generic FileFactory path only supports real file systems, so FILE_STREAM must use the + // same pipe-reader lookup as the old CSV reader. + RETURN_IF_ERROR(FileFactory::create_pipe_reader(*_stream_load_id, &_file_reader, + _runtime_state, /*need_schema=*/false)); + } else { + auto reader_options = FileFactory::get_reader_options(_runtime_state->query_options(), + *_file_description); + auto file_reader = DORIS_TRY(FileFactory::create_file_reader( + *_system_properties, *_file_description, reader_options, _profile)); + _file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(std::move(file_reader), + _io_ctx->file_reader_stats) + : file_reader; + } + if (_file_reader->size() == 0 && _scan_params->file_type != TFileType::FILE_STREAM && + _scan_params->file_type != TFileType::FILE_BROKER) { + return Status::EndOfFile("init reader failed, empty {} file: {}", _reader_name, + _file_description->path); + } + return Status::OK(); +} + +Status DelimitedTextReader::_read_next_line(Slice* line, bool* eof) { + DORIS_CHECK(line != nullptr); + DORIS_CHECK(eof != nullptr); + while (true) { + const uint8_t* ptr = nullptr; + size_t size = 0; + { + SCOPED_TIMER(_text_profile.read_line_time); + RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size, &_line_reader_eof, _io_ctx.get())); + } + if (_line_reader_eof && size == 0) { + *eof = true; + return Status::OK(); + } + if (_skip_lines == 0 && !_bom_removed) { + // BOM is stripped only from the first logical data line. Header lines are skipped + // before this branch, so a BOM inside a skipped header does not leak into user data. + ptr = _remove_bom(ptr, &size); + _bom_removed = true; + } + if (_skip_lines > 0) { + --_skip_lines; + _bom_removed = true; + update_counter(_text_profile.skipped_lines, 1); + continue; + } + *line = Slice(ptr, size); + *eof = false; + update_counter(_text_profile.raw_lines_read, 1); + return Status::OK(); + } +} + +Status DelimitedTextReader::_fill_columns_from_line(const Slice& line, + std::vector* columns, + size_t* rows) { + DORIS_CHECK(columns != nullptr); + if (line.size == 0) { + update_counter(_text_profile.empty_lines_read, 1); + if (!_empty_line_as_record()) { + if (_runtime_state != nullptr && _runtime_state->is_read_csv_empty_line_as_null()) { + for (const auto& column : _requested_columns) { + RETURN_IF_ERROR(_append_null((*columns)[column.block_position.value()].get())); + update_counter(_text_profile.cells_deserialized, 1); + } + ++(*rows); + } + return Status::OK(); + } + } + RETURN_IF_ERROR(_validate_line(line)); + + { + SCOPED_TIMER(_text_profile.split_line_time); + _split_line(line); + } + SCOPED_TIMER(_text_profile.deserialize_time); + for (const auto& column : _requested_columns) { + auto* output = (*columns)[column.block_position.value()].get(); + const int32_t field_index = column.file_column_id.value(); + // Missing trailing fields are query-compatible with the old readers: they become NULL + // rather than shifting subsequent projected columns or rejecting the row. + Slice value = field_index >= 0 && static_cast(field_index) < _split_values.size() + ? _split_values[field_index] + : Slice(_options.null_format, _options.null_len); + RETURN_IF_ERROR(_deserialize_one_cell(column, output, _normalize_value(value))); + update_counter(_text_profile.cells_deserialized, 1); + } + ++(*rows); + return Status::OK(); +} + +Status DelimitedTextReader::_validate_line(const Slice& line) { + (void)line; + return Status::OK(); +} + +Slice DelimitedTextReader::_normalize_value(Slice value) const { + return value; +} + +bool DelimitedTextReader::_empty_line_as_record() const { + return false; +} + +bool DelimitedTextReader::_can_split() const { + return _file_compress_type == TFileCompressType::PLAIN; +} + +void DelimitedTextReader::_on_bom_removed(size_t bom_size) { + (void)bom_size; +} + +Status DelimitedTextReader::_append_null(IColumn* output) { + DORIS_CHECK(output != nullptr); + auto* nullable = assert_cast(output); + nullable->insert_data(nullptr, 0); + return Status::OK(); +} + +const uint8_t* DelimitedTextReader::_remove_bom(const uint8_t* ptr, size_t* size) { + DORIS_CHECK(size != nullptr); + if (ptr != nullptr && *size >= 3 && static_cast(ptr[0]) == 0xEF && + static_cast(ptr[1]) == 0xBB && static_cast(ptr[2]) == 0xBF) { + constexpr size_t BOM_SIZE = 3; + *size -= BOM_SIZE; + _on_bom_removed(BOM_SIZE); + return ptr + BOM_SIZE; + } + return ptr; +} + +} // namespace doris::format diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h b/be/src/format_v2/delimited_text/delimited_text_reader.h new file mode 100644 index 00000000000000..daea3bc90942d8 --- /dev/null +++ b/be/src/format_v2/delimited_text/delimited_text_reader.h @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "core/data_type_serde/data_type_serde.h" +#include "format_v2/file_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "runtime/runtime_profile.h" +#include "util/slice.h" + +namespace doris { +class Decompressor; +class LineReader; +class SlotDescriptor; +} // namespace doris + +namespace doris::format { + +// Shared FileReader implementation for delimited text-like formats in FileScannerV2. +// +// CSV and Hive text have different row parsing and cell serde rules, but their v2 FileReader +// control flow is the same: expose a file-local schema from FE slot descriptors, resolve +// FileScanRequest local positions, read physical lines, materialize requested columns, apply +// file-local conjuncts, and optionally count rows by scanning. This base keeps that contract in one +// place while derived readers provide only format-specific hooks. +class DelimitedTextReader : public FileReader { +public: + ~DelimitedTextReader() override; + + Status init(RuntimeState* state) override; + Status get_schema(std::vector* file_schema) const override; + std::unique_ptr create_column_mapper( + TableColumnMapperOptions options) const override; + Status open(std::shared_ptr request) override; + Status get_block(Block* file_block, size_t* rows, bool* eof) override; + Status get_aggregate_result(const FileAggregateRequest& request, + FileAggregateResult* result) override; + Status close() override; + +protected: + struct DelimitedTextProfile { + RuntimeProfile::Counter* open_file_time = nullptr; + RuntimeProfile::Counter* create_line_reader_time = nullptr; + RuntimeProfile::Counter* read_line_time = nullptr; + RuntimeProfile::Counter* split_line_time = nullptr; + RuntimeProfile::Counter* deserialize_time = nullptr; + RuntimeProfile::Counter* conjunct_filter_time = nullptr; + RuntimeProfile::Counter* delete_conjunct_filter_time = nullptr; + RuntimeProfile::Counter* raw_lines_read = nullptr; + RuntimeProfile::Counter* rows_read_before_filter = nullptr; + RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; + RuntimeProfile::Counter* rows_filtered_by_delete_conjunct = nullptr; + RuntimeProfile::Counter* rows_returned = nullptr; + RuntimeProfile::Counter* empty_lines_read = nullptr; + RuntimeProfile::Counter* skipped_lines = nullptr; + RuntimeProfile::Counter* cells_deserialized = nullptr; + }; + + struct RequestedColumn { + LocalColumnId file_column_id = LocalColumnId::invalid(); + LocalIndex block_position; + const SlotDescriptor* slot_desc = nullptr; + DataTypeSerDeSPtr serde; + bool nullable_string_fast_path = false; + }; + + DelimitedTextReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileScanRangeParams* scan_params, + const std::vector& file_slot_descs, + TFileCompressType::type range_compress_type, + std::optional stream_load_id, std::string reader_name); + + // Initialize format-specific options after the common init path has validated scan params and + // runtime state. Implementations must fill `_value_separator`, `_line_delimiter`, + // `_file_compress_type`, `_options`, and any parser-specific state before the common schema + // construction reads column_idxs. + virtual Status _init_format_state() = 0; + // Create the decompressor used by the line reader. CSV may infer compression from the file + // format enum, while Hive text uses only the explicit compress_type. + virtual Status _create_decompressor() = 0; + // Create the physical line reader. Implementations choose plain/enclosed/binary line contexts, + // but must store the result in `_line_reader` for the common get_block/count paths. + virtual Status _create_line_reader() = 0; + // Validate one logical line before splitting. CSV enforces UTF-8 for query reads; Hive text + // deliberately accepts arbitrary bytes and uses the default OK implementation. + virtual Status _validate_line(const Slice& line); + // Split one logical line into `_split_values`. The common materialization path then resolves + // requested field ordinals against `_split_values`. + virtual void _split_line(const Slice& line) = 0; + // Deserialize a single normalized field into the requested output column using the + // format-specific serde API. + virtual Status _deserialize_one_cell(const RequestedColumn& column, IColumn* output, + Slice value) = 0; + // Let formats rewrite a raw field before serde. CSV uses this for empty_field_as_null; Hive + // text keeps the raw field because empty string and NULL are distinct unless null_format + // matches exactly. + virtual Slice _normalize_value(Slice value) const; + // Whether an empty physical line is one logical record. CSV keeps the existing default + // skip behavior, while Hive TEXTFILE treats an empty line as a record with one empty field. + virtual bool _empty_line_as_record() const; + // Whether this file can start at a non-zero split offset. Compressed delimited files cannot be + // split because the decompressor needs the stream from the beginning. + virtual bool _can_split() const; + // Let formats adjust parser metadata that was computed before the common reader removed a BOM. + virtual void _on_bom_removed(size_t bom_size); + + Status _append_null(IColumn* output); + // Match the generic nullable serde semantics exactly: a field is NULL when its raw slice is + // byte-for-byte equal to null_format. This also covers Hive tables that set + // serialization.null.format to the empty string. + bool _is_null_format(Slice value) const; + const uint8_t* _remove_bom(const uint8_t* ptr, size_t* size); + void _init_profile() override; + + const TFileScanRangeParams* _scan_params = nullptr; + std::vector _source_file_slot_descs; + std::vector _source_column_idxs; + DataTypeSerDeSPtrs _source_serdes; + std::vector _file_schema; + RuntimeState* _runtime_state = nullptr; + + std::vector _requested_columns; + std::unique_ptr _decompressor; + std::unique_ptr _line_reader; + std::vector _split_values; + DataTypeSerDe::FormatOptions _options; + + std::string _value_separator; + std::string _line_delimiter; + TFileCompressType::type _file_compress_type = TFileCompressType::UNKNOWN; + TFileCompressType::type _range_compress_type = TFileCompressType::UNKNOWN; + std::optional _stream_load_id; + int64_t _start_offset = 0; + int64_t _size = -1; + int _skip_lines = 0; + char _escape = 0; + bool _line_reader_eof = false; + bool _bom_removed = false; + // FE exposes this as an optional text-file attribute. Keep the default strict so missing thrift + // fields do not accidentally accept arbitrary bytes; CSV can still opt out through the session + // variable or TVF/file-format property `enable_text_validate_utf8=false`. + bool _enable_text_validate_utf8 = true; + DelimitedTextProfile _text_profile; + +private: + Status _build_requested_columns(const FileScanRequest& request, + std::vector* columns) const; + Status _open_file(); + Status _read_next_line(Slice* line, bool* eof); + Status _fill_columns_from_line(const Slice& line, std::vector* columns, + size_t* rows); + + std::string _reader_name; +}; + +} // namespace doris::format diff --git a/be/src/format_v2/delimited_text/hive_text_util.h b/be/src/format_v2/delimited_text/hive_text_util.h new file mode 100644 index 00000000000000..80dc19f7ec847b --- /dev/null +++ b/be/src/format_v2/delimited_text/hive_text_util.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris { + +inline bool is_hive_text_separator_escaped(const char* data, size_t separator_pos, + char escape_char) { + size_t escape_count = 0; + while (separator_pos > escape_count && data[separator_pos - escape_count - 1] == escape_char) { + ++escape_count; + } + return escape_count % 2 == 1; +} + +} // namespace doris diff --git a/be/src/format_v2/delimited_text/text_reader.cpp b/be/src/format_v2/delimited_text/text_reader.cpp new file mode 100644 index 00000000000000..02bd9cd92f8383 --- /dev/null +++ b/be/src/format_v2/delimited_text/text_reader.cpp @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/delimited_text/text_reader.h" + +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type_serde/data_type_string_serde.h" +#include "format/file_reader/new_plain_text_line_reader.h" +#include "format_v2/delimited_text/hive_text_util.h" +#include "runtime/descriptors.h" +#include "util/decompressor.h" + +namespace doris::format::text { +namespace { + +bool starts_with_at(const Slice& line, size_t pos, const std::string& needle) { + return !needle.empty() && pos + needle.size() <= line.size && + std::memcmp(line.data + pos, needle.data(), needle.size()) == 0; +} + +} // namespace + +TextReader::TextReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileScanRangeParams* scan_params, + const std::vector& file_slot_descs, + TFileCompressType::type range_compress_type, + std::optional stream_load_id) + : DelimitedTextReader(system_properties, file_description, std::move(io_ctx), profile, + scan_params, file_slot_descs, range_compress_type, + std::move(stream_load_id), "Text") {} + +TextReader::~TextReader() = default; + +Status TextReader::_init_format_state() { + _file_compress_type = + _range_compress_type != TFileCompressType::UNKNOWN + ? _range_compress_type + : (_scan_params->__isset.compress_type ? _scan_params->compress_type + : TFileCompressType::PLAIN); + + const auto& text_params = _scan_params->file_attributes.text_params; + _value_separator = text_params.column_separator; + _line_delimiter = text_params.line_delimiter; + if (text_params.__isset.escape) { + _escape = text_params.escape; + } + _options.escape_char = _escape; + _options.collection_delim = + text_params.collection_delimiter.empty() ? ',' : text_params.collection_delimiter[0]; + _options.map_key_delim = + text_params.mapkv_delimiter.empty() ? ':' : text_params.mapkv_delimiter[0]; + if (text_params.__isset.null_format) { + _options.null_format = text_params.null_format.data(); + _options.null_len = text_params.null_format.length(); + } + return Status::OK(); +} + +Status TextReader::_create_decompressor() { + return Decompressor::create_decompressor(_file_compress_type, &_decompressor); +} + +Status TextReader::_create_line_reader() { + auto text_line_reader_ctx = std::make_shared( + _line_delimiter, _line_delimiter.size(), false); + _line_reader = NewPlainTextLineReader::create_unique( + _profile, _file_reader, _decompressor.get(), std::move(text_line_reader_ctx), _size, + _start_offset); + return Status::OK(); +} + +void TextReader::_split_line(const Slice& line) { + _split_values.clear(); + if (_value_separator.size() == 1) { + _split_line_single_char(line); + } else { + _split_line_multi_char(line); + } +} + +void TextReader::_split_line_single_char(const Slice& line) { + size_t value_start = 0; + for (size_t i = 0; i < line.size; ++i) { + if (line.data[i] == _value_separator[0]) { + // Hive text lets a string escape the field separator. The backslash remains in the + // field slice so deserialize_one_cell_from_hive_text() can unescape the final value. + if (_escape != 0 && is_hive_text_separator_escaped(line.data, i, _escape)) { + continue; + } + _split_values.emplace_back(line.data + value_start, i - value_start); + value_start = i + _value_separator.size(); + } + } + _split_values.emplace_back(line.data + value_start, line.size - value_start); +} + +void TextReader::_split_line_multi_char(const Slice& line) { + size_t value_start = 0; + size_t i = 0; + while (i < line.size) { + if (starts_with_at(line, i, _value_separator)) { + if (_escape != 0 && is_hive_text_separator_escaped(line.data, i, _escape)) { + ++i; + continue; + } + _split_values.emplace_back(line.data + value_start, i - value_start); + i += _value_separator.size(); + value_start = i; + continue; + } + ++i; + } + _split_values.emplace_back(line.data + value_start, line.size - value_start); +} + +Status TextReader::_deserialize_one_cell(const RequestedColumn& column, IColumn* output, + Slice value) { + DORIS_CHECK(output != nullptr); + if (column.nullable_string_fast_path) { + auto& null_column = assert_cast(*output); + if (_is_null_format(value)) { + null_column.insert_data(nullptr, 0); + return Status::OK(); + } + static DataTypeStringSerDe string_serde(TYPE_STRING); + auto status = string_serde.deserialize_one_cell_from_hive_text( + null_column.get_nested_column(), value, _options); + if (!status.ok()) { + null_column.insert_data(nullptr, 0); + return Status::OK(); + } + null_column.get_null_map_data().push_back(0); + return Status::OK(); + } + return column.serde->deserialize_one_cell_from_hive_text(*output, value, _options); +} + +bool TextReader::_empty_line_as_record() const { + // Hive TEXTFILE treats an empty physical line as a record. The splitter maps it + // to one empty field and missing trailing fields are filled with null_format. + return true; +} + +} // namespace doris::format::text diff --git a/be/src/format_v2/delimited_text/text_reader.h b/be/src/format_v2/delimited_text/text_reader.h new file mode 100644 index 00000000000000..8efbfe359c7e64 --- /dev/null +++ b/be/src/format_v2/delimited_text/text_reader.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "format_v2/delimited_text/delimited_text_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "util/slice.h" + +namespace doris { +class SlotDescriptor; +} // namespace doris + +namespace doris::format::text { + +// FileScannerV2 Hive text reader. +// +// Text files do not have embedded schema, so FE-provided file slots and column_idxs are converted +// into a file-local schema in the same way as CSV v2. The row parser is intentionally different +// from CSV: field splitting follows Hive text escaping rules and cells are deserialized through +// deserialize_one_cell_from_hive_text(). +class TextReader final : public ::doris::format::DelimitedTextReader { +public: + TextReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileScanRangeParams* scan_params, + const std::vector& file_slot_descs, + TFileCompressType::type range_compress_type = TFileCompressType::UNKNOWN, + std::optional stream_load_id = std::nullopt); + ~TextReader() override; + +private: + Status _init_format_state() override; + Status _create_decompressor() override; + Status _create_line_reader() override; + void _split_line(const Slice& line) override; + void _split_line_single_char(const Slice& line); + void _split_line_multi_char(const Slice& line); + Status _deserialize_one_cell(const RequestedColumn& column, IColumn* output, + Slice value) override; + bool _empty_line_as_record() const override; +}; + +} // namespace doris::format::text diff --git a/be/src/format_v2/expr/cast.cpp b/be/src/format_v2/expr/cast.cpp new file mode 100644 index 00000000000000..5dd8de557c062b --- /dev/null +++ b/be/src/format_v2/expr/cast.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/expr/cast.h" + +#include +#include +#include + +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "core/block/column_with_type_and_name.h" +#include "core/block/columns_with_type_and_name.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" + +namespace doris::format { + +Status Cast::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { + RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); + if (_children.size() != 1) { + return Status::InternalError( + fmt::format("Cast should have exactly 1 child expr, but got {}", _children.size())); + } + ColumnsWithTypeAndName argument_template; + argument_template.reserve(_children.size()); + if (_children[0]->is_literal()) { + // For some functions, he needs some literal columns to derive the return type. + auto literal_node = std::dynamic_pointer_cast(_children[0]); + argument_template.emplace_back(literal_node->get_column_ptr(), _children[0]->data_type(), + _children[0]->expr_name()); + } else { + argument_template.emplace_back(nullptr, _children[0]->data_type(), + _children[0]->expr_name()); + } + + _expr_name = fmt::format("CAST(arguments={},return={})", _children[0]->data_type()->get_name(), + _data_type->get_name()); + // get the function. won't prepare function. + _function = SimpleFunctionFactory::instance().get_function( + "CAST", argument_template, _data_type, + {.new_version_unix_timestamp = state->query_options().new_version_unix_timestamp}, + state->be_exec_version()); + if (_function == nullptr) { + return Status::InternalError("Could not find function {} ", _expr_name); + } + VExpr::register_function_context(state, context); + _prepare_finished = true; + return Status::OK(); +} + +Status Cast::open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) { + DCHECK(_prepare_finished); + for (auto& i : _children) { + RETURN_IF_ERROR(i->open(state, context, scope)); + } + RETURN_IF_ERROR(VExpr::init_function_context(state, context, scope, _function)); + if (scope == FunctionContext::FRAGMENT_LOCAL) { + RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr)); + } + _open_finished = true; + return Status::OK(); +} + +void Cast::close(VExprContext* context, FunctionContext::FunctionStateScope scope) { + VExpr::close_function_context(context, scope, _function); + VExpr::close(context, scope); +} + +Status Cast::execute_column_impl(VExprContext* context, const Block* block, + const Selector* selector, size_t count, + ColumnPtr& result_column) const { + return _do_execute(context, block, selector, count, result_column); +} + +std::string Cast::debug_string() const { + return _expr_name; +} + +Status Cast::_do_execute(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const { + DCHECK(_open_finished || block == nullptr) << debug_string(); + if (_children.size() != 1) { + return Status::InternalError( + fmt::format("Cast should have exactly 1 child expr, but got {}", _children.size())); + } + if (is_const_and_have_executed()) { // const have executed in open function + result_column = get_result_from_const(count); + return Status::OK(); + } + + Block temp_block; + ColumnNumbers args(1); + + ColumnPtr tmp_arg_column; + RETURN_IF_ERROR(_children[0]->execute_column(context, block, const_cast(selector), + count, tmp_arg_column)); + auto arg_type = _children[0]->execute_type(block); + temp_block.insert({tmp_arg_column, arg_type, _children[0]->expr_name()}); + args[0] = 0; + + uint32_t num_columns_without_result = temp_block.columns(); + // prepare a column to save result + temp_block.insert({nullptr, _data_type, _expr_name}); + + RETURN_IF_ERROR(_function->execute(context->fn_context(_fn_context_index), temp_block, args, + num_columns_without_result, count)); + result_column = temp_block.get_by_position(num_columns_without_result).column; + DCHECK_EQ(result_column->size(), count); + return Status::OK(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/expr/cast.h b/be/src/format_v2/expr/cast.h new file mode 100644 index 00000000000000..1dc06bcf07f2bc --- /dev/null +++ b/be/src/format_v2/expr/cast.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/object_pool.h" +#include "common/status.h" +#include "exprs/function_context.h" +#include "exprs/vexpr.h" + +namespace doris { +class RowDescriptor; +class RuntimeState; +class TExprNode; +class Block; +class VExprContext; +} // namespace doris + +namespace doris::format { + +class Cast final : public VExpr { + ENABLE_FACTORY_CREATOR(Cast); + +public: + Cast(const DataTypePtr& type) { + _node_type = TExprNodeType::CAST_EXPR; + _opcode = TExprOpcode::CAST; + _data_type = type; + } + ~Cast() override = default; + Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; + Status open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) override; + void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override; + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override; + std::string debug_string() const override; + uint64_t get_digest(uint64_t seed) const override { return 0; } + const std::string& expr_name() const override { return _expr_name; } + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = Cast::create_shared(_data_type); + return Status::OK(); + } + +private: + Status _do_execute(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const; + std::string _expr_name; + FunctionBasePtr _function; +}; +} // namespace doris::format diff --git a/be/src/format_v2/expr/delete_predicate.cpp b/be/src/format_v2/expr/delete_predicate.cpp new file mode 100644 index 00000000000000..772305290de73c --- /dev/null +++ b/be/src/format_v2/expr/delete_predicate.cpp @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/expr/delete_predicate.h" + +#include +#include +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/block/block.h" +#include "core/block/column_numbers.h" +#include "core/block/column_with_type_and_name.h" +#include "core/block/columns_with_type_and_name.h" + +namespace doris::format { + +DeletePredicate::DeletePredicate(const std::vector& deleted_rows) + : VExpr(), _deleted_rows(&deleted_rows) { + _node_type = TExprNodeType::PREDICATE; + _opcode = TExprOpcode::DELETE; + _data_type = std::make_shared(); +} + +DeletePredicate::DeletePredicate(const roaring::Roaring64Map& deletion_vector) + : VExpr(), _deletion_vector(&deletion_vector) { + _node_type = TExprNodeType::PREDICATE; + _opcode = TExprOpcode::DELETE; + _data_type = std::make_shared(); +} + +Status DeletePredicate::prepare(RuntimeState* state, const RowDescriptor& desc, + VExprContext* context) { + RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); + _expr_name = "DeletePredicate"; + _prepare_finished = true; + return Status::OK(); +} + +Status DeletePredicate::open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) { + DCHECK(_prepare_finished); + RETURN_IF_ERROR_OR_PREPARED(VExpr::open(state, context, scope)); + _open_finished = true; + return Status::OK(); +} + +void DeletePredicate::close(VExprContext* context, FunctionContext::FunctionStateScope scope) { + VExpr::close(context, scope); +} + +/** + * DeletePredicate is derived from 2 cases: + * 1. All row IDs indicates deleted rows. (e.g. Delete rows with row_id in (1, 2, 3)) + * 2. Bit vector indicates whether each row is deleted or not. (e.g. Bit vector[0,1,0,0,1] indicates row 1 and row 4 are deleted) + * + * So DeletePredicate should have exactly 1 child expr, which is the slot of row id. + * Row IDs should be generated by file reader as a virtual column in `block`. + **/ +Status DeletePredicate::execute(VExprContext* context, Block* block, int* result_column_id) const { + if (_children.size() != 1) { + return Status::InternalError(fmt::format( + "DeletePredicate should have exactly 1 child expr, but got {}", _children.size())); + } + int slot = -1; + RETURN_IF_ERROR(_children[0]->execute(context, block, &slot)); + if (slot < 0 || static_cast(slot) >= block->columns()) { + return Status::InternalError( + "DeletePredicate row id child returned invalid column id {}, block has {} columns", + slot, block->columns()); + } + const auto& row_ids = + assert_cast(*block->get_by_position(slot).column).get_data(); + const auto count = row_ids.size(); + auto res_col = ColumnBool::create(count, 0); + if ((_deleted_rows == nullptr || _deleted_rows->empty()) && + (_deletion_vector == nullptr || _deletion_vector->isEmpty())) { + block->insert({std::move(res_col), std::make_shared(), expr_name()}); + *result_column_id = static_cast(block->get_columns().size() - 1); + return Status::OK(); + } + if (count == 0) { + block->insert({std::move(res_col), std::make_shared(), expr_name()}); + *result_column_id = static_cast(block->get_columns().size() - 1); + return Status::OK(); + } + if (_deletion_vector != nullptr) { + auto it = _deletion_vector->begin(); + it.move(cast_set(row_ids[0])); + const auto end = _deletion_vector->end(); + const auto last_row_id = cast_set(row_ids[count - 1]); + while (it != end && *it <= last_row_id) { + const auto row = cast_set(*it); + if (const auto row_it = std::ranges::lower_bound(row_ids, row); + row_it != row_ids.end() && *row_it == row) { + res_col->get_data()[row_it - row_ids.begin()] = true; + } + ++it; + } + block->insert({std::move(res_col), std::make_shared(), expr_name()}); + *result_column_id = static_cast(block->get_columns().size() - 1); + return Status::OK(); + } + + const int64_t* delete_rows = _deleted_rows->data(); + const int64_t* delete_rows_end = delete_rows + _deleted_rows->size(); + const int64_t* start_pos = std::lower_bound(delete_rows, delete_rows_end, row_ids[0]); + int64_t start_index = start_pos - delete_rows; + const int64_t* end_pos = std::upper_bound(start_pos, delete_rows_end, row_ids[count - 1]); + const int64_t end_index = end_pos - delete_rows; + + while (start_index < end_index) { + int64_t delete_row = delete_rows[start_index]; + if (const auto it = std::ranges::lower_bound(row_ids, delete_row); + it != row_ids.end() && *it == delete_row) { + const size_t index = it - row_ids.begin(); + res_col->get_data()[index] = true; + } + ++start_index; + } + block->insert({std::move(res_col), std::make_shared(), expr_name()}); + *result_column_id = static_cast(block->get_columns().size() - 1); + return Status::OK(); +} + +std::string DeletePredicate::debug_string() const { + return _expr_name; +} + +} // namespace doris::format diff --git a/be/src/format_v2/expr/delete_predicate.h b/be/src/format_v2/expr/delete_predicate.h new file mode 100644 index 00000000000000..85bde274d51045 --- /dev/null +++ b/be/src/format_v2/expr/delete_predicate.h @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/object_pool.h" +#include "common/status.h" +#include "exprs/function_context.h" +#include "exprs/vexpr.h" +#include "roaring/roaring64map.hh" + +namespace doris { +class RowDescriptor; +class RuntimeState; +class TExprNode; +class Block; +class VExprContext; +} // namespace doris + +namespace doris::format { + +class DeletePredicate final : public VExpr { + ENABLE_FACTORY_CREATOR(DeletePredicate); + +public: + DeletePredicate(const std::vector& deleted_rows); + DeletePredicate(const roaring::Roaring64Map& deletion_vector); + ~DeletePredicate() override = default; + Status execute(VExprContext* context, Block* block, int* result_column_id) const override; + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + return Status::InternalError("Not implement DeletePredicate::execute_column_impl"); + } + Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; + Status open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) override; + void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override; + std::string debug_string() const override; + uint64_t get_digest(uint64_t seed) const override { return 0; } + const std::string& expr_name() const override { return _expr_name; } + +private: + std::string _expr_name; + const std::vector* _deleted_rows = nullptr; + const roaring::Roaring64Map* _deletion_vector = nullptr; +}; +} // namespace doris::format diff --git a/be/src/format_v2/expr/equality_delete_predicate.cpp b/be/src/format_v2/expr/equality_delete_predicate.cpp new file mode 100644 index 00000000000000..42e50e7ba07364 --- /dev/null +++ b/be/src/format_v2/expr/equality_delete_predicate.cpp @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/expr/equality_delete_predicate.h" + +#include + +#include +#include + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/column_with_type_and_name.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" + +namespace doris::format { +namespace { + +bool column_value_equal(const ColumnPtr& lhs, size_t lhs_row, const ColumnPtr& rhs, + size_t rhs_row) { + if (lhs->is_nullable() && rhs->is_nullable()) { + return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; + } + if (lhs->is_nullable()) { + const auto& nullable_lhs = assert_cast(*lhs); + return !nullable_lhs.is_null_at(lhs_row) && + nullable_lhs.get_nested_column().compare_at(lhs_row, rhs_row, *rhs, -1) == 0; + } + if (rhs->is_nullable()) { + const auto& nullable_rhs = assert_cast(*rhs); + return !nullable_rhs.is_null_at(rhs_row) && + lhs->compare_at(lhs_row, rhs_row, nullable_rhs.get_nested_column(), -1) == 0; + } + return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; +} + +} // namespace + +EqualityDeletePredicate::EqualityDeletePredicate(Block delete_block, std::vector field_ids) + : VExpr(), _delete_block(std::move(delete_block)), _field_ids(std::move(field_ids)) { + _node_type = TExprNodeType::PREDICATE; + _opcode = TExprOpcode::DELETE; + _data_type = std::make_shared(); + _expr_name = "EqualityDeletePredicate"; + DCHECK_EQ(_delete_block.columns(), _field_ids.size()); + _delete_hashes = _build_hashes(_delete_block); + for (size_t row = 0; row < _delete_hashes.size(); ++row) { + _delete_hash_map.emplace(_delete_hashes[row], row); + } +} + +Status EqualityDeletePredicate::prepare(RuntimeState* state, const RowDescriptor& desc, + VExprContext* context) { + RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); + _expr_name = "EqualityDeletePredicate"; + _prepare_finished = true; + return Status::OK(); +} + +Status EqualityDeletePredicate::open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) { + DCHECK(_prepare_finished); + for (auto& child : _children) { + RETURN_IF_ERROR(child->open(state, context, scope)); + } + if (scope == FunctionContext::FRAGMENT_LOCAL) { + RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr)); + } + _open_finished = true; + return Status::OK(); +} + +void EqualityDeletePredicate::close(VExprContext* context, + FunctionContext::FunctionStateScope scope) { + VExpr::close(context, scope); +} + +Status EqualityDeletePredicate::execute(VExprContext* context, Block* block, + int* result_column_id) const { + size_t rows = 0; + for (const auto& column : block->get_columns()) { + rows = std::max(rows, column->size()); + } + // Lazy readers may leave an unread projected column at block position zero while predicate + // columns (or the all-missing-key row-count carrier) are populated later. Block::rows() only + // inspects the first column, so derive the explicit expression count from all materialized + // columns and let execute_column() enforce that every result has exactly this batch size. + ColumnPtr result_column; + RETURN_IF_ERROR(execute_column(context, block, nullptr, rows, result_column)); + block->insert({std::move(result_column), std::make_shared(), expr_name()}); + *result_column_id = static_cast(block->columns() - 1); + return Status::OK(); +} + +Status EqualityDeletePredicate::execute_column_impl(VExprContext* context, const Block* block, + const Selector* selector, size_t count, + ColumnPtr& result_column) const { + if (_children.size() != _field_ids.size()) { + return Status::InternalError( + "EqualityDeletePredicate should have {} child exprs, but got {}", _field_ids.size(), + _children.size()); + } + + // This path is required when every equality key is a literal, notably when all key columns are + // absent from an older Iceberg data file and are represented by typed NULL literals. Preserve + // `count` so the constant result can be expanded to the caller's batch size when necessary. + Block data_key_block; + for (const auto& child : _children) { + ColumnPtr key_column; + RETURN_IF_ERROR(child->execute_column_impl(context, block, selector, count, key_column)); + // Equality comparison operates on row-addressable columns. Materialize literal constants + // so nullable NULL keys and regular slot columns share the same compare_at contract. + data_key_block.insert({key_column->convert_to_full_column_if_const(), + child->execute_type(block), child->expr_name()}); + } + result_column = _evaluate_key_block(data_key_block); + return Status::OK(); +} + +ColumnPtr EqualityDeletePredicate::_evaluate_key_block(const Block& data_key_block) const { + const auto rows = data_key_block.rows(); + auto res_col = ColumnBool::create(rows, 0); + if (_delete_hash_map.empty() || rows == 0) { + return res_col; + } + auto data_hashes = _build_hashes(data_key_block); + auto& result_data = res_col->get_data(); + for (size_t row = 0; row < rows; ++row) { + const auto range = _delete_hash_map.equal_range(data_hashes[row]); + for (auto it = range.first; it != range.second; ++it) { + if (_equal(data_key_block, row, it->second)) { + result_data[row] = true; + break; + } + } + } + return res_col; +} + +std::vector EqualityDeletePredicate::_build_hashes(const Block& block) { + std::vector hashes(block.rows(), 0); + for (const auto& column : block.get_columns()) { + column->update_hashes_with_value(hashes.data(), nullptr); + } + return hashes; +} + +bool EqualityDeletePredicate::_equal(const Block& data_block, size_t data_row, + size_t delete_row) const { + for (size_t column_idx = 0; column_idx < _delete_block.columns(); ++column_idx) { + const auto& data_column = data_block.get_by_position(column_idx).column; + const auto& delete_column = _delete_block.get_by_position(column_idx).column; + if (!column_value_equal(data_column, data_row, delete_column, delete_row)) { + return false; + } + } + return true; +} + +std::string EqualityDeletePredicate::debug_string() const { + return _expr_name; +} + +} // namespace doris::format diff --git a/be/src/format_v2/expr/equality_delete_predicate.h b/be/src/format_v2/expr/equality_delete_predicate.h new file mode 100644 index 00000000000000..0e6f127cee23c1 --- /dev/null +++ b/be/src/format_v2/expr/equality_delete_predicate.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exprs/function_context.h" +#include "exprs/vexpr.h" + +namespace doris { +class RowDescriptor; +class RuntimeState; +class VExprContext; +} // namespace doris + +namespace doris::format { + +class EqualityDeletePredicate final : public VExpr { + ENABLE_FACTORY_CREATOR(EqualityDeletePredicate); + +public: + EqualityDeletePredicate(Block delete_block, std::vector field_ids); + ~EqualityDeletePredicate() override = default; + + Status execute(VExprContext* context, Block* block, int* result_column_id) const override; + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override; + Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; + Status open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) override; + void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override; + std::string debug_string() const override; + uint64_t get_digest(uint64_t seed) const override { return 0; } + const std::string& expr_name() const override { return _expr_name; } + +private: + static std::vector _build_hashes(const Block& block); + ColumnPtr _evaluate_key_block(const Block& data_key_block) const; + bool _equal(const Block& data_block, size_t data_row, size_t delete_row) const; + + std::string _expr_name; + Block _delete_block; + std::vector _field_ids; + std::vector _delete_hashes; + std::multimap _delete_hash_map; +}; + +} // namespace doris::format diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp new file mode 100644 index 00000000000000..4f5b247c791efd --- /dev/null +++ b/be/src/format_v2/file_reader.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/file_reader.h" + +#include + +#include "format_v2/column_mapper.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/runtime_state.h" + +namespace doris::format { +namespace { + +template +std::string join_debug_strings(const std::vector& values, Formatter formatter) { + std::ostringstream out; + out << "["; + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) { + out << ", "; + } + out << formatter(values[i]); + } + out << "]"; + return out.str(); +} + +} // namespace + +std::string FileScanRequest::debug_string() const { + std::ostringstream out; + out << "FileScanRequest{predicate_columns=" + << join_debug_strings( + predicate_columns, + [](const LocalColumnIndex& projection) { return projection.debug_string(); }) + << ", non_predicate_columns=" + << join_debug_strings( + non_predicate_columns, + [](const LocalColumnIndex& projection) { return projection.debug_string(); }) + << ", local_positions={"; + size_t position_idx = 0; + for (const auto& [column_id, block_position] : local_positions) { + if (position_idx++ > 0) { + out << ", "; + } + out << column_id << ":" << block_position; + } + out << "}, conjunct_count=" << conjuncts.size() + << ", delete_conjunct_count=" << delete_conjuncts.size() << "}"; + return out.str(); +} + +Status FileReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_RAW_TIMER(&_reader_statistics.file_reader_create_time); + ++_reader_statistics.open_file_num; + io::FileReaderOptions reader_options = + FileFactory::get_reader_options(state->query_options(), *_file_description); + _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( + _profile, *_system_properties, *_file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, _io_ctx)); + // IOContext can be present without file_reader_stats in standalone tests or callers that only + // need extra IO state. TracingFileReader dereferences the stats pointer on every read, so only + // wrap the physical reader when stats collection is actually available. + _tracing_file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared( + _file_reader, _io_ctx->file_reader_stats) + : _file_reader; + _eof = false; + return Status::OK(); +} + +std::unique_ptr FileReader::create_column_mapper( + TableColumnMapperOptions options) const { + return std::make_unique(std::move(options)); +} + +} // namespace doris::format diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h new file mode 100644 index 00000000000000..59f684121ce1e3 --- /dev/null +++ b/be/src/format_v2/file_reader.h @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/field.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/column_data.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/file_factory.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris { +class Block; +struct ConditionCacheContext; + +namespace io { +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format { + +class TableColumnMapper; +struct TableColumnMapperOptions; + +enum class FileFormat { + PARQUET, + ORC, + CSV, + JSON, + TEXT, + JNI, + NATIVE, + ARROW, +}; + +struct FileScanRequest { + virtual ~FileScanRequest() = default; + + std::string debug_string() const; + + // Columns that must be read before row-level filtering. They are materialized eagerly because + // conjuncts/delete_conjuncts need them to decide the selected rows. + std::vector predicate_columns; + // Columns read after row-level filtering. Predicate columns are also available for output and + // should not be duplicated here. + std::vector non_predicate_columns; + // file-local column id -> file-local output block position. + std::map local_positions; + // Row-level filters converted to file-local expressions from table-level predicates. + VExprContextSPtrs conjuncts; + // Delete predicates converted to file-local expressions. A TRUE result means that the row is + // deleted, so readers must invert each result when building their keep filter. + VExprContextSPtrs delete_conjuncts; +}; + +// Helper for constructing the scan-column layout in FileScanRequest. +// FileScanRequest keeps predicate and non-predicate columns separate because columnar readers such +// as Parquet can read predicate columns first, filter rows, and then lazily read the remaining +// projected columns. The two lists still share one file-local output block, whose positions are +// stored in local_positions. This builder centralizes the mechanical rules for that shared layout: +// - each root file column gets one stable block position; +// - predicate columns dominate non-predicate columns because they are already returned in the file +// block and can be reused for final materialization; +// - repeated nested projections for the same root are merged instead of duplicated. +// TableColumnMapper should still own table-to-file semantic resolution. This helper only owns the +// FileScanRequest layout contract after a file-local projection has been produced. +class FileScanRequestBuilder { +public: + explicit FileScanRequestBuilder(FileScanRequest* request) : _request(request) { + DORIS_CHECK(_request != nullptr); + } + + Status add_predicate_column(LocalColumnIndex projection) { + return _add_column(std::move(projection), &_request->predicate_columns, + /*is_predicate_column=*/true); + } + + Status add_non_predicate_column(LocalColumnIndex projection) { + return _add_column(std::move(projection), &_request->non_predicate_columns, + /*is_predicate_column=*/false); + } + + Status add_predicate_column(LocalColumnId column_id) { + return add_predicate_column(LocalColumnIndex::top_level(column_id)); + } + + Status add_non_predicate_column(LocalColumnId column_id) { + return add_non_predicate_column(LocalColumnIndex::top_level(column_id)); + } + +private: + static LocalIndex _next_block_position(const FileScanRequest& request) { + size_t next_position = 0; + for (const auto& [_, block_position] : request.local_positions) { + next_position = std::max(next_position, block_position.value() + 1); + } + return LocalIndex(next_position); + } + + static void _sort_projection_children_by_file_id(LocalColumnIndex* projection) { + DORIS_CHECK(projection != nullptr); + if (projection->project_all_children) { + return; + } + for (auto& child : projection->children) { + _sort_projection_children_by_file_id(&child); + } + std::ranges::sort(projection->children, + [](const LocalColumnIndex& lhs, const LocalColumnIndex& rhs) { + return lhs.local_id() < rhs.local_id(); + }); + } + + Status _add_column(LocalColumnIndex projection, std::vector* scan_columns, + bool is_predicate_column) { + DORIS_CHECK(scan_columns != nullptr); + const auto file_column_id = projection.column_id(); + DORIS_CHECK(file_column_id != LocalColumnId::invalid()); + if (!is_predicate_column && + std::ranges::find_if(_request->predicate_columns, [&](const LocalColumnIndex& p) { + return p.column_id() == file_column_id; + }) != _request->predicate_columns.end()) { + return Status::OK(); + } + if (!_request->local_positions.contains(file_column_id)) { + _request->local_positions.emplace(file_column_id, _next_block_position(*_request)); + } + + _sort_projection_children_by_file_id(&projection); + auto existing_projection_it = std::ranges::find_if( + *scan_columns, + [&](const LocalColumnIndex& p) { return p.column_id() == file_column_id; }); + if (existing_projection_it == scan_columns->end()) { + scan_columns->push_back(std::move(projection)); + } else { + RETURN_IF_ERROR(merge_local_column_index(&*existing_projection_it, projection)); + _sort_projection_children_by_file_id(&*existing_projection_it); + } + + if (is_predicate_column) { + auto it = std::ranges::find_if( + _request->non_predicate_columns, + [&](const LocalColumnIndex& p) { return p.column_id() == file_column_id; }); + if (it != _request->non_predicate_columns.end()) { + _request->non_predicate_columns.erase(it); + } + } + return Status::OK(); + } + + FileScanRequest* _request = nullptr; +}; + +struct FileAggregateRequest { + struct Column { + // File-local projection for the aggregate column. For nested MIN/MAX, this points to the + // single primitive leaf that can be represented by file statistics. For COUNT(col), this + // points to the top-level column whose NULL-ness should be counted. + LocalColumnIndex projection; + }; + + TPushAggOp::type agg_type = TPushAggOp::type::NONE; + // Empty for COUNT(*)/row-count pushdown. Non-empty for COUNT(col), where the file reader must + // return the number of non-NULL rows for the requested column instead of total rows. + std::vector columns; +}; + +struct FileAggregateResult { + struct Column { + // Mirrors FileAggregateRequest::Column::projection so TableReader can put the returned + // aggregate value back into the matching projected nested shape. + LocalColumnIndex projection; + bool has_min = false; + bool has_max = false; + Field min_value; + Field max_value; + }; + + int64_t count = 0; + std::vector columns; +}; + +/** + * +-----> get_schema() -----------------+ + * FileReader() -----> init() ----| -----> close() + * +-----> open() -----> get_block() ----+ + */ +class FileReader { +public: + struct ReaderStatistics { + int32_t filtered_row_groups = 0; + int32_t filtered_row_groups_by_min_max = 0; + int32_t filtered_row_groups_by_bloom_filter = 0; + int32_t read_row_groups = 0; + int64_t filtered_group_rows = 0; + int64_t filtered_page_rows = 0; + int64_t lazy_read_filtered_rows = 0; + int64_t read_rows = 0; + int64_t filtered_bytes = 0; + int64_t column_read_time = 0; + int64_t parse_meta_time = 0; + int64_t parse_footer_time = 0; + int64_t file_footer_read_calls = 0; + int64_t file_footer_hit_cache = 0; + int64_t file_reader_create_time = 0; + int64_t open_file_num = 0; + int64_t row_group_filter_time = 0; + int64_t page_index_filter_time = 0; + int64_t read_page_index_time = 0; + int64_t parse_page_index_time = 0; + int64_t predicate_filter_time = 0; + int64_t dict_filter_rewrite_time = 0; + int64_t bloom_filter_read_time = 0; + }; + + FileReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile) + : _system_properties(system_properties), + _file_description(std::move(file_description)), + _io_ctx(io_ctx), + _profile(profile) {} + virtual ~FileReader() = default; + + // Initialize file reader and parse file metadata. + virtual Status init(RuntimeState* state); + + // Set the maximum row count for the next physical read batch. Readers that do not batch by + // rows may ignore it. + virtual void set_batch_size(size_t batch_size) { (void)batch_size; } + + // Get semantic file-local schema from file metadata. The file schema is determined by file + // format and file content, and does not contain table/global schema semantics. A file reader may + // expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier, + // but it must not interpret table-format semantics such as Iceberg name mapping, + // default/generated columns, or partition columns. File-format physical wrappers should be + // normalized away before exposing this schema; for example, Parquet MAP is exposed as key/value + // children rather than key_value/entry. + // Doris plans external-table scan types as nullable, including all nested children of complex + // types. This protects Doris from illegal or inconsistent values produced by external systems. + // Therefore every ColumnDefinition::type returned here must be nullable. Complex types must + // also expose nullable child types recursively, even if the physical file marks those fields as + // required. + // This method can only be called after init() successfully, but does not require open() to be + // called. + virtual Status get_schema(std::vector* file_schema) const = 0; + + // Create the mapper that matches this reader's scan-request capabilities. TableReader still + // owns table-format semantics such as BY_NAME/BY_FIELD_ID/BY_INDEX, partition values and + // default expressions; the FileReader only chooses whether file-local requests support columnar + // lazy materialization/pruning or must materialize one flat list of required columns. + virtual std::unique_ptr create_column_mapper( + TableColumnMapperOptions options) const; + + // Open the file reader with file-local scan request. The file reader should initialize its internal state according to the request, but does not need to interpret table/global schema semantics. For example, all schema change, filter localization, default/generated/partition columns should be handled in table reader layer. This method can only be called after init() successfully. + virtual Status open(std::shared_ptr request) { + _request = std::move(request); + return Status::OK(); + } + + virtual Status get_block(Block* file_block, size_t* rows, bool* eof) { + if (rows != nullptr) { + *rows = 0; + } + if (eof != nullptr) { + *eof = true; + } + _eof = true; + return Status::OK(); + } + + virtual Status get_aggregate_result(const FileAggregateRequest& request, + FileAggregateResult* result) { + return Status::NotSupported("FileReader does not support aggregate pushdown"); + } + + // Condition cache is managed by TableReader and consumed by physical file readers. + // On cache HIT, readers may skip granules whose cached bit is false before doing column IO. + // On cache MISS, readers mark a granule true when row-level predicates keep at least one row + // in that granule. Readers that cannot map batch rows to stable file-global row ids should + // keep the default no-op implementation. + virtual void set_condition_cache_context(std::shared_ptr ctx) {} + + // Total rows covered by this physical reader. TableReader uses it to pre-size the miss bitmap. + // Readers should return 0 if the metadata is unavailable or the row coordinate is unstable. + virtual int64_t get_total_rows() const { return 0; } + + virtual Status close() { + _file_reader.reset(); + _tracing_file_reader.reset(); + _io_ctx.reset(); + _eof = true; + return Status::OK(); + } + +protected: + virtual void _init_profile() {} + void _record_scan_rows(int64_t rows) { + DORIS_CHECK(rows >= 0); + _reader_statistics.read_rows += rows; + if (_io_ctx != nullptr && _io_ctx->file_reader_stats != nullptr) { + _io_ctx->file_reader_stats->read_rows += cast_set(rows); + } + } + + io::FileReaderSPtr _file_reader; + // _tracing_file_reader wraps _file_reader. + // _file_reader is original file reader. + // _tracing_file_reader is tracing file reader with io context. + // If io_ctx is null, _tracing_file_reader will be the same as file_reader. + io::FileReaderSPtr _tracing_file_reader = nullptr; + std::shared_ptr _request; + bool _eof = true; + ReaderStatistics _reader_statistics; + std::shared_ptr _system_properties; + std::unique_ptr _file_description; + std::shared_ptr _io_ctx; + RuntimeProfile* _profile = nullptr; +}; + +} // namespace doris::format diff --git a/be/src/format_v2/jni/hudi_jni_reader.cpp b/be/src/format_v2/jni/hudi_jni_reader.cpp new file mode 100644 index 00000000000000..3247e3c683c2de --- /dev/null +++ b/be/src/format_v2/jni/hudi_jni_reader.cpp @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/hudi_jni_reader.h" + +#include + +#include "core/block/block.h" +#include "exprs/vexpr_context.h" +#include "util/string_util.h" +#include "util/uid_util.h" + +namespace doris::format::hudi { +namespace { + +constexpr std::string_view HOODIE_CONF_PREFIX = "hoodie."; +constexpr std::string_view HADOOP_CONF_PREFIX = "hadoop_conf."; + +} // namespace + +Status HudiJniReader::validate_scan_range(const TFileRangeDesc& range) const { + if (!range.__isset.table_format_params) { + return Status::InternalError("missing table_format_params for hudi jni reader"); + } + if (!range.table_format_params.__isset.hudi_params) { + return Status::InternalError("missing hudi_params for hudi jni reader"); + } + const auto& hudi_params = range.table_format_params.hudi_params; + if (!hudi_params.__isset.base_path || hudi_params.base_path.empty()) { + return Status::InternalError( + "missing base_path for hudi jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + if (!hudi_params.__isset.data_file_path || hudi_params.data_file_path.empty()) { + return Status::InternalError( + "missing data_file_path for hudi jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + if (!hudi_params.__isset.data_file_length) { + return Status::InternalError( + "missing data_file_length for hudi jni reader, possibly caused by FE/BE " + "protocol mismatch"); + } + if (!hudi_params.__isset.column_names) { + return Status::InternalError( + "missing column_names for hudi jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + if (!hudi_params.__isset.column_types) { + return Status::InternalError( + "missing column_types for hudi jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + DORIS_CHECK(hudi_params.column_names.size() == hudi_params.column_types.size()); + if (_scan_params == nullptr) { + return Status::InternalError( + "missing scan params for hudi jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + return Status::OK(); +} + +std::string HudiJniReader::connector_class() const { + return "org/apache/doris/hudi/HadoopHudiJniScanner"; +} + +Status HudiJniReader::build_scanner_params(std::map* params) const { + DORIS_CHECK(params != nullptr); + DORIS_CHECK(_scan_params != nullptr); + params->clear(); + + const auto& hudi_params = _current_range.table_format_params.hudi_params; + (*params)["base_path"] = hudi_params.base_path; + (*params)["data_file_path"] = hudi_params.data_file_path; + (*params)["data_file_length"] = std::to_string(hudi_params.data_file_length); + (*params)["delta_file_paths"] = join(hudi_params.delta_logs, ","); + (*params)["hudi_column_names"] = join(hudi_params.column_names, ","); + (*params)["hudi_column_types"] = join(hudi_params.column_types, "#"); + (*params)["instant_time"] = hudi_params.instant_time; + (*params)["serde"] = hudi_params.serde; + (*params)["input_format"] = hudi_params.input_format; + if (_runtime_state != nullptr) { + (*params)["query_id"] = print_id(_runtime_state->query_id()); + } + + for (const auto& kv : _scan_params->properties) { + if (kv.first.starts_with(HOODIE_CONF_PREFIX)) { + (*params)[kv.first] = kv.second; + } else { + (*params)[std::string(HADOOP_CONF_PREFIX) + kv.first] = kv.second; + } + } + return Status::OK(); +} + +Status HudiJniReader::build_jni_columns( + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + columns->reserve(_projected_columns.size()); + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + if (table_column.is_partition_key && + find_partition_value(table_column, _partition_values) != nullptr) { + continue; + } + columns->push_back({ + .java_name = table_column.name, + .output_index = i, + .output_type = table_column.type, + .transfer_type = table_column.type, + .replace_type = "not_replace", + }); + } + return Status::OK(); +} + +Status HudiJniReader::finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) { + DORIS_CHECK(jni_block != nullptr); + DORIS_CHECK(output_block != nullptr); + DORIS_CHECK(rows != nullptr); + const auto original_rows = *rows; + + const auto& columns = jni_columns(); + DORIS_CHECK(columns.size() == jni_block->columns()); + for (size_t i = 0; i < columns.size(); ++i) { + const auto& column = columns[i]; + DORIS_CHECK(column.output_index < output_block->columns()); + output_block->get_by_position(column.output_index).type = column.output_type; + output_block->replace_by_position(column.output_index, + jni_block->get_by_position(i).column); + } + + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + const auto* partition_value = find_partition_value(table_column, _partition_values); + if (!table_column.is_partition_key || partition_value == nullptr) { + continue; + } + output_block->get_by_position(i).type = table_column.type; + output_block->replace_by_position( + i, table_column.type->create_column_const(original_rows, *partition_value)); + } + DORIS_CHECK(output_block->rows() == original_rows); + if (!_conjuncts.empty()) { + RETURN_IF_ERROR( + VExprContext::filter_block(_conjuncts, output_block, output_block->columns())); + } + *rows = output_block->rows(); + return Status::OK(); +} + +} // namespace doris::format::hudi diff --git a/be/src/format_v2/jni/hudi_jni_reader.h b/be/src/format_v2/jni/hudi_jni_reader.h new file mode 100644 index 00000000000000..4beb6f2d1728b6 --- /dev/null +++ b/be/src/format_v2/jni/hudi_jni_reader.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "format_v2/jni/jni_table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::hudi { + +class HudiJniReader final : public format::JniTableReader { +public: + ~HudiJniReader() override = default; + +protected: + std::string connector_class() const override; + Status validate_scan_range(const TFileRangeDesc& range) const override; + Status build_scanner_params(std::map* params) const override; + Status build_jni_columns( + std::vector* columns) const override; + Status finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) override; +}; + +} // namespace doris::format::hudi diff --git a/be/src/format_v2/jni/iceberg_sys_table_reader.cpp b/be/src/format_v2/jni/iceberg_sys_table_reader.cpp new file mode 100644 index 00000000000000..896929925aab3f --- /dev/null +++ b/be/src/format_v2/jni/iceberg_sys_table_reader.cpp @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/iceberg_sys_table_reader.h" + +namespace doris::format::iceberg { + +Status IcebergSysTableJniReader::validate_scan_range(const TFileRangeDesc& range) const { + return Status::NotSupported("native Iceberg system-table splits are unavailable on branch-4.1"); +} + +std::string IcebergSysTableJniReader::connector_class() const { + return "org/apache/doris/iceberg/IcebergSysTableJniScanner"; +} + +Status IcebergSysTableJniReader::build_scanner_params( + std::map* params) const { + DORIS_CHECK(params != nullptr); + params->clear(); + return Status::NotSupported("native Iceberg system-table splits are unavailable on branch-4.1"); +} + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/jni/iceberg_sys_table_reader.h b/be/src/format_v2/jni/iceberg_sys_table_reader.h new file mode 100644 index 00000000000000..be254c39f3ffb5 --- /dev/null +++ b/be/src/format_v2/jni/iceberg_sys_table_reader.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "format_v2/jni/jni_table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::iceberg { + +class IcebergSysTableJniReader final : public format::JniTableReader { +public: + ~IcebergSysTableJniReader() override = default; + +protected: + std::string connector_class() const override; + Status validate_scan_range(const TFileRangeDesc& range) const override; + Status build_scanner_params(std::map* params) const override; +}; + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/jni/jdbc_reader.cpp b/be/src/format_v2/jni/jdbc_reader.cpp new file mode 100644 index 00000000000000..e9d225aeb46fc5 --- /dev/null +++ b/be/src/format_v2/jni/jdbc_reader.cpp @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/jdbc_reader.h" + +#include +#include + +#include "common/cast_set.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/block/columns_with_type_and_name.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/vexpr_context.h" +#include "format_v2/table_reader.h" + +namespace doris::format::jdbc { + +std::string JdbcJniReader::connector_class() const { + return "org/apache/doris/jdbc/JdbcJniScanner"; +} + +Status JdbcJniReader::prepare_split(const format::SplitReadOptions& options) { + return Status::NotSupported("native JDBC file splits are unavailable on branch-4.1"); +} + +// need pass to the java side, so the java scanner can parse the params and construct the JDBC connection +Status JdbcJniReader::build_scanner_params(std::map* params) const { + DORIS_CHECK(params != nullptr); + params->clear(); + return Status::NotSupported("native JDBC file splits are unavailable on branch-4.1"); +} + +Status JdbcJniReader::build_jni_columns( + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + columns->reserve(_projected_columns.size()); + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + const auto primitive_type = remove_nullable(table_column.type)->get_primitive_type(); + columns->push_back({ + .java_name = table_column.name, + .output_index = i, + .output_type = table_column.type, + .transfer_type = _transfer_type_for(table_column.type), + .replace_type = _replace_type_for(primitive_type), + }); + } + return Status::OK(); +} + +Status JdbcJniReader::finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) { + DORIS_CHECK(jni_block != nullptr); + DORIS_CHECK(output_block != nullptr); + DORIS_CHECK(rows != nullptr); + const auto original_rows = *rows; + const auto& columns = jni_columns(); + DORIS_CHECK(columns.size() == jni_block->columns()); + + for (size_t i = 0; i < columns.size(); ++i) { + const auto& column = columns[i]; + DORIS_CHECK(column.output_type != nullptr); + DORIS_CHECK(column.output_index < output_block->columns()); + if (_is_special_type(remove_nullable(column.output_type)->get_primitive_type())) { + RETURN_IF_ERROR(_cast_string_to_special_type(column, jni_block, i, output_block, + original_rows)); + continue; + } + output_block->get_by_position(column.output_index).type = column.output_type; + output_block->replace_by_position(column.output_index, + jni_block->get_by_position(i).column); + } + DORIS_CHECK(output_block->rows() == original_rows); + if (!_conjuncts.empty()) { + RETURN_IF_ERROR( + VExprContext::filter_block(_conjuncts, output_block, output_block->columns())); + } + *rows = output_block->rows(); + return Status::OK(); +} + +std::string JdbcJniReader::_replace_type_for(PrimitiveType type) const { + switch (type) { + case PrimitiveType::TYPE_BITMAP: + return "bitmap"; + case PrimitiveType::TYPE_HLL: + return "hll"; + case PrimitiveType::TYPE_QUANTILE_STATE: + return "quantile_state"; + case PrimitiveType::TYPE_JSONB: + return "jsonb"; + default: + return "not_replace"; + } +} + +bool JdbcJniReader::_is_special_type(PrimitiveType type) const { + return type == PrimitiveType::TYPE_BITMAP || type == PrimitiveType::TYPE_HLL || + type == PrimitiveType::TYPE_QUANTILE_STATE || type == PrimitiveType::TYPE_JSONB; +} + +DataTypePtr JdbcJniReader::_transfer_type_for(const DataTypePtr& output_type) const { + DORIS_CHECK(output_type != nullptr); + if (!_is_special_type(remove_nullable(output_type)->get_primitive_type())) { + return output_type; + } + DataTypePtr string_type = std::make_shared(); + if (output_type->is_nullable()) { + string_type = make_nullable(string_type); + } + return string_type; +} + +Status JdbcJniReader::_cast_string_to_special_type(const format::JniTableReader::JniColumn& column, + Block* jni_block, size_t jni_column_index, + Block* output_block, size_t rows) { + DORIS_CHECK(column.output_type != nullptr); + DORIS_CHECK(column.transfer_type != nullptr); + const auto target_type = column.output_type; + const auto target_type_name = target_type->get_name(); + + ColumnPtr input_column = jni_block->get_by_position(jni_column_index).column; + ColumnPtr cast_param = target_type->create_column_const_with_default_value(1); + + ColumnsWithTypeAndName argument_template; + argument_template.reserve(2); + argument_template.emplace_back(std::move(input_column), column.transfer_type, + "java.sql.String"); + argument_template.emplace_back(std::move(cast_param), target_type, target_type_name); + + FunctionBasePtr cast_function = SimpleFunctionFactory::instance().get_function( + "CAST", argument_template, make_nullable(target_type)); + if (cast_function == nullptr) { + return Status::InternalError("Failed to find CAST function for type {}", target_type_name); + } + + Block cast_block(argument_template); + const auto result_idx = cast_set(cast_block.columns()); + cast_block.insert({nullptr, make_nullable(target_type), "cast_result"}); + RETURN_IF_ERROR( + cast_function->execute(nullptr, cast_block, {0}, result_idx, cast_set(rows))); + + auto result_column = cast_block.get_by_position(result_idx).column; + output_block->get_by_position(column.output_index).type = target_type; + if (target_type->is_nullable()) { + output_block->replace_by_position(column.output_index, result_column); + } else { + const auto* nullable_column = assert_cast(result_column.get()); + output_block->replace_by_position(column.output_index, + nullable_column->get_nested_column_ptr()); + } + return Status::OK(); +} + +} // namespace doris::format::jdbc diff --git a/be/src/format_v2/jni/jdbc_reader.h b/be/src/format_v2/jni/jdbc_reader.h new file mode 100644 index 00000000000000..91a5878cb4622f --- /dev/null +++ b/be/src/format_v2/jni/jdbc_reader.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/types.h" +#include "format_v2/jni/jni_table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::jdbc { + +class JdbcJniReader final : public format::JniTableReader { +public: + ~JdbcJniReader() override = default; + + Status prepare_split(const format::SplitReadOptions& options) override; + +protected: + std::string connector_class() const override; + Status build_scanner_params(std::map* params) const override; + Status build_jni_columns( + std::vector* columns) const override; + Status finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) override; + +private: + bool _is_special_type(PrimitiveType type) const; + std::string _replace_type_for(PrimitiveType type) const; + DataTypePtr _transfer_type_for(const DataTypePtr& output_type) const; + Status _cast_string_to_special_type(const format::JniTableReader::JniColumn& column, + Block* jni_block, size_t jni_column_index, + Block* output_block, size_t rows); + + std::map _jdbc_params; +}; + +} // namespace doris::format::jdbc diff --git a/be/src/format_v2/jni/jni_data_bridge.cpp b/be/src/format_v2/jni/jni_data_bridge.cpp new file mode 100644 index 00000000000000..1a88f4c317b28a --- /dev/null +++ b/be/src/format_v2/jni/jni_data_bridge.cpp @@ -0,0 +1,640 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "jni_data_bridge.h" + +#include + +#include +#include + +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type/define_primitive_type.h" +#include "core/data_type/primitive_type.h" +#include "core/types.h" +#include "core/value/decimalv2_value.h" + +namespace doris { + +#define FOR_FIXED_LENGTH_TYPES(M) \ + M(PrimitiveType::TYPE_TINYINT, ColumnInt8, Int8) \ + M(PrimitiveType::TYPE_BOOLEAN, ColumnUInt8, UInt8) \ + M(PrimitiveType::TYPE_SMALLINT, ColumnInt16, Int16) \ + M(PrimitiveType::TYPE_INT, ColumnInt32, Int32) \ + M(PrimitiveType::TYPE_BIGINT, ColumnInt64, Int64) \ + M(PrimitiveType::TYPE_LARGEINT, ColumnInt128, Int128) \ + M(PrimitiveType::TYPE_FLOAT, ColumnFloat32, Float32) \ + M(PrimitiveType::TYPE_DOUBLE, ColumnFloat64, Float64) \ + M(PrimitiveType::TYPE_DECIMALV2, ColumnDecimal128V2, Int128) \ + M(PrimitiveType::TYPE_DECIMAL128I, ColumnDecimal128V3, Int128) \ + M(PrimitiveType::TYPE_DECIMAL32, ColumnDecimal32, Int32) \ + M(PrimitiveType::TYPE_DECIMAL64, ColumnDecimal64, Int64) \ + M(PrimitiveType::TYPE_DATE, ColumnDate, Int64) \ + M(PrimitiveType::TYPE_DATEV2, ColumnDateV2, UInt32) \ + M(PrimitiveType::TYPE_DATETIME, ColumnDateTime, Int64) \ + M(PrimitiveType::TYPE_DATETIMEV2, ColumnDateTimeV2, UInt64) \ + M(PrimitiveType::TYPE_TIMESTAMPTZ, ColumnTimeStampTz, UInt64) \ + M(PrimitiveType::TYPE_IPV4, ColumnIPv4, IPv4) \ + M(PrimitiveType::TYPE_IPV6, ColumnIPv6, IPv6) + +Status JniDataBridge::fill_block(Block* block, const ColumnNumbers& arguments, long table_address) { + if (table_address == 0) { + return Status::InternalError("table_address is 0"); + } + TableMetaAddress table_meta(table_address); + long num_rows = table_meta.next_meta_as_long(); + for (size_t i : arguments) { + if (block->get_by_position(i).column.get() == nullptr) { + auto return_type = block->get_data_type(i); + bool result_nullable = return_type->is_nullable(); + ColumnUInt8::MutablePtr null_col = nullptr; + if (result_nullable) { + return_type = remove_nullable(return_type); + null_col = ColumnUInt8::create(); + } + auto res_col = return_type->create_column(); + if (result_nullable) { + block->replace_by_position( + i, ColumnNullable::create(std::move(res_col), std::move(null_col))); + } else { + block->replace_by_position(i, std::move(res_col)); + } + } else if (is_column_const(*(block->get_by_position(i).column))) { + auto doris_column = block->get_by_position(i).column->convert_to_full_column_if_const(); + bool is_nullable = block->get_by_position(i).type->is_nullable(); + block->replace_by_position(i, is_nullable ? make_nullable(doris_column) : doris_column); + } + auto& column_with_type_and_name = block->get_by_position(i); + auto& column_ptr = column_with_type_and_name.column; + auto& column_type = column_with_type_and_name.type; + RETURN_IF_ERROR(fill_column(table_meta, column_ptr, column_type, num_rows)); + } + return Status::OK(); +} + +Status JniDataBridge::fill_column(TableMetaAddress& address, ColumnPtr& doris_column, + const DataTypePtr& data_type, size_t num_rows) { + auto logical_type = data_type->get_primitive_type(); + void* null_map_ptr = address.next_meta_as_ptr(); + if (null_map_ptr == nullptr) { + // org.apache.doris.common.jni.vec.ColumnType.Type#UNSUPPORTED will set column address as 0 + return Status::InternalError("Unsupported type {} in java side", data_type->get_name()); + } + auto mutable_doris_column = IColumn::mutate(std::move(doris_column)); + MutableColumnPtr data_column; + if (auto* nullable_column = check_and_get_column(mutable_doris_column.get())) { + data_column = nullable_column->get_nested_column_ptr(); + NullMap& null_map = nullable_column->get_null_map_data(); + size_t origin_size = null_map.size(); + null_map.resize(origin_size + num_rows); + memcpy(null_map.data() + origin_size, static_cast(null_map_ptr), num_rows); + } else { + data_column = mutable_doris_column->get_ptr(); + } + // Date and DateTime are deprecated and not supported. + Status status = Status::OK(); + switch (logical_type) { +#define DISPATCH(TYPE_INDEX, COLUMN_TYPE, CPP_TYPE) \ + case TYPE_INDEX: { \ + auto* data = reinterpret_cast(address.next_meta_as_ptr()); \ + status = _fill_fixed_length_column(data_column, data, num_rows); \ + break; \ + } + FOR_FIXED_LENGTH_TYPES(DISPATCH) +#undef DISPATCH + case PrimitiveType::TYPE_STRING: + [[fallthrough]]; + case PrimitiveType::TYPE_CHAR: + [[fallthrough]]; + case PrimitiveType::TYPE_VARCHAR: + status = _fill_string_column(address, data_column, num_rows); + break; + case PrimitiveType::TYPE_ARRAY: + status = _fill_array_column(address, data_column, data_type, num_rows); + break; + case PrimitiveType::TYPE_MAP: + status = _fill_map_column(address, data_column, data_type, num_rows); + break; + case PrimitiveType::TYPE_STRUCT: + status = _fill_struct_column(address, data_column, data_type, num_rows); + break; + case PrimitiveType::TYPE_VARBINARY: + status = _fill_varbinary_column(address, data_column, num_rows); + break; + default: + status = Status::InvalidArgument("Unsupported type {} in jni scanner", + data_type->get_name()); + break; + } + doris_column = std::move(mutable_doris_column); + return status; +} + +Status JniDataBridge::_fill_varbinary_column(TableMetaAddress& address, + MutableColumnPtr& doris_column, size_t num_rows) { + auto* meta_base = reinterpret_cast(address.next_meta_as_ptr()); + auto& varbinary_col = assert_cast(*doris_column); + // Java side writes per-row metadata as 16 bytes: [len: long][addr: long] + for (size_t i = 0; i < num_rows; ++i) { + // Read length (first 8 bytes) + int64_t len = 0; + memcpy(&len, meta_base + 16 * i, sizeof(len)); + if (len <= 0) { + varbinary_col.insert_default(); + } else { + // Read address (next 8 bytes) + uint64_t addr_u = 0; + memcpy(&addr_u, meta_base + 16 * i + 8, sizeof(addr_u)); + const char* src = reinterpret_cast(addr_u); + varbinary_col.insert_data(src, static_cast(len)); + } + } + return Status::OK(); +} + +Status JniDataBridge::_fill_string_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + size_t num_rows) { + auto& string_col = static_cast(*doris_column); + ColumnString::Chars& string_chars = string_col.get_chars(); + ColumnString::Offsets& string_offsets = string_col.get_offsets(); + int* offsets = reinterpret_cast(address.next_meta_as_ptr()); + char* chars = reinterpret_cast(address.next_meta_as_ptr()); + + // This judgment is necessary, otherwise the following statement `offsets[num_rows - 1]` out of bounds + // What's more, This judgment must be placed after `address.next_meta_as_ptr()` + // because `address.next_meta_as_ptr` will make `address._meta_index` plus 1 + if (num_rows == 0) { + return Status::OK(); + } + + size_t origin_chars_size = string_chars.size(); + string_chars.resize(origin_chars_size + offsets[num_rows - 1]); + memcpy(string_chars.data() + origin_chars_size, chars, offsets[num_rows - 1]); + + size_t origin_offsets_size = string_offsets.size(); + size_t start_offset = string_offsets[origin_offsets_size - 1]; + string_offsets.resize(origin_offsets_size + num_rows); + for (size_t i = 0; i < num_rows; ++i) { + string_offsets[origin_offsets_size + i] = + static_cast(offsets[i] + start_offset); + } + return Status::OK(); +} + +Status JniDataBridge::_fill_array_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + const DataTypePtr& data_type, size_t num_rows) { + ColumnPtr& element_column = static_cast(*doris_column).get_data_ptr(); + const DataTypePtr& element_type = + (assert_cast(remove_nullable(data_type).get())) + ->get_nested_type(); + ColumnArray::Offsets64& offsets_data = static_cast(*doris_column).get_offsets(); + + int64_t* offsets = reinterpret_cast(address.next_meta_as_ptr()); + size_t origin_size = offsets_data.size(); + offsets_data.resize(origin_size + num_rows); + size_t start_offset = offsets_data[origin_size - 1]; + for (size_t i = 0; i < num_rows; ++i) { + offsets_data[origin_size + i] = offsets[i] + start_offset; + } + + return fill_column(address, element_column, element_type, + offsets_data[origin_size + num_rows - 1] - start_offset); +} + +Status JniDataBridge::_fill_map_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + const DataTypePtr& data_type, size_t num_rows) { + auto& map = static_cast(*doris_column); + const DataTypePtr& key_type = + reinterpret_cast(remove_nullable(data_type).get())->get_key_type(); + const DataTypePtr& value_type = + reinterpret_cast(remove_nullable(data_type).get()) + ->get_value_type(); + ColumnPtr& key_column = map.get_keys_ptr(); + ColumnPtr& value_column = map.get_values_ptr(); + ColumnArray::Offsets64& map_offsets = map.get_offsets(); + + int64_t* offsets = reinterpret_cast(address.next_meta_as_ptr()); + size_t origin_size = map_offsets.size(); + map_offsets.resize(origin_size + num_rows); + size_t start_offset = map_offsets[origin_size - 1]; + for (size_t i = 0; i < num_rows; ++i) { + map_offsets[origin_size + i] = offsets[i] + start_offset; + } + + RETURN_IF_ERROR(fill_column(address, key_column, key_type, + map_offsets[origin_size + num_rows - 1] - start_offset)); + RETURN_IF_ERROR(fill_column(address, value_column, value_type, + map_offsets[origin_size + num_rows - 1] - start_offset)); + return Status::OK(); +} + +Status JniDataBridge::_fill_struct_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + const DataTypePtr& data_type, size_t num_rows) { + auto& doris_struct = static_cast(*doris_column); + const DataTypeStruct* doris_struct_type = + reinterpret_cast(remove_nullable(data_type).get()); + for (int i = 0; i < doris_struct.tuple_size(); ++i) { + ColumnPtr& struct_field = doris_struct.get_column_ptr(i); + const DataTypePtr& field_type = doris_struct_type->get_element(i); + RETURN_IF_ERROR(fill_column(address, struct_field, field_type, num_rows)); + } + return Status::OK(); +} + +std::string JniDataBridge::get_jni_type(const DataTypePtr& data_type) { + DataTypePtr type = remove_nullable(data_type); + std::ostringstream buffer; + switch (type->get_primitive_type()) { + case TYPE_BOOLEAN: + return "boolean"; + case TYPE_TINYINT: + return "tinyint"; + case TYPE_SMALLINT: + return "smallint"; + case TYPE_INT: + return "int"; + case TYPE_BIGINT: + return "bigint"; + case TYPE_LARGEINT: + return "largeint"; + case TYPE_FLOAT: + return "float"; + case TYPE_DOUBLE: + return "double"; + case TYPE_IPV4: + return "ipv4"; + case TYPE_IPV6: + return "ipv6"; + case TYPE_VARCHAR: + [[fallthrough]]; + case TYPE_CHAR: + [[fallthrough]]; + case TYPE_STRING: + return "string"; + case TYPE_DATE: + return "datev1"; + case TYPE_DATEV2: + return "datev2"; + case TYPE_DATETIME: + return "datetimev1"; + case TYPE_DATETIMEV2: + [[fallthrough]]; + case TYPE_TIMEV2: { + buffer << "datetimev2(" << type->get_scale() << ")"; + return buffer.str(); + } + case TYPE_TIMESTAMPTZ: { + buffer << "timestamptz(" << type->get_scale() << ")"; + return buffer.str(); + } + case TYPE_BINARY: + return "binary"; + case TYPE_DECIMALV2: { + buffer << "decimalv2(" << DecimalV2Value::PRECISION << "," << DecimalV2Value::SCALE << ")"; + return buffer.str(); + } + case TYPE_DECIMAL32: { + buffer << "decimal32(" << type->get_precision() << "," << type->get_scale() << ")"; + return buffer.str(); + } + case TYPE_DECIMAL64: { + buffer << "decimal64(" << type->get_precision() << "," << type->get_scale() << ")"; + return buffer.str(); + } + case TYPE_DECIMAL128I: { + buffer << "decimal128(" << type->get_precision() << "," << type->get_scale() << ")"; + return buffer.str(); + } + case TYPE_STRUCT: { + const DataTypeStruct* struct_type = reinterpret_cast(type.get()); + buffer << "struct<"; + for (int i = 0; i < struct_type->get_elements().size(); ++i) { + if (i != 0) { + buffer << ","; + } + buffer << struct_type->get_element_names()[i] << ":" + << get_jni_type(struct_type->get_element(i)); + } + buffer << ">"; + return buffer.str(); + } + case TYPE_ARRAY: { + const DataTypeArray* array_type = reinterpret_cast(type.get()); + buffer << "array<" << get_jni_type(array_type->get_nested_type()) << ">"; + return buffer.str(); + } + case TYPE_MAP: { + const DataTypeMap* map_type = reinterpret_cast(type.get()); + buffer << "map<" << get_jni_type(map_type->get_key_type()) << "," + << get_jni_type(map_type->get_value_type()) << ">"; + return buffer.str(); + } + case TYPE_VARBINARY: + return "varbinary"; + // bitmap, hll, quantile_state, jsonb are transferred as strings via JNI + case TYPE_BITMAP: + [[fallthrough]]; + case TYPE_HLL: + [[fallthrough]]; + case TYPE_QUANTILE_STATE: + [[fallthrough]]; + case TYPE_JSONB: + return "string"; + default: + return "unsupported"; + } +} + +std::string JniDataBridge::get_jni_type_with_different_string(const DataTypePtr& data_type) { + DataTypePtr type = remove_nullable(data_type); + std::ostringstream buffer; + switch (data_type->get_primitive_type()) { + case TYPE_BOOLEAN: + return "boolean"; + case TYPE_TINYINT: + return "tinyint"; + case TYPE_SMALLINT: + return "smallint"; + case TYPE_INT: + return "int"; + case TYPE_BIGINT: + return "bigint"; + case TYPE_LARGEINT: + return "largeint"; + case TYPE_FLOAT: + return "float"; + case TYPE_DOUBLE: + return "double"; + case TYPE_IPV4: + return "ipv4"; + case TYPE_IPV6: + return "ipv6"; + case TYPE_VARCHAR: { + buffer << "varchar(" + << assert_cast(remove_nullable(data_type).get())->len() + << ")"; + return buffer.str(); + } + case TYPE_DATE: + return "datev1"; + case TYPE_DATEV2: + return "datev2"; + case TYPE_DATETIME: + return "datetimev1"; + case TYPE_DATETIMEV2: + [[fallthrough]]; + case TYPE_TIMEV2: { + buffer << "datetimev2(" << data_type->get_scale() << ")"; + return buffer.str(); + } + case TYPE_TIMESTAMPTZ: { + buffer << "timestamptz(" << data_type->get_scale() << ")"; + return buffer.str(); + } + case TYPE_BINARY: + return "binary"; + case TYPE_CHAR: { + buffer << "char(" + << assert_cast(remove_nullable(data_type).get())->len() + << ")"; + return buffer.str(); + } + case TYPE_STRING: + return "string"; + case TYPE_VARBINARY: + buffer << "varbinary(" + << assert_cast(remove_nullable(data_type).get())->len() + << ")"; + return buffer.str(); + case TYPE_DECIMALV2: { + buffer << "decimalv2(" << DecimalV2Value::PRECISION << "," << DecimalV2Value::SCALE << ")"; + return buffer.str(); + } + case TYPE_DECIMAL32: { + buffer << "decimal32(" << data_type->get_precision() << "," << data_type->get_scale() + << ")"; + return buffer.str(); + } + case TYPE_DECIMAL64: { + buffer << "decimal64(" << data_type->get_precision() << "," << data_type->get_scale() + << ")"; + return buffer.str(); + } + case TYPE_DECIMAL128I: { + buffer << "decimal128(" << data_type->get_precision() << "," << data_type->get_scale() + << ")"; + return buffer.str(); + } + case TYPE_STRUCT: { + const auto* type_struct = + assert_cast(remove_nullable(data_type).get()); + buffer << "struct<"; + for (int i = 0; i < type_struct->get_elements().size(); ++i) { + if (i != 0) { + buffer << ","; + } + buffer << type_struct->get_element_name(i) << ":" + << get_jni_type_with_different_string(type_struct->get_element(i)); + } + buffer << ">"; + return buffer.str(); + } + case TYPE_ARRAY: { + const auto* type_arr = assert_cast(remove_nullable(data_type).get()); + buffer << "array<" << get_jni_type_with_different_string(type_arr->get_nested_type()) + << ">"; + return buffer.str(); + } + case TYPE_MAP: { + const auto* type_map = assert_cast(remove_nullable(data_type).get()); + buffer << "map<" << get_jni_type_with_different_string(type_map->get_key_type()) << "," + << get_jni_type_with_different_string(type_map->get_value_type()) << ">"; + return buffer.str(); + } + // bitmap, hll, quantile_state, jsonb are transferred as strings via JNI + case TYPE_BITMAP: + [[fallthrough]]; + case TYPE_HLL: + [[fallthrough]]; + case TYPE_QUANTILE_STATE: + [[fallthrough]]; + case TYPE_JSONB: + return "string"; + default: + return "unsupported"; + } +} + +Status JniDataBridge::_fill_column_meta(const ColumnPtr& doris_column, const DataTypePtr& data_type, + std::vector& meta_data) { + auto logical_type = data_type->get_primitive_type(); + const IColumn* column = nullptr; + // insert const flag + if (is_column_const(*doris_column)) { + meta_data.emplace_back((long)1); + const auto& const_column = assert_cast(*doris_column); + column = &(const_column.get_data_column()); + } else { + meta_data.emplace_back((long)0); + column = &(*doris_column); + } + + // insert null map address + const IColumn* data_column = nullptr; + if (const auto* nullable_column = check_and_get_column(column)) { + data_column = &(nullable_column->get_nested_column()); + const auto& null_map = nullable_column->get_null_map_data(); + meta_data.emplace_back((long)null_map.data()); + } else { + meta_data.emplace_back(0); + data_column = column; + } + switch (logical_type) { +#define DISPATCH(TYPE_INDEX, COLUMN_TYPE, CPP_TYPE) \ + case TYPE_INDEX: { \ + meta_data.emplace_back(_get_fixed_length_column_address(*data_column)); \ + break; \ + } + FOR_FIXED_LENGTH_TYPES(DISPATCH) +#undef DISPATCH + case PrimitiveType::TYPE_STRING: + [[fallthrough]]; + case PrimitiveType::TYPE_CHAR: + [[fallthrough]]; + case PrimitiveType::TYPE_VARCHAR: { + const auto& string_column = assert_cast(*data_column); + // insert offsets + meta_data.emplace_back((long)string_column.get_offsets().data()); + meta_data.emplace_back((long)string_column.get_chars().data()); + break; + } + case PrimitiveType::TYPE_ARRAY: { + const auto& element_column = assert_cast(*data_column).get_data_ptr(); + meta_data.emplace_back( + (long)assert_cast(*data_column).get_offsets().data()); + const auto& element_type = + (assert_cast(remove_nullable(data_type).get())) + ->get_nested_type(); + RETURN_IF_ERROR(_fill_column_meta(element_column, element_type, meta_data)); + break; + } + case PrimitiveType::TYPE_STRUCT: { + const auto& doris_struct = assert_cast(*data_column); + const auto* doris_struct_type = + assert_cast(remove_nullable(data_type).get()); + for (int i = 0; i < doris_struct.tuple_size(); ++i) { + const auto& struct_field = doris_struct.get_column_ptr(i); + const auto& field_type = doris_struct_type->get_element(i); + RETURN_IF_ERROR(_fill_column_meta(struct_field, field_type, meta_data)); + } + break; + } + case PrimitiveType::TYPE_MAP: { + const auto& map = assert_cast(*data_column); + const auto& key_type = + assert_cast(remove_nullable(data_type).get())->get_key_type(); + const auto& value_type = + assert_cast(remove_nullable(data_type).get())->get_value_type(); + const auto& key_column = map.get_keys_ptr(); + const auto& value_column = map.get_values_ptr(); + meta_data.emplace_back((long)map.get_offsets().data()); + RETURN_IF_ERROR(_fill_column_meta(key_column, key_type, meta_data)); + RETURN_IF_ERROR(_fill_column_meta(value_column, value_type, meta_data)); + break; + } + case PrimitiveType::TYPE_VARBINARY: { + const auto& varbinary_col = assert_cast(*data_column); + meta_data.emplace_back((long)varbinary_col.get_data().data()); + break; + } + default: + return Status::InternalError("Unsupported type: {}", data_type->get_name()); + } + return Status::OK(); +} + +Status JniDataBridge::to_java_table(Block* block, std::unique_ptr& meta) { + ColumnNumbers arguments; + for (size_t i = 0; i < block->columns(); ++i) { + arguments.emplace_back(i); + } + return to_java_table(block, block->rows(), arguments, meta); +} + +Status JniDataBridge::to_java_table(Block* block, size_t num_rows, const ColumnNumbers& arguments, + std::unique_ptr& meta) { + std::vector meta_data; + // insert number of rows + meta_data.emplace_back(num_rows); + for (size_t i : arguments) { + auto& column_with_type_and_name = block->get_by_position(i); + RETURN_IF_ERROR(_fill_column_meta(column_with_type_and_name.column, + column_with_type_and_name.type, meta_data)); + } + + meta.reset(new long[meta_data.size()]); + memcpy(meta.get(), &meta_data[0], meta_data.size() * 8); + return Status::OK(); +} + +std::pair JniDataBridge::parse_table_schema( + Block* block, const ColumnNumbers& arguments, bool ignore_column_name) { + // prepare table schema + std::ostringstream required_fields; + std::ostringstream columns_types; + for (int i = 0; i < arguments.size(); ++i) { + std::string type = JniDataBridge::get_jni_type(block->get_by_position(arguments[i]).type); + if (i == 0) { + if (ignore_column_name) { + required_fields << "_col_" << arguments[i]; + } else { + required_fields << block->get_by_position(arguments[i]).name; + } + columns_types << type; + } else { + if (ignore_column_name) { + required_fields << "," + << "_col_" << arguments[i]; + } else { + required_fields << "," << block->get_by_position(arguments[i]).name; + } + columns_types << "#" << type; + } + } + return std::make_pair(required_fields.str(), columns_types.str()); +} + +std::pair JniDataBridge::parse_table_schema(Block* block) { + ColumnNumbers arguments; + for (size_t i = 0; i < block->columns(); ++i) { + arguments.emplace_back(i); + } + return parse_table_schema(block, arguments, true); +} + +} // namespace doris diff --git a/be/src/format_v2/jni/jni_data_bridge.h b/be/src/format_v2/jni/jni_data_bridge.h new file mode 100644 index 00000000000000..d3deca8a85076d --- /dev/null +++ b/be/src/format_v2/jni/jni_data_bridge.h @@ -0,0 +1,237 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_decimal.h" +#include "core/data_type/data_type.h" +#include "core/data_type/define_primitive_type.h" +#include "core/data_type/primitive_type.h" +#include "core/string_ref.h" +#include "core/types.h" +#include "exprs/aggregate/aggregate_function.h" + +namespace doris { + +class Block; +template +class ColumnDecimal; +template +class ColumnVector; + +/** + * JniDataBridge is a stateless utility class that handles data exchange + * between C++ Blocks and Java-side shared memory via JNI. + * + * It is data-source agnostic — it only cares about data types and Block structure. + * All methods are static. + * + * This class was extracted from JniConnector to separate the data exchange + * concerns from the JNI scanner lifecycle management. + */ +class JniDataBridge { +public: + /** + * Helper class to read metadata from the address returned by Java side. + * The metadata is stored as a long array in shared memory. + */ + class TableMetaAddress { + private: + long* _meta_ptr; + int _meta_index; + + public: + TableMetaAddress() { + _meta_ptr = nullptr; + _meta_index = 0; + } + + TableMetaAddress(long meta_addr) { + _meta_ptr = static_cast(reinterpret_cast(meta_addr)); + _meta_index = 0; + } + + void set_meta(long meta_addr) { + _meta_ptr = static_cast(reinterpret_cast(meta_addr)); + _meta_index = 0; + } + + long next_meta_as_long() { return _meta_ptr[_meta_index++]; } + + void* next_meta_as_ptr() { return reinterpret_cast(_meta_ptr[_meta_index++]); } + }; + + // ========================================================================= + // Read direction: Java shared memory → C++ Block + // ========================================================================= + + /** + * Fill specified columns in a Block from a Java-side table address. + * The table_address points to metadata returned by Java JniScanner/JdbcExecutor. + */ + static Status fill_block(Block* block, const ColumnNumbers& arguments, long table_address); + + /** + * Fill a single column from a TableMetaAddress. Supports all Doris types + * including nested types (Array, Map, Struct). + */ + static Status fill_column(TableMetaAddress& address, ColumnPtr& doris_column, + const DataTypePtr& data_type, size_t num_rows); + + // ========================================================================= + // Write direction: C++ Block → Java shared memory + // ========================================================================= + + /** + * Serialize all columns of a Block into a long[] metadata array + * that Java side can read via VectorTable.createReadableTable(). + */ + static Status to_java_table(Block* block, std::unique_ptr& meta); + + /** + * Serialize specified columns of a Block into a long[] metadata array. + */ + static Status to_java_table(Block* block, size_t num_rows, const ColumnNumbers& arguments, + std::unique_ptr& meta); + + /** + * Parse Block schema into JNI format strings. + * Returns (required_fields, columns_types) pair. + */ + static std::pair parse_table_schema(Block* block); + + static std::pair parse_table_schema(Block* block, + const ColumnNumbers& arguments, + bool ignore_column_name = true); + + // ========================================================================= + // Type mapping + // ========================================================================= + + /** + * Convert a Doris DataType to its JNI type string representation. + * e.g., TYPE_INT -> "int", TYPE_DECIMAL128I -> "decimal128(p,s)" + */ + static std::string get_jni_type(const DataTypePtr& data_type); + + /** + * Like get_jni_type but preserves varchar/char length info in the type string. + * e.g., TYPE_VARCHAR -> "varchar(len)" instead of just "string" + */ + static std::string get_jni_type_with_different_string(const DataTypePtr& data_type); + +private: + // Column fill helpers for various types + static Status _fill_string_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + size_t num_rows); + + static Status _fill_varbinary_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + size_t num_rows); + + static Status _fill_array_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + const DataTypePtr& data_type, size_t num_rows); + + static Status _fill_map_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + const DataTypePtr& data_type, size_t num_rows); + + static Status _fill_struct_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + const DataTypePtr& data_type, size_t num_rows); + + /** + * Fill column metadata (addresses) for a single column, used by to_java_table. + */ + static Status _fill_column_meta(const ColumnPtr& doris_column, const DataTypePtr& data_type, + std::vector& meta_data); + + // Fixed-length column fill specializations + template + requires(!std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v) + static Status _fill_fixed_length_column(MutableColumnPtr& doris_column, CPP_TYPE* ptr, + size_t num_rows) { + auto& column_data = assert_cast(*doris_column).get_data(); + size_t origin_size = column_data.size(); + column_data.resize(origin_size + num_rows); + memcpy(column_data.data() + origin_size, ptr, sizeof(CPP_TYPE) * num_rows); + return Status::OK(); + } + + template + requires(std::is_same_v || + std::is_same_v) + static Status _fill_fixed_length_column(MutableColumnPtr& doris_column, CPP_TYPE* ptr, + size_t num_rows) { + auto& column_data = assert_cast(*doris_column).get_data(); + size_t origin_size = column_data.size(); + column_data.resize(origin_size + num_rows); + memcpy((int64_t*)column_data.data() + origin_size, ptr, sizeof(CPP_TYPE) * num_rows); + return Status::OK(); + } + + template + requires(std::is_same_v) + static Status _fill_fixed_length_column(MutableColumnPtr& doris_column, CPP_TYPE* ptr, + size_t num_rows) { + auto& column_data = assert_cast(*doris_column).get_data(); + size_t origin_size = column_data.size(); + column_data.resize(origin_size + num_rows); + memcpy((uint32_t*)column_data.data() + origin_size, ptr, sizeof(CPP_TYPE) * num_rows); + return Status::OK(); + } + + template + requires(std::is_same_v || + std::is_same_v) + static Status _fill_fixed_length_column(MutableColumnPtr& doris_column, CPP_TYPE* ptr, + size_t num_rows) { + auto& column_data = assert_cast(*doris_column).get_data(); + size_t origin_size = column_data.size(); + column_data.resize(origin_size + num_rows); + memcpy((uint64_t*)column_data.data() + origin_size, ptr, sizeof(CPP_TYPE) * num_rows); + return Status::OK(); + } + + template + requires(std::is_same_v) + static Status _fill_fixed_length_column(MutableColumnPtr& doris_column, CPP_TYPE* ptr, + size_t num_rows) { + auto& column_data = assert_cast(*doris_column).get_data(); + size_t origin_size = column_data.size(); + column_data.resize(origin_size + num_rows); + for (size_t i = 0; i < num_rows; i++) { + column_data[origin_size + i] = DecimalV2Value(ptr[i]); + } + return Status::OK(); + } + + template + static long _get_fixed_length_column_address(const IColumn& doris_column) { + return (long)assert_cast(doris_column).get_data().data(); + } +}; + +} // namespace doris diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp new file mode 100644 index 00000000000000..3cb105193b56ae --- /dev/null +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -0,0 +1,556 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/jni_table_reader.h" + +#include + +#include "common/cast_set.h" +#include "common/logging.h" +#include "core/block/block.h" +#include "exprs/vexpr_context.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" +#include "util/string_util.h" + +namespace doris::format { + +Status JniTableReader::init(TableReadOptions&& options) { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + _init_profile(); + return Status::OK(); +} + +Status JniTableReader::prepare_split(const SplitReadOptions& options) { + // EOF belongs to the previous split. Keep it set after closing that split so repeated reads + // are idempotent, and clear it only when a new split is explicitly prepared. + _eof = false; + _current_range = options.current_range; + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + RETURN_IF_ERROR(TableReader::prepare_split(options)); + if (current_split_pruned()) { + return Status::OK(); + } + DORIS_CHECK(!_closed); + DORIS_CHECK(!_scanner_opened); + if (_is_table_level_count_active()) { + return Status::OK(); + } + // JNI readers do not go through TableReader::open_reader(), where native readers prepare + // file-local filters. Prepare the fresh per-split snapshot before it filters JNI blocks. + RowDescriptor row_desc; + for (const auto& conjunct : _conjuncts) { + RETURN_IF_ERROR(conjunct->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(conjunct->open(_runtime_state)); + } + // Subclasses populate split-specific scanner params before calling this method, so the Java + // scanner can be opened here instead of being lazily opened by the first get_block() call. + return _open_jni_scanner(); +} + +Status JniTableReader::get_block(Block* output_block, bool* eos) { + DORIS_CHECK(output_block != nullptr); + DORIS_CHECK(eos != nullptr); + DORIS_CHECK(output_block->columns() == _projected_columns.size()); + output_block->clear_column_data(_projected_columns.size()); + if (_is_table_level_count_active()) { + return _read_table_level_count(output_block, eos); + } + + if (_eof) { + *eos = true; + return Status::OK(); + } + DORIS_CHECK(_scanner_opened); + + while (true) { + // JNI readers can loop internally when conjuncts filter every Java batch. Mirror the base + // TableReader cancellation contract so a cancelled query does not drain the whole split. + if (_io_ctx != nullptr && _io_ctx->should_stop) { + _eof = true; + RETURN_IF_ERROR(_close_jni_scanner()); + *eos = true; + return Status::OK(); + } + size_t current_rows = 0; + bool current_eof = false; + // get next block data from Java scanner, and fill the data to _jni_block_template + RETURN_IF_ERROR(_get_next_jni_block(¤t_rows, ¤t_eof)); + if (current_eof) { + _eof = true; + RETURN_IF_ERROR(_close_jni_scanner()); + *eos = true; + return Status::OK(); + } + + _record_scan_rows(current_rows); + RETURN_IF_ERROR(finalize_jni_block(&_jni_block_template, output_block, ¤t_rows)); + if (current_rows == 0) { + output_block->clear_column_data(_projected_columns.size()); + continue; + } + *eos = false; + return Status::OK(); + } +} + +Status JniTableReader::abort_split() { + RETURN_IF_ERROR(_close_jni_scanner()); + return TableReader::abort_split(); +} + +Status JniTableReader::_get_next_jni_block(size_t* rows, bool* eof) { + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + *rows = 0; + _jni_block_template.clear_column_data(_jni_columns.size()); + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + long meta_address = 0; + { + SCOPED_RAW_TIMER(&_java_scan_watcher); + //getNextBatchMeta function, return the meta address + RETURN_IF_ERROR(_jni_scanner_obj.call_long_method(env, _jni_scanner_get_next_batch) + .call(&meta_address)); + } + RETURN_ERROR_IF_EXC(env); + if (meta_address == 0) { + *eof = true; + return Status::OK(); + } + + JniDataBridge::TableMetaAddress table_meta(meta_address); + const auto num_rows = table_meta.next_meta_as_long(); + if (num_rows == 0) { + *eof = true; + return Status::OK(); + } + + *rows = cast_set(num_rows); + // fill data from Java table meta to C++ block + RETURN_IF_ERROR(_fill_jni_block(table_meta, *rows)); + // call releaseTable() method in JAVA side to release the Java table Heap free Memory + RETURN_IF_ERROR(_jni_scanner_obj.call_void_method(env, _jni_scanner_release_table).call()); + RETURN_ERROR_IF_EXC(env); + *eof = false; + return Status::OK(); +} + +// Java table to C++ block +Status JniTableReader::_fill_jni_block(JniDataBridge::TableMetaAddress& table_meta, + size_t num_rows) { + SCOPED_RAW_TIMER(&_fill_block_watcher); + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + for (size_t i = 0; i < _jni_columns.size(); ++i) { + const auto& read_column = _jni_columns[i]; + auto& column_with_type_and_name = _jni_block_template.get_by_position(i); + auto& column_ptr = column_with_type_and_name.column; + RETURN_IF_ERROR(JniDataBridge::fill_column(table_meta, column_ptr, + read_column.transfer_type, num_rows)); + // call releaseColumn(int columnIndex) method in JAVA side to release the Java column Heap free Memory + RETURN_IF_ERROR(_jni_scanner_obj.call_void_method(env, _jni_scanner_release_column) + .with_arg(cast_set(i)) + .call()); + RETURN_ERROR_IF_EXC(env); + } + return Status::OK(); +} + +Status JniTableReader::finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) { + DORIS_CHECK(jni_block != nullptr); + DORIS_CHECK(output_block != nullptr); + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(jni_block->columns() == _jni_columns.size()); + const auto original_rows = *rows; + for (size_t i = 0; i < _jni_columns.size(); ++i) { + const auto& column = _jni_columns[i]; + DORIS_CHECK(column.output_index < output_block->columns()); + output_block->get_by_position(column.output_index).type = column.output_type; + output_block->replace_by_position(column.output_index, + jni_block->get_by_position(i).column); + } + DORIS_CHECK(output_block->rows() == original_rows); + // Apply conjuncts on the output block + if (!_conjuncts.empty()) { + RETURN_IF_ERROR( + VExprContext::filter_block(_conjuncts, output_block, output_block->columns())); + } + *rows = output_block->rows(); + return Status::OK(); +} + +Status JniTableReader::_get_statistics(JNIEnv* env, std::map* result) { + DORIS_CHECK(result != nullptr); + result->clear(); + Jni::LocalObject metrics; + RETURN_IF_ERROR( + _jni_scanner_obj.call_object_method(env, _jni_scanner_get_statistics).call(&metrics)); + RETURN_IF_ERROR(Jni::Util::convert_to_cpp_map(env, metrics, result)); + return Status::OK(); +} + +void JniTableReader::_collect_jni_scanner_profile(JNIEnv* env) { + if (_scanner_profile == nullptr) { + return; + } + + std::map statistics_result; + Status st = _get_statistics(env, &statistics_result); + if (!st) { + LOG(WARNING) << "failed to get_statistics when collect profile: " << st; + return; + } + + const auto connector_name = _connector_name(); + const auto update_peak = [](int64_t previous, int64_t current) { return current > previous; }; + for (const auto& metric : statistics_result) { + std::vector type_and_name = split(metric.first, ":"); + if (type_and_name.size() != 2) { + LOG(WARNING) << "Name of JNI Scanner metric should be pattern like " + << "'metricType:metricName'"; + continue; + } + int64_t metric_value = std::stoll(metric.second); + RuntimeProfile::Counter* scanner_counter; + if (type_and_name[0] == "timer") { + scanner_counter = + ADD_CHILD_TIMER(_scanner_profile, type_and_name[1], connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); + } else if (type_and_name[0] == "counter") { + scanner_counter = ADD_CHILD_COUNTER(_scanner_profile, type_and_name[1], TUnit::UNIT, + connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); + } else if (type_and_name[0] == "bytes") { + scanner_counter = ADD_CHILD_COUNTER(_scanner_profile, type_and_name[1], TUnit::BYTES, + connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); + } else if (type_and_name[0] == "timer_gauge") { + scanner_counter = + ADD_CHILD_TIMER(_scanner_profile, type_and_name[1], connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "gauge") { + scanner_counter = ADD_CHILD_COUNTER(_scanner_profile, type_and_name[1], TUnit::UNIT, + connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "bytes_gauge") { + scanner_counter = ADD_CHILD_COUNTER(_scanner_profile, type_and_name[1], TUnit::BYTES, + connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "timer_peak") { + auto* scanner_peak_counter = _scanner_profile->add_conditition_counter( + type_and_name[1], TUnit::TIME_NS, update_peak, connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); + } else if (type_and_name[0] == "peak") { + auto* scanner_peak_counter = _scanner_profile->add_conditition_counter( + type_and_name[1], TUnit::UNIT, update_peak, connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); + } else if (type_and_name[0] == "bytes_peak") { + auto* scanner_peak_counter = _scanner_profile->add_conditition_counter( + type_and_name[1], TUnit::BYTES, update_peak, connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); + } else { + LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter, bytes, " + << "timer_gauge, gauge, bytes_gauge, timer_peak, peak or bytes_peak"; + continue; + } + } +} + +Status JniTableReader::build_jni_columns(std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + columns->reserve(_projected_columns.size()); + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + columns->push_back({ + .java_name = table_column.name, + .output_index = i, + .output_type = table_column.type, + .transfer_type = table_column.type, + .replace_type = "not_replace", + }); + } + return Status::OK(); +} + +int64_t JniTableReader::self_split_weight() const { + return _current_range.__isset.self_split_weight ? _current_range.self_split_weight : -1; +} + +bool JniTableReader::_reserve_split_profile_publication() { + if (_split_profile_published) { + return false; + } + _split_profile_published = true; + return true; +} + +void JniTableReader::_publish_split_profile(JNIEnv* env) { + // Cleanup can fail while the Java scanner and split watchers must remain available for a + // retry. Reserve profile publication separately so a retry only repeats resource cleanup. + if (!_reserve_split_profile_publication()) { + return; + } + + if (_scanner_profile != nullptr) { + COUNTER_UPDATE(_open_scanner_time, _jni_scanner_open_watcher); + COUNTER_UPDATE(_fill_block_time, _fill_block_watcher); + } + + jlong append_data_time = 0; + const auto append_time_status = + _jni_scanner_obj.call_long_method(env, _jni_scanner_get_append_data_time) + .call(&append_data_time); + jlong create_vector_table_time = 0; + const auto create_table_time_status = + _jni_scanner_obj.call_long_method(env, _jni_scanner_get_create_vector_table_time) + .call(&create_vector_table_time); + if (!append_time_status.ok()) { + LOG(WARNING) << "failed to collect JNI append-data time during close: " + << append_time_status; + } + if (!create_table_time_status.ok()) { + LOG(WARNING) << "failed to collect JNI vector-table time during close: " + << create_table_time_status; + } + if (_scanner_profile != nullptr && append_time_status.ok() && create_table_time_status.ok()) { + COUNTER_UPDATE(_java_append_data_time, append_data_time); + COUNTER_UPDATE(_java_create_vector_table_time, create_vector_table_time); + COUNTER_UPDATE(_java_scan_time, + _java_scan_watcher - append_data_time - create_vector_table_time); + _max_time_split_weight_counter->conditional_update( + _jni_scanner_open_watcher + _fill_block_watcher + _java_scan_watcher, + self_split_weight()); + } + _collect_jni_scanner_profile(env); +} + +Status JniTableReader::close() { + if (_closed) { + return Status::OK(); + } + auto close_status = _close_jni_scanner(); + auto table_status = TableReader::close(); + if (close_status.ok() && !table_status.ok()) { + close_status = std::move(table_status); + } + if (close_status.ok()) { + _closed = true; + } + return close_status; +} + +Status JniTableReader::_close_jni_scanner() { + if (!_scanner_opened) { + JNIEnv* env = nullptr; + if (!_jni_scanner_obj.uninitialized()) { + RETURN_IF_ERROR(Jni::Env::Get(&env)); + } + _reset_split_state(env); + return Status::OK(); + } + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + _publish_split_profile(env); + + // _fill_jni_block may fail before releasing the current Java table. JniScanner::releaseTable() + // is idempotent, so closing the split always releases it. Java close must still run if that + // release fails; otherwise connector resources such as JDBC connections can leak. + auto cleanup_status = _jni_scanner_obj.call_void_method(env, _jni_scanner_release_table).call(); + auto java_close_status = _jni_scanner_obj.call_void_method(env, _jni_scanner_close).call(); + if (cleanup_status.ok() && !java_close_status.ok()) { + cleanup_status = std::move(java_close_status); + } + if (cleanup_status.ok()) { + // Keep the Java object and opened state on failure so close() can retry the cleanup. + _reset_split_state(env); + } + return cleanup_status; +} + +void JniTableReader::_reset_split_state(JNIEnv* env) { + if (!_jni_scanner_obj.uninitialized()) { + DORIS_CHECK(env != nullptr); + _jni_scanner_obj.reset(env); + } + _scanner_opened = false; + _scanner_params.clear(); + _jni_columns.clear(); + _jni_block_template.clear(); + _jni_scanner_open_watcher = 0; + _java_scan_watcher = 0; + _fill_block_watcher = 0; + _split_profile_published = false; +} + +Status JniTableReader::_open_jni_scanner() { + // subclasses build map _scanner_params to JAVA side + RETURN_IF_ERROR(build_scanner_params(&_scanner_params)); + // subclasses build _jni_columns info to JAVA side, including column name and column type + RETURN_IF_ERROR(build_jni_columns(&_jni_columns)); + // _jni_columns info is used to build Java scanner schema params and JNI block template. + _prepare_jni_scanner_schema(); + + if (_runtime_state != nullptr && _batch_size == 0) { + _batch_size = _runtime_state->batch_size(); + } + if (_runtime_state != nullptr) { + _scanner_params["time_zone"] = _runtime_state->timezone(); + } + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + SCOPED_RAW_TIMER(&_jni_scanner_open_watcher); + RETURN_IF_ERROR(_register_jni_class_functions_once(env)); + RETURN_IF_ERROR(_create_jni_scanner_object(env, cast_set(_batch_size))); + // Once the Java object exists, close it even if open() fails partway through initialization. + // Connector implementations may already own streams, off-heap tables, or JDBC connections. + _scanner_opened = true; + // call open() method in JAVA side. + const auto open_status = _jni_scanner_obj.call_void_method(env, _jni_scanner_open).call(); + if (!open_status.ok()) { + const auto close_status = _close_jni_scanner(); + if (!close_status.ok()) { + LOG(WARNING) << "failed to clean up JNI scanner after open failure: " << close_status; + } + return open_status; + } + return Status::OK(); +} + +void JniTableReader::set_batch_size(size_t batch_size) { + if (_scanner_opened && !supports_batch_size_update_after_open()) { + // Some connectors bake the constructor batch size into an already-open physical reader. + // Keep C++ and Java on that initial size instead of pretending a later resize took effect. + return; + } + TableReader::set_batch_size(batch_size); + if (!_scanner_opened) { + return; + } + const auto status = _set_open_scanner_batch_size(_batch_size); + if (!status.ok()) { + // Adaptive batch sizing is an optimization. Keep the scanner usable with its previous + // size if Java rejects a mid-split update, but surface the failure for diagnosis. + LOG(WARNING) << "failed to update JNI scanner batch size: " << status; + } +} + +Status JniTableReader::_set_open_scanner_batch_size(size_t batch_size) { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + return _jni_scanner_obj.call_void_method(env, _jni_scanner_set_batch_size) + .with_arg(cast_set(batch_size)) + .call(); +} + +void JniTableReader::_prepare_jni_scanner_schema() { + std::vector required_fields; + std::vector column_types; + std::vector replace_types; + required_fields.reserve(_jni_columns.size()); + column_types.reserve(_jni_columns.size()); + replace_types.reserve(_jni_columns.size()); + _jni_block_template.clear(); + _jni_block_template.reserve(_jni_columns.size()); + + bool has_replace_type = false; + for (const auto& column : _jni_columns) { + DORIS_CHECK(column.transfer_type != nullptr); + required_fields.push_back(column.java_name); + column_types.push_back( + JniDataBridge::get_jni_type_with_different_string(column.transfer_type)); + replace_types.push_back(column.replace_type); + has_replace_type = has_replace_type || column.replace_type != "not_replace"; + _jni_block_template.insert( + {column.transfer_type->create_column(), column.transfer_type, column.java_name}); + } + _scanner_params["required_fields"] = join(required_fields, ","); + _scanner_params["columns_types"] = join(column_types, "#"); + if (has_replace_type) { + _scanner_params["replace_string"] = join(replace_types, ","); + } +} + +Status JniTableReader::_register_jni_class_functions_once(JNIEnv* env) { + if (!_jni_scanner_cls.uninitialized()) { + return Status::OK(); + } + + RETURN_IF_ERROR( + Jni::Util::get_jni_scanner_class(env, connector_class().c_str(), &_jni_scanner_cls)); + RETURN_IF_ERROR(_jni_scanner_cls.get_method(env, "", "(ILjava/util/Map;)V", + &_jni_scanner_constructor)); + RETURN_IF_ERROR(_jni_scanner_cls.get_method(env, "open", "()V", &_jni_scanner_open)); + RETURN_IF_ERROR(_jni_scanner_cls.get_method(env, "getNextBatchMeta", "()J", + &_jni_scanner_get_next_batch)); + RETURN_IF_ERROR(_jni_scanner_cls.get_method(env, "getAppendDataTime", "()J", + &_jni_scanner_get_append_data_time)); + RETURN_IF_ERROR(_jni_scanner_cls.get_method(env, "getCreateVectorTableTime", "()J", + &_jni_scanner_get_create_vector_table_time)); + RETURN_IF_ERROR(_jni_scanner_cls.get_method(env, "close", "()V", &_jni_scanner_close)); + RETURN_IF_ERROR(_jni_scanner_cls.get_method(env, "releaseColumn", "(I)V", + &_jni_scanner_release_column)); + RETURN_IF_ERROR( + _jni_scanner_cls.get_method(env, "releaseTable", "()V", &_jni_scanner_release_table)); + RETURN_IF_ERROR(_jni_scanner_cls.get_method(env, "getStatistics", "()Ljava/util/Map;", + &_jni_scanner_get_statistics)); + RETURN_IF_ERROR( + _jni_scanner_cls.get_method(env, "setBatchSize", "(I)V", &_jni_scanner_set_batch_size)); + return Status::OK(); +} + +Status JniTableReader::_create_jni_scanner_object(JNIEnv* env, int batch_size) { + DORIS_CHECK(!_jni_scanner_cls.uninitialized()); + DORIS_CHECK(!_jni_scanner_constructor.uninitialized()); + DORIS_CHECK(_jni_scanner_obj.uninitialized()); + Jni::LocalObject hashmap_object; + RETURN_IF_ERROR(Jni::Util::convert_to_java_map(env, _scanner_params, &hashmap_object)); + RETURN_IF_ERROR(_jni_scanner_cls.new_object(env, _jni_scanner_constructor) + .with_arg(batch_size) + .with_arg(hashmap_object) + .call(&_jni_scanner_obj)); + return Status::OK(); +} + +void JniTableReader::_init_profile() { + if (_scanner_profile == nullptr) { + return; + } + const auto connector_name = _connector_name(); + ADD_TIMER(_scanner_profile, connector_name); + _open_scanner_time = ADD_CHILD_TIMER(_scanner_profile, "OpenScannerTime", connector_name); + _java_scan_time = ADD_CHILD_TIMER(_scanner_profile, "JavaScanTime", connector_name); + _java_append_data_time = + ADD_CHILD_TIMER(_scanner_profile, "JavaAppendDataTime", connector_name); + _java_create_vector_table_time = + ADD_CHILD_TIMER(_scanner_profile, "JavaCreateVectorTableTime", connector_name); + _fill_block_time = ADD_CHILD_TIMER(_scanner_profile, "FillBlockTime", connector_name); + _max_time_split_weight_counter = _scanner_profile->add_conditition_counter( + "MaxTimeSplitWeight", TUnit::UNIT, [](int64_t _c, int64_t c) { return c > _c; }, + connector_name); +} + +std::string JniTableReader::_connector_name() const { + const auto parts = split(connector_class(), "/"); + return parts.empty() ? connector_class() : parts.back(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h new file mode 100644 index 00000000000000..fe908d654863eb --- /dev/null +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/jni/jni_data_bridge.h" +#include "format_v2/table_reader.h" +#include "runtime/runtime_profile.h" +#include "util/jni-util.h" + +namespace doris::format { + +class JniTableReader : public TableReader { +public: + struct JniColumn { + std::string java_name; + // The index of the column in the output block, which is used to place the data from Java side to the correct position in the output block. + size_t output_index = 0; + // The original output type of the column, which is used for type casting after getting the data from Java side. like Bitmap column + // For columns without special types, the transfer_type and output_type are the same. + DataTypePtr output_type; + //Bitmap Type transfer type is String, so the Java scanner will convert the Bitmap column to String before transferring the data to C++, and then C++ side can convert the String back to Bitmap. + DataTypePtr transfer_type; + std::string replace_type = "not_replace"; + }; + + ~JniTableReader() override = default; + + Status init(TableReadOptions&& options) override; + Status prepare_split(const SplitReadOptions& options) override; + Status get_block(Block* block, bool* eos) override; + Status abort_split() override; + Status close() override; + void set_batch_size(size_t batch_size) override; + +#ifdef BE_TEST + void TEST_set_split_state(bool scanner_opened, bool eof) { + _scanner_opened = scanner_opened; + _eof = eof; + if (!scanner_opened) { + _split_profile_published = false; + } + } + bool TEST_scanner_opened() const { return _scanner_opened; } + bool TEST_eof() const { return _eof; } + bool TEST_closed() const { return _closed; } +#endif + +protected: + // Subclasses should implement these methods to specify the Java scanner class + virtual std::string connector_class() const = 0; + virtual Status validate_scan_range(const TFileRangeDesc&) const { return Status::OK(); } + // Subclasses should implement this method to build the scanner params map + virtual Status build_scanner_params(std::map* params) const = 0; + // Subclasses can override this method when Java transfer types differ from output types. + virtual Status build_jni_columns(std::vector* columns) const; + virtual Status finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows); + // used for profile + virtual int64_t self_split_weight() const; + virtual Status _get_next_jni_block(size_t* rows, bool* eof); + virtual Status _close_jni_scanner(); + virtual Status _set_open_scanner_batch_size(size_t batch_size); + virtual bool supports_batch_size_update_after_open() const { return true; } + virtual Status _open_jni_scanner(); + bool _reserve_split_profile_publication(); + const std::vector& jni_columns() const { return _jni_columns; } + TFileRangeDesc _current_range; + +private: + // init + void _init_profile(); + std::string _connector_name() const; + // open + void _reset_split_state(JNIEnv* env); + void _prepare_jni_scanner_schema(); + Status _register_jni_class_functions_once(JNIEnv* env); + Status _create_jni_scanner_object(JNIEnv* env, int batch_size); + // get_next + Status _fill_jni_block(JniDataBridge::TableMetaAddress& table_meta, size_t num_rows); + Status _get_statistics(JNIEnv* env, std::map* result); + void _collect_jni_scanner_profile(JNIEnv* env); + void _publish_split_profile(JNIEnv* env); + + std::map _scanner_params; + std::vector _jni_columns; + Block _jni_block_template; + + bool _closed = false; + bool _scanner_opened = false; + bool _eof = false; + bool _split_profile_published = false; + + RuntimeProfile::Counter* _open_scanner_time = nullptr; + RuntimeProfile::Counter* _java_scan_time = nullptr; + RuntimeProfile::Counter* _java_append_data_time = nullptr; + RuntimeProfile::Counter* _java_create_vector_table_time = nullptr; + RuntimeProfile::Counter* _fill_block_time = nullptr; + RuntimeProfile::ConditionCounter* _max_time_split_weight_counter = nullptr; + + int64_t _jni_scanner_open_watcher = 0; + int64_t _java_scan_watcher = 0; + int64_t _fill_block_watcher = 0; + + Jni::GlobalClass _jni_scanner_cls; + Jni::GlobalObject _jni_scanner_obj; + Jni::MethodId _jni_scanner_constructor; + Jni::MethodId _jni_scanner_open; + Jni::MethodId _jni_scanner_get_append_data_time; + Jni::MethodId _jni_scanner_get_create_vector_table_time; + Jni::MethodId _jni_scanner_get_next_batch; + Jni::MethodId _jni_scanner_close; + Jni::MethodId _jni_scanner_release_column; + Jni::MethodId _jni_scanner_release_table; + Jni::MethodId _jni_scanner_get_statistics; + Jni::MethodId _jni_scanner_set_batch_size; +}; + +} // namespace doris::format diff --git a/be/src/format_v2/jni/max_compute_jni_reader.cpp b/be/src/format_v2/jni/max_compute_jni_reader.cpp new file mode 100644 index 00000000000000..a26e9e229b5d82 --- /dev/null +++ b/be/src/format_v2/jni/max_compute_jni_reader.cpp @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/max_compute_jni_reader.h" + +#include "core/block/block.h" +#include "exprs/vexpr_context.h" + +namespace doris::format::max_compute { + +MaxComputeJniReader::MaxComputeJniReader(const doris::MaxComputeTableDescriptor* table_desc) + : _table_desc(table_desc) {} + +Status MaxComputeJniReader::validate_scan_range(const TFileRangeDesc& range) const { + if (!range.__isset.table_format_params) { + return Status::InternalError("missing table_format_params for max compute jni reader"); + } + if (!range.table_format_params.__isset.max_compute_params) { + return Status::InternalError("missing max_compute_params for max compute jni reader"); + } + const auto& max_compute_params = range.table_format_params.max_compute_params; + if (!max_compute_params.__isset.session_id || max_compute_params.session_id.empty()) { + return Status::InternalError( + "missing session_id for max compute jni reader, possibly caused by FE/BE " + "protocol mismatch"); + } + if (!max_compute_params.__isset.table_batch_read_session || + max_compute_params.table_batch_read_session.empty()) { + return Status::InternalError( + "missing table_batch_read_session for max compute jni reader, possibly caused " + "by FE/BE protocol mismatch"); + } + if (!range.__isset.start_offset) { + return Status::InternalError( + "missing start_offset for max compute jni reader, possibly caused by FE/BE " + "protocol mismatch"); + } + if (!range.__isset.size) { + return Status::InternalError( + "missing size for max compute jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + if (_scan_params == nullptr) { + return Status::InternalError( + "missing scan params for max compute jni reader, possibly caused by FE/BE " + "protocol mismatch"); + } + return Status::OK(); +} + +std::string MaxComputeJniReader::connector_class() const { + return "org/apache/doris/maxcompute/MaxComputeJniScanner"; +} + +Status MaxComputeJniReader::build_scanner_params(std::map* params) const { + DORIS_CHECK(params != nullptr); + DORIS_CHECK(_table_desc != nullptr); + params->clear(); + + *params = _table_desc->properties(); + (*params)["endpoint"] = _table_desc->endpoint(); + (*params)["quota"] = _table_desc->quota(); + (*params)["project"] = _table_desc->project(); + (*params)["table"] = _table_desc->table(); + + const auto& max_compute_params = _current_range.table_format_params.max_compute_params; + (*params)["session_id"] = max_compute_params.session_id; + (*params)["scan_serializer"] = max_compute_params.table_batch_read_session; + (*params)["start_offset"] = std::to_string(_current_range.start_offset); + (*params)["split_size"] = std::to_string(_current_range.size); + (*params)["connect_timeout"] = std::to_string(max_compute_params.connect_timeout); + (*params)["read_timeout"] = std::to_string(max_compute_params.read_timeout); + (*params)["retry_count"] = std::to_string(max_compute_params.retry_times); + return Status::OK(); +} + +Status MaxComputeJniReader::build_jni_columns( + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + columns->reserve(_projected_columns.size()); + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + if (table_column.is_partition_key && + find_partition_value(table_column, _partition_values) != nullptr) { + continue; + } + columns->push_back({ + .java_name = table_column.name, + .output_index = i, + .output_type = table_column.type, + .transfer_type = table_column.type, + .replace_type = "not_replace", + }); + } + return Status::OK(); +} + +Status MaxComputeJniReader::finalize_jni_block(Block* jni_block, Block* output_block, + size_t* rows) { + DORIS_CHECK(jni_block != nullptr); + DORIS_CHECK(output_block != nullptr); + DORIS_CHECK(rows != nullptr); + const auto original_rows = *rows; + + const auto& columns = jni_columns(); + DORIS_CHECK(columns.size() == jni_block->columns()); + for (size_t i = 0; i < columns.size(); ++i) { + const auto& column = columns[i]; + DORIS_CHECK(column.output_index < output_block->columns()); + output_block->get_by_position(column.output_index).type = column.output_type; + output_block->replace_by_position(column.output_index, + jni_block->get_by_position(i).column); + } + + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + const auto* partition_value = find_partition_value(table_column, _partition_values); + if (!table_column.is_partition_key || partition_value == nullptr) { + continue; + } + output_block->get_by_position(i).type = table_column.type; + output_block->replace_by_position( + i, table_column.type->create_column_const(original_rows, *partition_value)); + } + DORIS_CHECK(output_block->rows() == original_rows); + if (!_conjuncts.empty()) { + RETURN_IF_ERROR( + VExprContext::filter_block(_conjuncts, output_block, output_block->columns())); + } + *rows = output_block->rows(); + return Status::OK(); +} + +} // namespace doris::format::max_compute diff --git a/be/src/format_v2/jni/max_compute_jni_reader.h b/be/src/format_v2/jni/max_compute_jni_reader.h new file mode 100644 index 00000000000000..8addce07988e4c --- /dev/null +++ b/be/src/format_v2/jni/max_compute_jni_reader.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "format_v2/jni/jni_table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { +class MaxComputeTableDescriptor; +} // namespace doris + +namespace doris::format::max_compute { + +class MaxComputeJniReader final : public format::JniTableReader { +public: + explicit MaxComputeJniReader(const doris::MaxComputeTableDescriptor* table_desc); + ~MaxComputeJniReader() override = default; + +protected: + std::string connector_class() const override; + Status validate_scan_range(const TFileRangeDesc& range) const override; + Status build_scanner_params(std::map* params) const override; + Status build_jni_columns( + std::vector* columns) const override; + Status finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) override; + +private: + const doris::MaxComputeTableDescriptor* _table_desc = nullptr; +}; + +} // namespace doris::format::max_compute diff --git a/be/src/format_v2/jni/paimon_jni_reader.cpp b/be/src/format_v2/jni/paimon_jni_reader.cpp new file mode 100644 index 00000000000000..47c0ef6c7bcac4 --- /dev/null +++ b/be/src/format_v2/jni/paimon_jni_reader.cpp @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/paimon_jni_reader.h" + +#include + +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "util/string_util.h" + +namespace doris::format::paimon { +namespace { + +constexpr std::string_view PAIMON_OPTION_PREFIX = "paimon."; +constexpr std::string_view HADOOP_OPTION_PREFIX = "hadoop."; +constexpr std::string_view DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; +constexpr std::string_view DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; +constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR = "paimon_jni_scanner_io_tmp"; + +const std::string* get_paimon_predicate(const TFileScanRangeParams* scan_params, + const TPaimonFileDesc& paimon_params) { + if (scan_params != nullptr && scan_params->__isset.paimon_predicate && + !scan_params->paimon_predicate.empty()) { + return &scan_params->paimon_predicate; + } + if (paimon_params.__isset.paimon_predicate && !paimon_params.paimon_predicate.empty()) { + return &paimon_params.paimon_predicate; + } + return nullptr; +} + +} // namespace + +Status PaimonJniReader::validate_scan_range(const TFileRangeDesc& range) const { + if (!range.__isset.table_format_params) { + return Status::InternalError("missing table_format_params for paimon jni reader"); + } + if (!range.table_format_params.__isset.paimon_params) { + return Status::InternalError("missing paimon_params for paimon jni reader"); + } + if (!range.table_format_params.paimon_params.__isset.paimon_split || + range.table_format_params.paimon_params.paimon_split.empty()) { + return Status::InternalError( + "missing paimon_split for paimon jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + if (!range.table_format_params.paimon_params.__isset.reader_type || + range.table_format_params.paimon_params.reader_type != TPaimonReaderType::PAIMON_JNI) { + return Status::InternalError( + "invalid reader_type for paimon jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + if (_scan_params == nullptr || !_scan_params->__isset.serialized_table || + _scan_params->serialized_table.empty()) { + return Status::InternalError( + "missing serialized_table for paimon jni reader, possibly caused by FE/BE " + "protocol mismatch"); + } + if (get_paimon_predicate(_scan_params, range.table_format_params.paimon_params) == nullptr) { + return Status::InternalError( + "missing paimon_predicate for paimon jni reader, possibly caused by FE/BE " + "protocol mismatch"); + } + return Status::OK(); +} + +std::string PaimonJniReader::connector_class() const { + return "org/apache/doris/paimon/PaimonJniScanner"; +} + +Status PaimonJniReader::build_scanner_params(std::map* params) const { + DORIS_CHECK(params != nullptr); + DORIS_CHECK(_scan_params != nullptr); + params->clear(); + + const auto& paimon_params = _current_range.table_format_params.paimon_params; + const auto* paimon_predicate = get_paimon_predicate(_scan_params, paimon_params); + DORIS_CHECK(paimon_predicate != nullptr); + (*params)["paimon_split"] = paimon_params.paimon_split; + (*params)["paimon_predicate"] = *paimon_predicate; + (*params)["serialized_table"] = _scan_params->serialized_table; + + if (_scan_params->__isset.paimon_options && !_scan_params->paimon_options.empty()) { + for (const auto& kv : _scan_params->paimon_options) { + (*params)[std::string(PAIMON_OPTION_PREFIX) + kv.first] = kv.second; + } + } else if (paimon_params.__isset.paimon_options) { + // Rolling upgrades can pair this BE with an older FE that only sends options per split. + for (const auto& kv : paimon_params.paimon_options) { + (*params)[std::string(PAIMON_OPTION_PREFIX) + kv.first] = kv.second; + } + } + const std::string enable_io_manager_key = + std::string(PAIMON_OPTION_PREFIX) + std::string(DORIS_ENABLE_JNI_IO_MANAGER); + const std::string io_manager_tmp_dir_key = + std::string(PAIMON_OPTION_PREFIX) + std::string(DORIS_JNI_IO_MANAGER_TMP_DIR); + auto enable_io_manager_it = params->find(enable_io_manager_key); + if (enable_io_manager_it != params->end() && iequal(enable_io_manager_it->second, "true") && + params->find(io_manager_tmp_dir_key) == params->end()) { + DORIS_CHECK(_runtime_state != nullptr); + // Keep Format V2 consistent with the legacy JNI path. Paimon creates and + // removes its own paimon-* child directory under these Doris storage-root scoped + // parent directories. + (*params)[io_manager_tmp_dir_key] = + build_default_io_manager_tmp_dirs(_runtime_state->exec_env()->store_paths()); + } + if (_scan_params->__isset.properties && !_scan_params->properties.empty()) { + for (const auto& kv : _scan_params->properties) { + (*params)[std::string(HADOOP_OPTION_PREFIX) + kv.first] = kv.second; + } + } else if (paimon_params.__isset.hadoop_conf) { + for (const auto& kv : paimon_params.hadoop_conf) { + (*params)[std::string(HADOOP_OPTION_PREFIX) + kv.first] = kv.second; + } + } + // TODO: Remove legacy split-level paimon_predicate, paimon_options and hadoop_conf from thrift + // after the minimum supported FE always sends their scan-level replacements. + return Status::OK(); +} + +std::string PaimonJniReader::build_default_io_manager_tmp_dirs( + const std::vector& store_paths) { + std::vector tmp_dirs; + tmp_dirs.reserve(store_paths.size()); + for (const auto& store_path : store_paths) { + tmp_dirs.push_back(store_path.path + "/" + std::string(PAIMON_JNI_SCANNER_IO_TMP_DIR)); + } + DORIS_CHECK(!tmp_dirs.empty()); + return join(tmp_dirs, ":"); +} + +} // namespace doris::format::paimon diff --git a/be/src/format_v2/jni/paimon_jni_reader.h b/be/src/format_v2/jni/paimon_jni_reader.h new file mode 100644 index 00000000000000..26767458b427db --- /dev/null +++ b/be/src/format_v2/jni/paimon_jni_reader.h @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/jni/jni_table_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "storage/options.h" + +namespace doris::format::paimon { + +class PaimonJniReader final : public format::JniTableReader { +public: + ~PaimonJniReader() override = default; + +#ifdef BE_TEST + void TEST_set_scan_params(TFileScanRangeParams* params) { _scan_params = params; } + void TEST_set_runtime_state(RuntimeState* state) { _runtime_state = state; } + void TEST_set_current_range(TFileRangeDesc range) { _current_range = std::move(range); } + Status TEST_build_scanner_params(std::map* params) const { + return build_scanner_params(params); + } + static std::string TEST_build_default_io_manager_tmp_dirs( + const std::vector& store_paths) { + return build_default_io_manager_tmp_dirs(store_paths); + } +#endif + +protected: + std::string connector_class() const override; + Status validate_scan_range(const TFileRangeDesc& range) const override; + Status build_scanner_params(std::map* params) const override; + bool supports_batch_size_update_after_open() const override { return false; } + +private: + static std::string build_default_io_manager_tmp_dirs(const std::vector& store_paths); +}; + +} // namespace doris::format::paimon diff --git a/be/src/format_v2/jni/trino_connector_jni_reader.cpp b/be/src/format_v2/jni/trino_connector_jni_reader.cpp new file mode 100644 index 00000000000000..11c9945c5dea16 --- /dev/null +++ b/be/src/format_v2/jni/trino_connector_jni_reader.cpp @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/trino_connector_jni_reader.h" + +#include + +#include "common/config.h" +#include "util/jni-util.h" + +namespace doris::format::trino_connector { +namespace { + +constexpr std::string_view TRINO_CONNECTOR_OPTION_PREFIX = "trino."; +constexpr std::string_view TRINO_CONNECTOR_NAME = "connector.name"; + +} // namespace + +Status TrinoConnectorJniReader::validate_scan_range(const TFileRangeDesc& range) const { + if (!range.__isset.table_format_params) { + return Status::InternalError("missing table_format_params for trino connector jni reader"); + } + if (!range.table_format_params.__isset.trino_connector_params) { + return Status::InternalError( + "missing trino_connector_params for trino connector jni reader"); + } + + const auto& trino_params = range.table_format_params.trino_connector_params; + if (!trino_params.__isset.catalog_name || trino_params.catalog_name.empty()) { + return Status::InternalError( + "missing catalog_name for trino connector jni reader, possibly caused by FE/BE " + "protocol mismatch"); + } + if (!trino_params.__isset.trino_connector_options || + !trino_params.trino_connector_options.contains(std::string(TRINO_CONNECTOR_NAME))) { + return Status::InternalError( + "missing trino connector.name option for trino connector jni reader, possibly " + "caused by FE/BE protocol mismatch"); + } + if (!trino_params.__isset.trino_connector_split || trino_params.trino_connector_split.empty()) { + return Status::InternalError( + "missing trino_connector_split for trino connector jni reader, possibly caused " + "by FE/BE protocol mismatch"); + } + if (!trino_params.__isset.trino_connector_table_handle || + trino_params.trino_connector_table_handle.empty()) { + return Status::InternalError( + "missing trino_connector_table_handle for trino connector jni reader, possibly " + "caused by FE/BE protocol mismatch"); + } + if (!trino_params.__isset.trino_connector_column_handles || + trino_params.trino_connector_column_handles.empty()) { + return Status::InternalError( + "missing trino_connector_column_handles for trino connector jni reader, possibly " + "caused by FE/BE protocol mismatch"); + } + if (!trino_params.__isset.trino_connector_column_metadata || + trino_params.trino_connector_column_metadata.empty()) { + return Status::InternalError( + "missing trino_connector_column_metadata for trino connector jni reader, possibly " + "caused by FE/BE protocol mismatch"); + } + if (!trino_params.__isset.trino_connector_trascation_handle || + trino_params.trino_connector_trascation_handle.empty()) { + return Status::InternalError( + "missing trino_connector_trascation_handle for trino connector jni reader, " + "possibly caused by FE/BE protocol mismatch"); + } + return Status::OK(); +} + +Status TrinoConnectorJniReader::prepare_split(const format::SplitReadOptions& options) { + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + RETURN_IF_ERROR(_set_spi_plugins_dir()); + return format::JniTableReader::prepare_split(options); +} + +std::string TrinoConnectorJniReader::connector_class() const { + return "org/apache/doris/trinoconnector/TrinoConnectorJniScanner"; +} + +Status TrinoConnectorJniReader::build_scanner_params( + std::map* params) const { + DORIS_CHECK(params != nullptr); + params->clear(); + + const auto& trino_params = _current_range.table_format_params.trino_connector_params; + (*params)["catalog_name"] = trino_params.catalog_name; + (*params)["db_name"] = trino_params.db_name; + (*params)["table_name"] = trino_params.table_name; + (*params)["trino_connector_split"] = trino_params.trino_connector_split; + (*params)["trino_connector_table_handle"] = trino_params.trino_connector_table_handle; + (*params)["trino_connector_column_handles"] = trino_params.trino_connector_column_handles; + (*params)["trino_connector_column_metadata"] = trino_params.trino_connector_column_metadata; + (*params)["trino_connector_predicate"] = trino_params.trino_connector_predicate; + (*params)["trino_connector_trascation_handle"] = trino_params.trino_connector_trascation_handle; + + for (const auto& kv : trino_params.trino_connector_options) { + (*params)[std::string(TRINO_CONNECTOR_OPTION_PREFIX) + kv.first] = kv.second; + } + return Status::OK(); +} + +Status TrinoConnectorJniReader::_set_spi_plugins_dir() const { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + + Jni::LocalClass plugin_loader_cls; + const std::string plugin_loader_class = + "org/apache/doris/trinoconnector/TrinoConnectorPluginLoader"; + RETURN_IF_ERROR( + Jni::Util::get_jni_scanner_class(env, plugin_loader_class.c_str(), &plugin_loader_cls)); + + Jni::MethodId set_plugins_dir_method; + RETURN_IF_ERROR(plugin_loader_cls.get_static_method( + env, "setPluginsDir", "(Ljava/lang/String;)V", &set_plugins_dir_method)); + + Jni::LocalString trino_connector_plugin_path; + RETURN_IF_ERROR(Jni::LocalString::new_string( + env, doris::config::trino_connector_plugin_dir.c_str(), &trino_connector_plugin_path)); + + return plugin_loader_cls.call_static_void_method(env, set_plugins_dir_method) + .with_arg(trino_connector_plugin_path) + .call(); +} + +} // namespace doris::format::trino_connector diff --git a/be/src/format_v2/jni/trino_connector_jni_reader.h b/be/src/format_v2/jni/trino_connector_jni_reader.h new file mode 100644 index 00000000000000..a20c3a5f62ef96 --- /dev/null +++ b/be/src/format_v2/jni/trino_connector_jni_reader.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "format_v2/jni/jni_table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::trino_connector { + +class TrinoConnectorJniReader final : public format::JniTableReader { +public: + ~TrinoConnectorJniReader() override = default; + + Status prepare_split(const format::SplitReadOptions& options) override; + +protected: + std::string connector_class() const override; + Status validate_scan_range(const TFileRangeDesc& range) const override; + Status build_scanner_params(std::map* params) const override; + +private: + Status _set_spi_plugins_dir() const; +}; + +} // namespace doris::format::trino_connector diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp new file mode 100644 index 00000000000000..313115e82ce744 --- /dev/null +++ b/be/src/format_v2/json/json_reader.cpp @@ -0,0 +1,1123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/json/json_reader.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "format/file_reader/new_plain_text_line_reader.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "io/file_factory.h" +#include "io/fs/file_reader.h" +#include "io/fs/stream_load_pipe.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" +#include "util/decompressor.h" +#include "util/slice.h" + +namespace doris::format::json { +namespace { + +DataTypePtr json_file_type_from_slot_type(const DataTypePtr& type) { + if (type == nullptr) { + return nullptr; + } + + // Text-like file readers expose CHAR/VARCHAR as STRING and let the table column mapper cast to + // the destination slot type. JSON follows the same file-schema convention so that v2 mapping + // behaves consistently across text formats. + const bool is_nullable = type->is_nullable(); + const auto nested_type = remove_nullable(type); + DataTypePtr file_type; + switch (nested_type->get_primitive_type()) { + case TYPE_CHAR: + case TYPE_VARCHAR: + file_type = std::make_shared(); + break; + case TYPE_ARRAY: { + const auto* array_type = assert_cast(nested_type.get()); + file_type = std::make_shared( + json_file_type_from_slot_type(array_type->get_nested_type())); + break; + } + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + file_type = std::make_shared( + json_file_type_from_slot_type(map_type->get_key_type()), + json_file_type_from_slot_type(map_type->get_value_type())); + break; + } + case TYPE_STRUCT: { + const auto* struct_type = assert_cast(nested_type.get()); + DataTypes file_children; + file_children.reserve(struct_type->get_elements().size()); + for (const auto& child_type : struct_type->get_elements()) { + file_children.push_back(json_file_type_from_slot_type(child_type)); + } + file_type = + std::make_shared(file_children, struct_type->get_element_names()); + break; + } + default: + file_type = nested_type; + break; + } + + return is_nullable ? make_nullable(file_type) : file_type; +} + +ColumnDefinition synthetic_file_child(const std::string& name, DataTypePtr type, int32_t local_id); + +std::vector synthesize_file_children_from_type(const DataTypePtr& type) { + std::vector children; + if (type == nullptr) { + return children; + } + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_ARRAY: { + const auto* array_type = assert_cast(nested_type.get()); + children.push_back(synthetic_file_child("element", array_type->get_nested_type(), 0)); + break; + } + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + children.push_back(synthetic_file_child("key", map_type->get_key_type(), 0)); + children.push_back(synthetic_file_child("value", map_type->get_value_type(), 1)); + break; + } + case TYPE_STRUCT: { + const auto* struct_type = assert_cast(nested_type.get()); + children.reserve(struct_type->get_elements().size()); + for (size_t idx = 0; idx < struct_type->get_elements().size(); ++idx) { + children.push_back(synthetic_file_child(struct_type->get_element_name(idx), + struct_type->get_element(idx), + cast_set(idx))); + } + break; + } + default: + break; + } + return children; +} + +ColumnDefinition synthetic_file_child(const std::string& name, DataTypePtr type, int32_t local_id) { + ColumnDefinition child; + child.identifier = Field::create_field(name); + child.local_id = local_id; + child.name = name; + child.type = std::move(type); + child.children = synthesize_file_children_from_type(child.type); + return child; +} + +std::string lower_key(std::string_view key) { + std::string lowered(key.data(), key.size()); + std::transform(lowered.begin(), lowered.end(), lowered.begin(), ::tolower); + return lowered; +} + +} // namespace + +JsonReader::JsonReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileScanRangeParams* scan_params, const TFileRangeDesc& range, + const std::vector& file_slot_descs, + TFileCompressType::type range_compress_type, + std::optional stream_load_id) + : FileReader(system_properties, file_description, std::move(io_ctx), profile), + _scan_params(scan_params), + _range(range), + _source_file_slot_descs(file_slot_descs), + _range_compress_type(range_compress_type), + _stream_load_id(std::move(stream_load_id)) {} + +JsonReader::~JsonReader() { + static_cast(close()); +} + +Status JsonReader::init(RuntimeState* state) { + _runtime_state = state; + if (_scan_params == nullptr) { + return Status::InvalidArgument("JSON v2 reader requires scan params"); + } + if (_file_description == nullptr) { + return Status::InvalidArgument("JSON v2 reader requires file description"); + } + if (_runtime_state == nullptr) { + return Status::InvalidArgument("JSON v2 reader requires runtime state"); + } + if (!_scan_params->__isset.file_attributes) { + return Status::InvalidArgument("JSON v2 reader requires file attributes"); + } + + const auto& attributes = _scan_params->file_attributes; + if (attributes.__isset.text_params && attributes.text_params.__isset.line_delimiter) { + _line_delimiter = attributes.text_params.line_delimiter; + } else { + _line_delimiter = "\n"; + } + _line_delimiter_length = _line_delimiter.size(); + _jsonpaths = attributes.__isset.jsonpaths ? attributes.jsonpaths : ""; + _json_root = attributes.__isset.json_root ? attributes.json_root : ""; + _read_json_by_line = attributes.__isset.read_json_by_line && attributes.read_json_by_line; + _strip_outer_array = attributes.__isset.strip_outer_array && attributes.strip_outer_array; + _num_as_string = attributes.__isset.num_as_string && attributes.num_as_string; + _fuzzy_parse = attributes.__isset.fuzzy_parse && attributes.fuzzy_parse; + _openx_json_ignore_malformed = attributes.__isset.openx_json_ignore_malformed && + attributes.openx_json_ignore_malformed; + _is_hive_table = _range.table_format_params.table_format_type == "hive"; + _file_compress_type = _range_compress_type != TFileCompressType::UNKNOWN + ? _range_compress_type + : _scan_params->compress_type; + + _source_serdes = create_data_type_serdes(_source_file_slot_descs); + _file_schema.clear(); + _file_schema.reserve(_source_file_slot_descs.size()); + // JSON has no physical footer schema. The FE file slots are therefore the authoritative schema + // for both field names and source local ids. + for (size_t idx = 0; idx < _source_file_slot_descs.size(); ++idx) { + const auto* slot = _source_file_slot_descs[idx]; + DORIS_CHECK(slot != nullptr); + ColumnDefinition field; + field.identifier = Field::create_field(slot->col_name()); + field.local_id = cast_set(idx); + field.name = slot->col_name(); + field.type = json_file_type_from_slot_type(slot->get_data_type_ptr()); + field.children = synthesize_file_children_from_type(field.type); + _file_schema.push_back(std::move(field)); + } + _eof = false; + return Status::OK(); +} + +Status JsonReader::get_schema(std::vector* file_schema) const { + if (file_schema == nullptr) { + return Status::InvalidArgument("JSON v2 file_schema is null"); + } + *file_schema = _file_schema; + return Status::OK(); +} + +std::unique_ptr JsonReader::create_column_mapper( + TableColumnMapperOptions options) const { + return std::make_unique(std::move(options)); +} + +Status JsonReader::open(std::shared_ptr request) { + RETURN_IF_ERROR(FileReader::open(std::move(request))); + DORIS_CHECK(_request != nullptr); + RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); + _slot_name_to_index.clear(); + _slot_name_to_index.reserve(_requested_columns.size()); + for (size_t idx = 0; idx < _requested_columns.size(); ++idx) { + auto name = _requested_columns[idx].slot_desc->col_name(); + _slot_name_to_index.emplace(_is_hive_table ? lower_key(name) : name, idx); + } + _previous_positions.clear(); + _reader_range = _json_range(); + RETURN_IF_ERROR(_open_file_reader()); + RETURN_IF_ERROR(_create_decompressor()); + if (_read_json_by_line) { + RETURN_IF_ERROR(_create_line_reader()); + } + RETURN_IF_ERROR(_parse_jsonpath_and_json_root()); + _json_parser = std::make_unique(); + _padding_buffer.resize(_padded_size); + _reader_eof = false; + _single_document_read = false; + _eof = false; + return Status::OK(); +} + +Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + if (_json_parser == nullptr || _physical_file_reader == nullptr) { + return Status::InternalError("JSON v2 reader is not open"); + } + + const auto batch_size = _runtime_state->batch_size(); + const auto max_block_bytes = _runtime_state->preferred_block_size_bytes(); + *rows = 0; + *eof = false; + + while (file_block->rows() < batch_size && !_reader_eof && + file_block->bytes() < max_block_bytes) { + if (_read_json_by_line && _skip_first_line) { + size_t skipped_size = 0; + const uint8_t* skipped_line = nullptr; + RETURN_IF_ERROR(_line_reader->read_line(&skipped_line, &skipped_size, &_reader_eof, + _io_ctx.get())); + _skip_first_line = false; + continue; + } + + const size_t original_rows = file_block->rows(); + size_t size = 0; + bool is_empty_row = false; + Status st = Status::OK(); + try { + st = _parse_next_json(&size, &_reader_eof); + if (st.ok() && !_reader_eof) { + if (size == 0) { + is_empty_row = true; + } else { + st = _extract_json_value(size, &_reader_eof, &is_empty_row); + } + } + if (st.ok() && !_reader_eof && !is_empty_row) { + st = _append_rows_from_current_value(file_block, &is_empty_row, &_reader_eof); + } + } catch (simdjson::simdjson_error& e) { + st = Status::DataQualityError("Parse json data failed. code: {}, error info: {}", + e.error(), e.what()); + } + if (!st.ok()) { + RETURN_IF_ERROR(_handle_json_error(st, file_block, original_rows, &is_empty_row)); + } + // An ignored or empty JSON object can produce no row. Avoid spinning forever on a document + // that was consumed but produced no materialized value. + if (!is_empty_row && file_block->rows() == original_rows) { + break; + } + } + + *rows = file_block->rows(); + _record_scan_rows(cast_set(*rows)); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + *eof = _reader_eof && *rows == 0; + _eof = *eof; + return Status::OK(); +} + +Status JsonReader::close() { + if (_line_reader != nullptr) { + _line_reader->close(); + _line_reader.reset(); + } + _json_parser.reset(); + _decompressor.reset(); + _physical_file_reader.reset(); + _tracing_file_reader.reset(); + _file_reader.reset(); + _requested_columns.clear(); + _slot_name_to_index.clear(); + _previous_positions.clear(); + _cached_string_values.clear(); + return Status::OK(); +} + +Status JsonReader::_build_requested_columns(const FileScanRequest& request, + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + // FileScanRequest stores a map from file-local id to output block position. Materialization is + // position-driven, so normalize it into a dense vector ordered by block position while keeping + // the original source index for jsonpaths. + std::vector by_position(request.local_positions.size()); + for (const auto& [file_column_id, block_position] : request.local_positions) { + if (file_column_id.value() < 0 || + static_cast(file_column_id.value()) >= _source_file_slot_descs.size()) { + return Status::InvalidArgument("JSON v2 request references unknown local column id {}", + file_column_id.value()); + } + if (block_position.value() >= by_position.size()) { + return Status::InvalidArgument("JSON v2 request has invalid block position {}", + block_position.value()); + } + const auto source_index = cast_set(file_column_id.value()); + RequestedColumn requested_column; + requested_column.file_column_id = file_column_id; + requested_column.block_position = block_position; + requested_column.source_index = source_index; + requested_column.slot_desc = _source_file_slot_descs[source_index]; + requested_column.serde = _source_serdes[source_index]; + by_position[block_position.value()] = std::move(requested_column); + } + for (size_t pos = 0; pos < by_position.size(); ++pos) { + if (!by_position[pos].file_column_id.is_valid()) { + return Status::InvalidArgument("JSON v2 request misses block position {}", pos); + } + } + *columns = std::move(by_position); + return Status::OK(); +} + +TFileRangeDesc JsonReader::_json_range() const { + auto range = _range; + range.__set_path(_file_description->path); + range.__set_start_offset(_file_description->range_start_offset); + range.__set_size(_file_description->range_size); + if (_file_description->file_size >= 0) { + range.__set_file_size(_file_description->file_size); + } + if (!_file_description->fs_name.empty()) { + range.__set_fs_name(_file_description->fs_name); + } + range.__set_file_cache_admission(_file_description->file_cache_admission); + if (_range_compress_type != TFileCompressType::UNKNOWN) { + range.__set_compress_type(_range_compress_type); + } + if (_stream_load_id.has_value()) { + range.__set_load_id(*_stream_load_id); + } + return range; +} + +Status JsonReader::_open_file_reader() { + _current_offset = _reader_range.start_offset; + if (_current_offset != 0) { + --_current_offset; + } + if (_scan_params->file_type == TFileType::FILE_STREAM) { + if (!_stream_load_id.has_value()) { + return Status::InvalidArgument("JSON v2 stream reader requires load id"); + } + RETURN_IF_ERROR(FileFactory::create_pipe_reader(*_stream_load_id, &_physical_file_reader, + _runtime_state, /*need_schema=*/false)); + } else { + _file_description->mtime = + _reader_range.__isset.modification_time ? _reader_range.modification_time : 0; + auto reader_options = FileFactory::get_reader_options(_runtime_state->query_options(), + *_file_description); + auto file_reader = DORIS_TRY(FileFactory::create_file_reader( + *_system_properties, *_file_description, reader_options, _profile)); + _physical_file_reader = + _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(std::move(file_reader), + _io_ctx->file_reader_stats) + : file_reader; + } + _file_reader = _physical_file_reader; + _tracing_file_reader = _physical_file_reader; + return Status::OK(); +} + +Status JsonReader::_create_decompressor() { + return Decompressor::create_decompressor(_file_compress_type, &_decompressor); +} + +Status JsonReader::_create_line_reader() { + int64_t size = _reader_range.size; + if (_reader_range.start_offset != 0) { + // Start one byte earlier and discard the first partial line, matching split semantics used + // by text readers. + ++size; + _skip_first_line = true; + } else { + _skip_first_line = false; + } + _line_reader = NewPlainTextLineReader::create_unique( + _profile, _physical_file_reader, _decompressor.get(), + std::make_shared(_line_delimiter, _line_delimiter_length, + false), + size, _current_offset); + return Status::OK(); +} + +Status JsonReader::_parse_jsonpath_and_json_root() { + _parsed_jsonpaths.clear(); + _parsed_json_root.clear(); + if (!_jsonpaths.empty()) { + rapidjson::Document jsonpaths_doc; + if (jsonpaths_doc.Parse(_jsonpaths.c_str(), _jsonpaths.length()).HasParseError() || + !jsonpaths_doc.IsArray()) { + return Status::InvalidJsonPath("Invalid json path: {}", _jsonpaths); + } + for (int i = 0; i < jsonpaths_doc.Size(); ++i) { + const rapidjson::Value& path = jsonpaths_doc[i]; + if (!path.IsString()) { + return Status::InvalidJsonPath("Invalid json path: {}", _jsonpaths); + } + std::string json_path = path.GetString(); + if (json_path.size() == 1 && json_path[0] == '$') { + json_path.insert(1, "."); + } + std::vector parsed_paths; + JsonFunctions::parse_json_paths(json_path, &parsed_paths); + _parsed_jsonpaths.push_back(std::move(parsed_paths)); + } + } + if (!_json_root.empty()) { + std::string json_root = _json_root; + if (json_root.size() == 1 && json_root[0] == '$') { + json_root.insert(1, "."); + } + JsonFunctions::parse_json_paths(json_root, &_parsed_json_root); + } + return Status::OK(); +} + +Status JsonReader::_read_one_document(size_t* size, bool* eof) { + DORIS_CHECK(size != nullptr); + DORIS_CHECK(eof != nullptr); + *size = 0; + *eof = false; + if (_line_reader != nullptr) { + const uint8_t* line = nullptr; + RETURN_IF_ERROR(_line_reader->read_line(&line, size, eof, _io_ctx.get())); + if (*eof) { + return Status::OK(); + } + _document_buffer.assign(reinterpret_cast(line), *size); + return Status::OK(); + } + // Non-line mode treats the split as one JSON document. This supports a single object or an + // array with strip_outer_array=true. + if (_single_document_read) { + *eof = true; + return Status::OK(); + } + _single_document_read = true; + if (_scan_params->file_type == TFileType::FILE_STREAM) { + return _read_one_document_from_pipe(size); + } + + auto read_size = _reader_range.size; + if (read_size <= 0 && _reader_range.__isset.file_size) { + read_size = _reader_range.file_size - _current_offset; + } + if (read_size <= 0) { + *eof = true; + return Status::OK(); + } + _document_buffer.resize(cast_set(read_size)); + Slice result(_document_buffer.data(), _document_buffer.size()); + RETURN_IF_ERROR(_physical_file_reader->read_at(_current_offset, result, size, _io_ctx.get())); + _document_buffer.resize(*size); + if (*size == 0) { + *eof = true; + } + return Status::OK(); +} + +Status JsonReader::_read_one_document_from_pipe(size_t* read_size) { + auto* stream_load_pipe = dynamic_cast(_physical_file_reader.get()); + if (stream_load_pipe == nullptr) { + return Status::InternalError("JSON v2 stream reader requires StreamLoadPipe"); + } + DorisUniqueBufferPtr file_buf; + RETURN_IF_ERROR(stream_load_pipe->read_one_message(&file_buf, read_size)); + _document_buffer.assign(reinterpret_cast(file_buf.get()), *read_size); + if (!stream_load_pipe->is_chunked_transfer()) { + return Status::OK(); + } + + while (true) { + DorisUniqueBufferPtr next_buf; + size_t next_size = 0; + RETURN_IF_ERROR(stream_load_pipe->read_one_message(&next_buf, &next_size)); + if (next_size == 0) { + break; + } + _document_buffer.append(reinterpret_cast(next_buf.get()), next_size); + *read_size += next_size; + } + return Status::OK(); +} + +Status JsonReader::_parse_next_json(size_t* size, bool* eof) { + RETURN_IF_ERROR(_read_one_document(size, eof)); + if (*eof || *size == 0) { + return Status::OK(); + } + if (*size >= 3 && static_cast(_document_buffer[0]) == 0xEF && + static_cast(_document_buffer[1]) == 0xBB && + static_cast(_document_buffer[2]) == 0xBF) { + _document_buffer.erase(0, 3); + *size -= 3; + } + if (*size + simdjson::SIMDJSON_PADDING > _padded_size) { + _padded_size = *size + simdjson::SIMDJSON_PADDING; + _padding_buffer.resize(_padded_size); + } + // Ondemand values reference the input buffer. Keep the padded bytes in a member buffer until the + // current document is fully materialized. + std::memcpy(_padding_buffer.data(), _document_buffer.data(), *size); + _original_doc_size = *size; + const auto error = + _json_parser->iterate(std::string_view(_padding_buffer.data(), *size), _padded_size) + .get(_original_json_doc); + if (error != simdjson::error_code::SUCCESS) { + return Status::DataQualityError( + "Parse json data for JsonDoc failed. code: {}, error info: {}", error, + simdjson::error_message(error)); + } + return Status::OK(); +} + +Status JsonReader::_extract_json_value(size_t size, bool* eof, bool* is_empty_row) { + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(is_empty_row != nullptr); + *is_empty_row = false; + if (size == 0 || *eof) { + *is_empty_row = true; + return Status::OK(); + } + auto type_res = _original_json_doc.type(); + if (type_res.error() != simdjson::error_code::SUCCESS) { + return Status::DataQualityError( + "Parse json data for JsonDoc failed. code: {}, error info: {}", type_res.error(), + simdjson::error_message(type_res.error())); + } + const auto type = type_res.value(); + if (type != simdjson::ondemand::json_type::object && + type != simdjson::ondemand::json_type::array) { + return Status::DataQualityError("Not an json object or json array"); + } + _parsed_from_json_root = false; + if (!_parsed_json_root.empty() && type == simdjson::ondemand::json_type::object) { + // In object mode json_root can be applied once here. In outer-array mode each array element + // needs its own root extraction, which is handled while iterating the array. + simdjson::ondemand::object object = _original_json_doc; + Status st = JsonFunctions::extract_from_object(object, _parsed_json_root, &_json_value); + if (!st.ok()) { + return Status::DataQualityError("{}", st.to_string()); + } + _parsed_from_json_root = true; + } else { + _json_value = _original_json_doc; + } + + const auto value_type = _json_value.type().value(); + if (value_type == simdjson::ondemand::json_type::array && !_strip_outer_array) { + return Status::DataQualityError( + "JSON data is array-object, `strip_outer_array` must be TRUE."); + } + if (value_type != simdjson::ondemand::json_type::array && _strip_outer_array) { + return Status::DataQualityError( + "JSON data is not an array-object, `strip_outer_array` must be FALSE."); + } + if (!_parsed_jsonpaths.empty() && _strip_outer_array && + _json_value.count_elements().value() == 0) { + *is_empty_row = true; + } + return Status::OK(); +} + +Status JsonReader::_append_rows_from_current_value(Block* block, bool* is_empty_row, bool* eof) { + if (_parsed_jsonpaths.empty()) { + return _append_simple_json_rows(block, is_empty_row, eof); + } + if (_strip_outer_array) { + return _append_flat_array_jsonpath_rows(block, is_empty_row, eof); + } + return _append_nested_jsonpath_row(block, is_empty_row, eof); +} + +Status JsonReader::_append_simple_json_rows(Block* block, bool* is_empty_row, bool* eof) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(is_empty_row != nullptr); + DORIS_CHECK(eof != nullptr); + bool valid = false; + if (_json_value.type().value() == simdjson::ondemand::json_type::array) { + _array = _json_value.get_array(); + if (_array.count_elements() == 0) { + *is_empty_row = true; + return Status::OK(); + } + _array_iter = _array.begin(); + while (_array_iter != _array.end()) { + simdjson::ondemand::object object_value = (*_array_iter).get_object(); + RETURN_IF_ERROR(_set_column_values_from_object(&object_value, block, &valid)); + ++_array_iter; + if (!valid) { + *is_empty_row = true; + return Status::OK(); + } + } + } else { + simdjson::ondemand::object object_value = _json_value.get_object(); + RETURN_IF_ERROR(_set_column_values_from_object(&object_value, block, &valid)); + if (!valid) { + *is_empty_row = true; + return Status::OK(); + } + } + *is_empty_row = false; + return Status::OK(); +} + +Status JsonReader::_append_flat_array_jsonpath_rows(Block* block, bool* is_empty_row, bool* eof) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(is_empty_row != nullptr); + DORIS_CHECK(eof != nullptr); + const size_t original_rows = block->rows(); + bool valid = true; + _array = _json_value.get_array(); + _array_iter = _array.begin(); + while (_array_iter != _array.end()) { + simdjson::ondemand::object object_value = (*_array_iter).get_object(); + if (!_parsed_from_json_root && !_parsed_json_root.empty()) { + // For strip_outer_array, json_root is evaluated against each element. Elements without + // the requested root do not produce rows, matching the load reader behavior. + simdjson::ondemand::value rooted_value; + Status st = JsonFunctions::extract_from_object(object_value, _parsed_json_root, + &rooted_value); + if (!st.ok()) { + if (st.is()) { + ++_array_iter; + continue; + } + return st; + } + if (rooted_value.type().value() != simdjson::ondemand::json_type::object) { + ++_array_iter; + continue; + } + object_value = rooted_value.get_object(); + } + RETURN_IF_ERROR(_write_columns_by_jsonpath(&object_value, block, &valid)); + ++_array_iter; + } + *is_empty_row = block->rows() == original_rows; + return Status::OK(); +} + +Status JsonReader::_append_nested_jsonpath_row(Block* block, bool* is_empty_row, bool* eof) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(is_empty_row != nullptr); + DORIS_CHECK(eof != nullptr); + if (_json_value.type().value() != simdjson::ondemand::json_type::object) { + return Status::DataQualityError("Not object item"); + } + bool valid = true; + simdjson::ondemand::object object_value = _json_value.get_object(); + RETURN_IF_ERROR(_write_columns_by_jsonpath(&object_value, block, &valid)); + *is_empty_row = !valid; + return Status::OK(); +} + +Status JsonReader::_set_column_values_from_object(simdjson::ondemand::object* object_value, + Block* block, bool* valid) { + DORIS_CHECK(object_value != nullptr); + DORIS_CHECK(block != nullptr); + DORIS_CHECK(valid != nullptr); + std::vector seen_columns(block->columns(), false); + const size_t cur_row_count = block->rows(); + bool has_valid_value = false; + size_t key_index = 0; + + for (auto field : *object_value) { + std::string_view key = field.unescaped_key().value(); + const size_t column_index = _column_index(key, key_index++); + if (column_index == static_cast(-1)) { + continue; + } + if (seen_columns[column_index]) { + if (_is_hive_table) { + // Hive JSON keeps the last duplicate key ignoring case. The earlier value has + // already been appended, so remove it before writing the replacement. + _pop_back_last_inserted_value(block, column_index); + } else { + continue; + } + } + simdjson::ondemand::value value = field.value().value(); + const auto& requested = _requested_columns[column_index]; + auto* column_ptr = block->get_by_position(column_index).column->assert_mutable().get(); + RETURN_IF_ERROR(_write_data_to_column( + value, requested.slot_desc->get_data_type_ptr(), column_ptr, + requested.slot_desc->col_name(), requested.serde, valid)); + if (!*valid) { + return Status::OK(); + } + seen_columns[column_index] = true; + has_valid_value = true; + } + + for (size_t i = 0; i < _requested_columns.size(); ++i) { + if (seen_columns[i]) { + continue; + } + auto* column_ptr = block->get_by_position(i).column->assert_mutable().get(); + RETURN_IF_ERROR(_fill_missing_column(_requested_columns[i], column_ptr, valid)); + if (!*valid) { + _truncate_block_to_rows(block, cur_row_count); + return Status::OK(); + } + } + *valid = true; + if (!has_valid_value) { + return Status::OK(); + } + return Status::OK(); +} + +Status JsonReader::_write_columns_by_jsonpath(simdjson::ondemand::object* object_value, + Block* block, bool* valid) { + DORIS_CHECK(object_value != nullptr); + DORIS_CHECK(block != nullptr); + DORIS_CHECK(valid != nullptr); + bool has_valid_value = false; + const size_t cur_row_count = block->rows(); + _cached_string_values.clear(); + + for (size_t i = 0; i < _requested_columns.size(); ++i) { + const auto& requested = _requested_columns[i]; + auto* column_ptr = block->get_by_position(i).column->assert_mutable().get(); + simdjson::ondemand::value json_value; + Status st = Status::OK(); + if (requested.source_index < _parsed_jsonpaths.size()) { + st = JsonFunctions::extract_from_object( + *object_value, _parsed_jsonpaths[requested.source_index], &json_value); + if (!st.ok() && !st.is()) { + return st; + } + } + if (_is_root_path_for_column(requested)) { + // A root jsonpath means "materialize the whole current JSON document" instead of a + // field under it. Use the original bytes so callers receive the same document text. + if (is_column_nullable(*column_ptr)) { + auto* nullable_column = assert_cast(column_ptr); + nullable_column->get_null_map_data().push_back(0); + auto* column_string = + assert_cast(nullable_column->get_nested_column_ptr().get()); + column_string->insert_data(_padding_buffer.data(), _original_doc_size); + } else { + auto* column_string = assert_cast(column_ptr); + column_string->insert_data(_padding_buffer.data(), _original_doc_size); + } + has_valid_value = true; + } else if (requested.source_index >= _parsed_jsonpaths.size() || + st.is()) { + RETURN_IF_ERROR(_fill_missing_column(requested, column_ptr, valid)); + if (!*valid) { + _truncate_block_to_rows(block, cur_row_count); + return Status::OK(); + } + } else { + RETURN_IF_ERROR(_write_data_to_column( + json_value, requested.slot_desc->get_data_type_ptr(), column_ptr, + requested.slot_desc->col_name(), requested.serde, valid)); + if (!*valid) { + _truncate_block_to_rows(block, cur_row_count); + return Status::OK(); + } + has_valid_value = true; + } + } + + if (!has_valid_value) { + // jsonpaths can legally match nothing. Roll the row back so an all-missing path set does + // not create a synthetic row of nulls. + _truncate_block_to_rows(block, cur_row_count); + *valid = false; + return Status::OK(); + } + *valid = true; + return Status::OK(); +} + +template +Status JsonReader::_write_data_to_column(simdjson::ondemand::value& value, + const DataTypePtr& type_desc, IColumn* column_ptr, + const std::string& column_name, + const DataTypeSerDeSPtr& serde, bool* valid) { + ColumnNullable* nullable_column = nullptr; + IColumn* data_column_ptr = column_ptr; + DataTypeSerDeSPtr data_serde = serde; + const auto value_type = value.type().value(); + + if (is_column_nullable(*column_ptr)) { + nullable_column = assert_cast(column_ptr); + data_column_ptr = nullable_column->get_nested_column().get_ptr().get(); + if (type_desc->is_nullable()) { + data_serde = serde->get_nested_serdes()[0]; + } + if (value_type == simdjson::ondemand::json_type::null) { + nullable_column->insert_default(); + *valid = true; + return Status::OK(); + } + } else if (value_type == simdjson::ondemand::json_type::null) { + return Status::DataQualityError("Json value is null, but the column `{}` is not nullable.", + column_name); + } + + const auto primitive_type = type_desc->get_primitive_type(); + if (!is_complex_type(primitive_type)) { + if (value_type == simdjson::ondemand::json_type::string) { + std::string_view value_string; + if constexpr (use_string_cache) { + const auto cache_key = value.raw_json().value(); + if (_cached_string_values.contains(cache_key)) { + value_string = _cached_string_values[cache_key]; + } else { + value_string = value.get_string(); + _cached_string_values.emplace(cache_key, value_string); + } + } else { + value_string = value.get_string(); + } + Slice slice {value_string.data(), value_string.size()}; + RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice, + _serde_options)); + } else if (value_type == simdjson::ondemand::json_type::boolean) { + const char* str_value = value.get_bool() ? "1" : "0"; + Slice slice {str_value, 1}; + RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice, + _serde_options)); + } else { + std::string_view json_str = simdjson::to_json_string(value); + Slice slice {json_str.data(), json_str.size()}; + RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice, + _serde_options)); + } + } else if (primitive_type == TYPE_STRUCT) { + if (value_type != simdjson::ondemand::json_type::object) { + return Status::DataQualityError( + "Json value isn't object, but the column `{}` is struct.", column_name); + } + const auto* type_struct = + assert_cast(remove_nullable(type_desc).get()); + auto* struct_column_ptr = assert_cast(data_column_ptr); + const auto sub_serdes = data_serde->get_nested_serdes(); + std::map sub_col_name_to_idx; + for (size_t sub_col_idx = 0; sub_col_idx < type_struct->get_elements().size(); + ++sub_col_idx) { + sub_col_name_to_idx.emplace(lower_key(type_struct->get_element_name(sub_col_idx)), + sub_col_idx); + } + std::vector has_value(type_struct->get_elements().size(), false); + simdjson::ondemand::object struct_value = value.get_object(); + for (auto sub : struct_value) { + const auto sub_key = lower_key(sub.unescaped_key().value()); + const auto it = sub_col_name_to_idx.find(sub_key); + if (it == sub_col_name_to_idx.end()) { + continue; + } + const auto sub_column_idx = it->second; + auto sub_column_ptr = struct_column_ptr->get_column(sub_column_idx).get_ptr(); + if (has_value[sub_column_idx]) { + // Struct fields follow Hive-style duplicate handling: the last matching nested key + // wins. Remove the earlier nested value before appending the new one. + sub_column_ptr->pop_back(1); + } + has_value[sub_column_idx] = true; + auto sub_value = sub.value().value(); + RETURN_IF_ERROR(_write_data_to_column( + sub_value, type_struct->get_element(sub_column_idx), sub_column_ptr.get(), + column_name + "." + sub_key, sub_serdes[sub_column_idx], valid)); + } + for (size_t sub_col_idx = 0; sub_col_idx < type_struct->get_elements().size(); + ++sub_col_idx) { + if (has_value[sub_col_idx]) { + continue; + } + auto sub_column_ptr = struct_column_ptr->get_column(sub_col_idx).get_ptr(); + if (!is_column_nullable(*sub_column_ptr)) { + return Status::DataQualityError( + "Json file structColumn miss field {} and this column isn't nullable.", + column_name + "." + type_struct->get_element_name(sub_col_idx)); + } + sub_column_ptr->insert_default(); + } + } else if (primitive_type == TYPE_MAP) { + if (value_type != simdjson::ondemand::json_type::object) { + return Status::DataQualityError("Json value isn't object, but the column `{}` is map.", + column_name); + } + const auto* map_type = assert_cast(remove_nullable(type_desc).get()); + auto* map_column_ptr = assert_cast(data_column_ptr); + const auto sub_serdes = data_serde->get_nested_serdes(); + size_t field_count = 0; + simdjson::ondemand::object object_value = value.get_object(); + for (auto member_value : object_value) { + auto* key_column = map_column_ptr->get_keys_ptr()->assert_mutable()->get_ptr().get(); + auto key_serde = sub_serdes[0]; + if (is_column_nullable(*key_column)) { + auto* nullable_key = assert_cast(key_column); + nullable_key->get_null_map_data().push_back(0); + key_column = nullable_key->get_nested_column().get_ptr().get(); + if (map_type->get_key_type()->is_nullable()) { + key_serde = key_serde->get_nested_serdes()[0]; + } + } + std::string_view key_view = member_value.unescaped_key().value(); + Slice key_slice(key_view.data(), key_view.size()); + RETURN_IF_ERROR(key_serde->deserialize_one_cell_from_json(*key_column, key_slice, + _serde_options)); + simdjson::ondemand::value field_value = member_value.value().value(); + RETURN_IF_ERROR(_write_data_to_column( + field_value, map_type->get_value_type(), + map_column_ptr->get_values_ptr()->assert_mutable()->get_ptr().get(), + column_name + ".value", sub_serdes[1], valid)); + ++field_count; + } + auto& offsets = map_column_ptr->get_offsets(); + offsets.emplace_back(offsets.back() + field_count); + } else if (primitive_type == TYPE_ARRAY) { + if (value_type != simdjson::ondemand::json_type::array) { + return Status::DataQualityError("Json value isn't array, but the column `{}` is array.", + column_name); + } + const auto* array_type = + assert_cast(remove_nullable(type_desc).get()); + auto* array_column_ptr = assert_cast(data_column_ptr); + const auto sub_serdes = data_serde->get_nested_serdes(); + size_t field_count = 0; + simdjson::ondemand::array array_value = value.get_array(); + for (simdjson::ondemand::value sub_value : array_value) { + RETURN_IF_ERROR(_write_data_to_column( + sub_value, array_type->get_nested_type(), + array_column_ptr->get_data().get_ptr().get(), column_name + ".element", + sub_serdes[0], valid)); + ++field_count; + } + auto& offsets = array_column_ptr->get_offsets(); + offsets.emplace_back(offsets.back() + field_count); + } else { + return Status::InternalError("Not support JSON value to complex column"); + } + + if (nullable_column && value_type != simdjson::ondemand::json_type::null) { + nullable_column->get_null_map_data().push_back(0); + } + *valid = true; + return Status::OK(); +} + +Status JsonReader::_fill_missing_column(const RequestedColumn& column, IColumn* column_ptr, + bool* valid) { + if (column.slot_desc->is_nullable()) { + auto* nullable_column = assert_cast(column_ptr); + nullable_column->insert_default(); + *valid = true; + return Status::OK(); + } + return Status::DataQualityError( + "The column `{}` is not nullable, but it's not found in jsondata.", + column.slot_desc->col_name()); +} + +Status JsonReader::_append_null_for_malformed_json(Block* block) { + DORIS_CHECK(block != nullptr); + for (int i = 0; i < block->columns(); ++i) { + auto& column_with_type = block->get_by_position(i); + if (!is_column_nullable(*column_with_type.column)) { + return Status::DataQualityError("malformed json, but the column `{}` is not nullable.", + column_with_type.column->get_name()); + } + auto column = IColumn::mutate(std::move(column_with_type.column)); + assert_cast(column.get())->insert_default(); + column_with_type.column = std::move(column); + } + return Status::OK(); +} + +Status JsonReader::_handle_json_error(const Status& status, Block* block, size_t original_rows, + bool* is_empty_row) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(is_empty_row != nullptr); + // Deserialization can fail after several columns have already appended data. Always restore the + // block to the row count before this document before either surfacing the error or appending + // the ignore-malformed null row. + _truncate_block_to_rows(block, original_rows); + if (_openx_json_ignore_malformed && status.is()) { + RETURN_IF_ERROR(_append_null_for_malformed_json(block)); + *is_empty_row = false; + return Status::OK(); + } + return status; +} + +Status JsonReader::_apply_filters(Block* file_block, size_t* rows) { + return apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows); +} + +void JsonReader::_truncate_block_to_rows(Block* block, size_t num_rows) { + DORIS_CHECK(block != nullptr); + for (int i = 0; i < block->columns(); ++i) { + auto& column_with_type = block->get_by_position(i); + auto column = IColumn::mutate(std::move(column_with_type.column)); + if (column->size() > num_rows) { + column->pop_back(column->size() - num_rows); + } + column_with_type.column = std::move(column); + } +} + +void JsonReader::_pop_back_last_inserted_value(Block* block, size_t column_index) { + DORIS_CHECK(block != nullptr); + auto& column = block->get_by_position(column_index).column; + auto mutable_column = IColumn::mutate(std::move(column)); + mutable_column->pop_back(1); + column = std::move(mutable_column); +} + +size_t JsonReader::_column_index(std::string_view key, size_t key_index) { + std::string hive_key; + std::string_view lookup_key = key; + if (_is_hive_table) { + hive_key = lower_key(key); + lookup_key = hive_key; + } + if (key_index < _previous_positions.size()) { + // Most JSON lines share field order. Reuse the previous line's key-position mapping before + // falling back to the hash table lookup. + const auto previous = _previous_positions[key_index]; + if (previous < _requested_columns.size()) { + const auto previous_name = _requested_columns[previous].slot_desc->col_name(); + if ((_is_hive_table ? lower_key(previous_name) : previous_name) == lookup_key) { + return previous; + } + } + } + const auto it = _slot_name_to_index.find(std::string(lookup_key)); + if (it == _slot_name_to_index.end()) { + return static_cast(-1); + } + if (key_index >= _previous_positions.size()) { + _previous_positions.resize(key_index + 1, static_cast(-1)); + } + _previous_positions[key_index] = it->second; + return it->second; +} + +bool JsonReader::_is_root_path_for_column(const RequestedColumn& column) const { + return column.source_index < _parsed_jsonpaths.size() && + JsonFunctions::is_root_path(_parsed_jsonpaths[column.source_index]); +} + +} // namespace doris::format::json diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h new file mode 100644 index 00000000000000..52cdfad6728d64 --- /dev/null +++ b/be/src/format_v2/json/json_reader.h @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "core/data_type_serde/data_type_serde.h" +#include "exprs/json_functions.h" +#include "format_v2/file_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "runtime/runtime_profile.h" + +namespace doris { +class Decompressor; +class LineReader; +class SlotDescriptor; +class IColumn; +} // namespace doris + +namespace doris::format::json { + +// FileScannerV2 JSON reader. +// +// JSON files do not carry an embedded physical schema. The v2 table layer still needs a +// file-local schema and FileScanRequest contract, so this reader exposes FE-provided file slots as +// v2 file-local columns and performs JSON parsing/materialization directly in the v2 path. +class JsonReader final : public FileReader { +public: + // `file_slot_descs` is the FE-planned file schema. JSON has no physical schema, so the reader + // exposes these slots as synthetic file-local columns and materializes only the columns + // requested by FileScanRequest. + JsonReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileScanRangeParams* scan_params, const TFileRangeDesc& range, + const std::vector& file_slot_descs, + TFileCompressType::type range_compress_type = TFileCompressType::UNKNOWN, + std::optional stream_load_id = std::nullopt); + ~JsonReader() override; + + // Initializes scan attributes and builds the synthetic schema from FE slots. + Status init(RuntimeState* state) override; + Status get_schema(std::vector* file_schema) const override; + std::unique_ptr create_column_mapper( + TableColumnMapperOptions options) const override; + // Opens the underlying file or stream and binds requested local column ids to output block + // positions. After this call, `get_block` can be called until it returns eof. + Status open(std::shared_ptr request) override; + // Appends rows into `file_block` according to the FileScanRequest order. The block must already + // contain columns matching the requested positions. + Status get_block(Block* file_block, size_t* rows, bool* eof) override; + Status close() override; + +private: + // A requested column keeps both identities: + // - `source_index`: index in FE file slots, used for jsonpaths and SerDe lookup. + // - `block_position`: index in the caller's output block, used for materialization. + struct RequestedColumn { + LocalColumnId file_column_id = LocalColumnId::invalid(); + LocalIndex block_position; + size_t source_index = 0; + SlotDescriptor* slot_desc = nullptr; + DataTypeSerDeSPtr serde; + }; + + Status _build_requested_columns(const FileScanRequest& request, + std::vector* columns) const; + // Reconciles TableReader's split/range descriptor with FileReader's concrete file description. + TFileRangeDesc _json_range() const; + Status _open_file_reader(); + Status _create_decompressor(); + Status _create_line_reader(); + Status _parse_jsonpath_and_json_root(); + // Reads one logical JSON document: one line for JSON Lines, or the whole range/pipe payload for + // single-document mode. + Status _read_one_document(size_t* size, bool* eof); + Status _read_one_document_from_pipe(size_t* read_size); + // Moves the logical document into a simdjson-padded buffer and creates an ondemand document. + Status _parse_next_json(size_t* size, bool* eof); + // Applies json_root and validates the object/array shape required by strip_outer_array. + Status _extract_json_value(size_t size, bool* eof, bool* is_empty_row); + Status _append_rows_from_current_value(Block* block, bool* is_empty_row, bool* eof); + Status _append_simple_json_rows(Block* block, bool* is_empty_row, bool* eof); + Status _append_flat_array_jsonpath_rows(Block* block, bool* is_empty_row, bool* eof); + Status _append_nested_jsonpath_row(Block* block, bool* is_empty_row, bool* eof); + Status _set_column_values_from_object(simdjson::ondemand::object* object_value, Block* block, + bool* valid); + Status _write_columns_by_jsonpath(simdjson::ondemand::object* object_value, Block* block, + bool* valid); + template + Status _write_data_to_column(simdjson::ondemand::value& value, const DataTypePtr& type_desc, + IColumn* column_ptr, const std::string& column_name, + const DataTypeSerDeSPtr& serde, bool* valid); + Status _fill_missing_column(const RequestedColumn& column, IColumn* column_ptr, bool* valid); + Status _append_null_for_malformed_json(Block* block); + Status _handle_json_error(const Status& status, Block* block, size_t original_rows, + bool* is_empty_row); + Status _apply_filters(Block* file_block, size_t* rows); + void _truncate_block_to_rows(Block* block, size_t num_rows); + void _pop_back_last_inserted_value(Block* block, size_t column_index); + size_t _column_index(std::string_view key, size_t key_index); + bool _is_root_path_for_column(const RequestedColumn& column) const; + + const TFileScanRangeParams* _scan_params = nullptr; + TFileRangeDesc _range; + TFileRangeDesc _reader_range; + std::vector _source_file_slot_descs; + DataTypeSerDeSPtrs _source_serdes; + std::vector _file_schema; + RuntimeState* _runtime_state = nullptr; + TFileCompressType::type _range_compress_type = TFileCompressType::UNKNOWN; + std::optional _stream_load_id; + std::vector _requested_columns; + std::unordered_map _slot_name_to_index; + std::vector _previous_positions; + + io::FileReaderSPtr _physical_file_reader; + std::unique_ptr _decompressor; + std::unique_ptr _line_reader; + int64_t _current_offset = 0; + bool _reader_eof = false; + bool _skip_first_line = false; + bool _single_document_read = false; + + std::string _line_delimiter; + size_t _line_delimiter_length = 0; + std::string _jsonpaths; + std::string _json_root; + bool _read_json_by_line = false; + bool _strip_outer_array = false; + bool _num_as_string = false; + bool _fuzzy_parse = false; + bool _is_hive_table = false; + bool _openx_json_ignore_malformed = false; + TFileCompressType::type _file_compress_type = TFileCompressType::UNKNOWN; + + std::vector> _parsed_jsonpaths; + std::vector _parsed_json_root; + bool _parsed_from_json_root = false; + DataTypeSerDe::FormatOptions _serde_options; + + // simdjson ondemand values point into `_padding_buffer`, so the buffer must outlive all values + // created from the current document. + std::unique_ptr _json_parser; + simdjson::ondemand::document _original_json_doc; + simdjson::ondemand::value _json_value; + simdjson::ondemand::array _array; + simdjson::ondemand::array_iterator _array_iter; + std::string _document_buffer; + std::string _padding_buffer; + size_t _original_doc_size = 0; + size_t _padded_size = 1024 * 1024 * 8 + simdjson::SIMDJSON_PADDING; + std::unordered_map _cached_string_values; +}; + +} // namespace doris::format::json diff --git a/be/src/format_v2/materialized_reader_util.cpp b/be/src/format_v2/materialized_reader_util.cpp new file mode 100644 index 00000000000000..d27f066b30bc2b --- /dev/null +++ b/be/src/format_v2/materialized_reader_util.cpp @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/materialized_reader_util.h" + +#include + +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/types.h" +#include "exprs/vexpr_context.h" +#include "format_v2/file_reader.h" +#include "io/io_common.h" + +namespace doris::format { +namespace { + +void update_counter(RuntimeProfile::Counter* counter, int64_t value) { + if (counter != nullptr) { + COUNTER_UPDATE(counter, value); + } +} + +} // namespace + +ColumnPtr make_column_nullable_if_needed(ColumnPtr column, const DataTypePtr& target_type) { + if (target_type != nullptr && target_type->is_nullable() && column.get() != nullptr && + !column->is_nullable()) { + return make_nullable(std::move(column)); + } + return column; +} + +Status apply_materialized_reader_filters(const FileScanRequest* request, io::IOContext* io_ctx, + Block* file_block, size_t* rows, + const MaterializedReaderFilterProfile* profile) { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + const size_t rows_before_filter = *rows; + size_t rows_after_delete_filter = rows_before_filter; + if (request != nullptr && rows_before_filter > 0 && !request->delete_conjuncts.empty()) { + { + SCOPED_TIMER(profile == nullptr ? nullptr : profile->delete_conjunct_filter_time); + // Delete conjuncts use the opposite polarity from ordinary predicates: TRUE marks a + // row for deletion. Multiple delete files are combined as a union, so a row remains + // visible only when every delete conjunct returns FALSE. + IColumn::Filter keep_filter(rows_before_filter, 1); + for (const auto& delete_conjunct : request->delete_conjuncts) { + DORIS_CHECK(delete_conjunct != nullptr); + int result_column_id = -1; + RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, + &result_column_id)); + DORIS_CHECK(result_column_id >= 0 && + result_column_id < static_cast(file_block->columns())); + const auto& delete_filter = + assert_cast( + *file_block->get_by_position(result_column_id).column) + .get_data(); + DORIS_CHECK(delete_filter.size() == rows_before_filter); + for (size_t row = 0; row < rows_before_filter; ++row) { + keep_filter[row] &= !delete_filter[row]; + } + file_block->erase(result_column_id); + } + RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, keep_filter)); + } + rows_after_delete_filter = + file_block->columns() == 0 ? rows_before_filter : file_block->rows(); + if (profile != nullptr) { + update_counter(profile->rows_filtered_by_delete_conjunct, + rows_before_filter - rows_after_delete_filter); + } + } + + size_t rows_after_filter = rows_after_delete_filter; + if (request != nullptr && rows_after_delete_filter > 0 && !request->conjuncts.empty()) { + { + SCOPED_TIMER(profile == nullptr ? nullptr : profile->conjunct_filter_time); + RETURN_IF_ERROR(VExprContext::filter_block(request->conjuncts, file_block, + file_block->columns())); + } + rows_after_filter = + file_block->columns() == 0 ? rows_after_delete_filter : file_block->rows(); + const auto rows_filtered_by_conjunct = rows_after_delete_filter - rows_after_filter; + if (profile != nullptr) { + update_counter(profile->rows_filtered_by_conjunct, rows_filtered_by_conjunct); + } + if (io_ctx != nullptr) { + io_ctx->predicate_filtered_rows += rows_filtered_by_conjunct; + } + } + *rows = rows_after_filter; + return Status::OK(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/materialized_reader_util.h b/be/src/format_v2/materialized_reader_util.h new file mode 100644 index 00000000000000..2fb1383dfb9569 --- /dev/null +++ b/be/src/format_v2/materialized_reader_util.h @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/column/column.h" +#include "core/data_type/data_type.h" +#include "runtime/runtime_profile.h" + +namespace doris { +class Block; + +namespace io { +struct IOContext; +} // namespace io + +namespace format { +struct FileScanRequest; + +// Shared helpers for FileReader implementations that deserialize or build already materialized +// Doris columns and then hand those columns to TableReader for final mapping. +ColumnPtr make_column_nullable_if_needed(ColumnPtr column, const DataTypePtr& target_type); + +// Optional profile counters for text-like readers. Native/JSON do not expose per-reader filter +// counters today, so they call apply_materialized_reader_filters() without this struct. +struct MaterializedReaderFilterProfile { + RuntimeProfile::Counter* delete_conjunct_filter_time = nullptr; + RuntimeProfile::Counter* conjunct_filter_time = nullptr; + RuntimeProfile::Counter* rows_filtered_by_delete_conjunct = nullptr; + RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; +}; + +// Applies file-local filters in the same order used by FileScannerV2 readers: +// 1. delete_conjuncts remove rows that should not be visible to the scan output; +// 2. conjuncts apply ordinary file-local predicates. +// +// Only ordinary conjunct filtering contributes to IOContext::predicate_filtered_rows. This matches +// the previous JSON/Text/CSV behavior and keeps scanner accounting separate from delete filtering. +// When `profile` is provided, the helper also updates text-reader timer and row counters so CSV +// and Hive text keep their existing observability after sharing this implementation. +Status apply_materialized_reader_filters(const FileScanRequest* request, io::IOContext* io_ctx, + Block* file_block, size_t* rows, + const MaterializedReaderFilterProfile* profile = nullptr); + +} // namespace format +} // namespace doris diff --git a/be/src/format_v2/native/native_reader.cpp b/be/src/format_v2/native/native_reader.cpp new file mode 100644 index 00000000000000..0e348f3de2ccf5 --- /dev/null +++ b/be/src/format_v2/native/native_reader.cpp @@ -0,0 +1,311 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/native/native_reader.h" + +#include +#include + +#include "common/cast_set.h" +#include "core/block/block.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_nullable.h" +#include "format/native/native_format.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "io/file_factory.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/runtime_state.h" +#include "util/slice.h" + +namespace doris::format::native { +namespace { + +Status parse_native_pblock(const std::string& buffer, const std::string& path, PBlock* pblock) { + DORIS_CHECK(pblock != nullptr); + if (!pblock->ParseFromArray(buffer.data(), cast_set(buffer.size()))) { + return Status::InternalError("Failed to parse native PBlock from file {}", path); + } + return Status::OK(); +} + +} // namespace + +NativeReader::NativeReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile) + : FileReader(system_properties, file_description, std::move(io_ctx), profile) {} + +NativeReader::~NativeReader() { + static_cast(close()); +} + +Status NativeReader::init(RuntimeState* state) { + _runtime_state = state; + if (_file_description == nullptr) { + return Status::InvalidArgument("Native v2 reader requires file description"); + } + RETURN_IF_ERROR(FileReader::init(state)); + RETURN_IF_ERROR(_validate_and_consume_header()); + return Status::OK(); +} + +Status NativeReader::get_schema(std::vector* file_schema) const { + if (file_schema == nullptr) { + return Status::InvalidArgument("Native v2 file_schema is null"); + } + RETURN_IF_ERROR(_ensure_schema_loaded()); + *file_schema = _file_schema; + return Status::OK(); +} + +std::unique_ptr NativeReader::create_column_mapper( + TableColumnMapperOptions options) const { + return std::make_unique(std::move(options)); +} + +Status NativeReader::open(std::shared_ptr request) { + RETURN_IF_ERROR(FileReader::open(std::move(request))); + DORIS_CHECK(_request != nullptr); + _first_block_consumed = false; + _reader_eof = false; + _eof = false; + return Status::OK(); +} + +Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + if (_request == nullptr) { + return Status::InternalError("Native v2 reader is not open"); + } + + *rows = 0; + *eof = false; + if (_reader_eof) { + *eof = true; + _eof = true; + return Status::OK(); + } + + std::string buffer; + bool local_eof = false; + if (_first_block_loaded && !_first_block_consumed) { + buffer = _first_block_buffer; + } else { + RETURN_IF_ERROR(_read_next_pblock(&buffer, &local_eof)); + } + + if (local_eof && buffer.empty()) { + _reader_eof = true; + *eof = true; + _eof = true; + return Status::OK(); + } + if (buffer.empty()) { + return Status::InternalError("read empty native block from file {}", + _file_description->path); + } + + PBlock pblock; + RETURN_IF_ERROR(parse_native_pblock(buffer, _file_description->path, &pblock)); + if (!_schema_inited) { + RETURN_IF_ERROR(_init_schema_from_pblock(pblock)); + } + + Block source_block; + size_t uncompressed_bytes = 0; + int64_t decompress_time = 0; + RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); + RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + *rows = file_block->rows(); + _record_scan_rows(cast_set(*rows)); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + + if (_first_block_loaded && !_first_block_consumed) { + _first_block_consumed = true; + } + if (_current_offset >= _file_size) { + _reader_eof = true; + } + *eof = _reader_eof && *rows == 0; + _eof = *eof; + return Status::OK(); +} + +Status NativeReader::close() { + _file_reader.reset(); + _tracing_file_reader.reset(); + _request.reset(); + _reader_eof = true; + _eof = true; + return Status::OK(); +} + +Status NativeReader::_validate_and_consume_header() { + DORIS_CHECK(_tracing_file_reader != nullptr); + _file_size = _tracing_file_reader->size(); + _current_offset = 0; + _reader_eof = (_file_size == 0); + + static constexpr size_t HEADER_SIZE = sizeof(DORIS_NATIVE_MAGIC) + sizeof(uint32_t); + if (_reader_eof || _file_size < cast_set(HEADER_SIZE)) { + return Status::InternalError( + "invalid Doris Native file {}, file size {} is smaller than header size {}", + _file_description->path, _file_size, HEADER_SIZE); + } + + char header[HEADER_SIZE]; + Slice header_slice(header, sizeof(header)); + size_t bytes_read = 0; + RETURN_IF_ERROR(_tracing_file_reader->read_at(0, header_slice, &bytes_read, _io_ctx.get())); + if (bytes_read != sizeof(header)) { + return Status::InternalError( + "failed to read Doris Native header from file {}, expect {} bytes, got {} bytes", + _file_description->path, sizeof(header), bytes_read); + } + if (std::memcmp(header, DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)) != 0) { + return Status::InternalError("invalid Doris Native magic header in file {}", + _file_description->path); + } + + uint32_t version = 0; + std::memcpy(&version, header + sizeof(DORIS_NATIVE_MAGIC), sizeof(uint32_t)); + if (version != DORIS_NATIVE_FORMAT_VERSION) { + return Status::InternalError( + "unsupported Doris Native format version {} in file {}, expect {}", version, + _file_description->path, DORIS_NATIVE_FORMAT_VERSION); + } + + _current_offset = sizeof(header); + _reader_eof = (_file_size == _current_offset); + return Status::OK(); +} + +Status NativeReader::_ensure_schema_loaded() const { + if (_schema_inited) { + return Status::OK(); + } + if (!_first_block_loaded) { + bool local_eof = false; + RETURN_IF_ERROR(_read_next_pblock(&_first_block_buffer, &local_eof)); + if (local_eof && _first_block_buffer.empty()) { + return Status::EndOfFile("empty native file {}", _file_description->path); + } + if (_first_block_buffer.empty()) { + return Status::InternalError("first native block is empty {}", _file_description->path); + } + _first_block_loaded = true; + } + + PBlock pblock; + RETURN_IF_ERROR(parse_native_pblock(_first_block_buffer, _file_description->path, &pblock)); + RETURN_IF_ERROR(_init_schema_from_pblock(pblock)); + return Status::OK(); +} + +Status NativeReader::_read_next_pblock(std::string* buffer, bool* eof) const { + DORIS_CHECK(buffer != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(_tracing_file_reader != nullptr); + buffer->clear(); + *eof = false; + + if (_current_offset >= _file_size) { + *eof = true; + return Status::OK(); + } + + uint64_t block_len = 0; + Slice len_slice(reinterpret_cast(&block_len), sizeof(block_len)); + size_t bytes_read = 0; + RETURN_IF_ERROR( + _tracing_file_reader->read_at(_current_offset, len_slice, &bytes_read, _io_ctx.get())); + if (bytes_read == 0) { + *eof = true; + return Status::OK(); + } + if (bytes_read != sizeof(block_len)) { + return Status::InternalError( + "Failed to read native block length from file {}, expect {}, actual {}", + _file_description->path, sizeof(block_len), bytes_read); + } + _current_offset += sizeof(block_len); + if (block_len == 0) { + *eof = (_current_offset >= _file_size); + return Status::OK(); + } + + buffer->assign(block_len, '\0'); + Slice data_slice(buffer->data(), block_len); + bytes_read = 0; + RETURN_IF_ERROR( + _tracing_file_reader->read_at(_current_offset, data_slice, &bytes_read, _io_ctx.get())); + if (bytes_read != block_len) { + return Status::InternalError( + "Failed to read native block body from file {}, expect {}, actual {}", + _file_description->path, block_len, bytes_read); + } + _current_offset += block_len; + *eof = (_current_offset >= _file_size); + return Status::OK(); +} + +Status NativeReader::_init_schema_from_pblock(const PBlock& pblock) const { + _file_schema.clear(); + _file_schema.reserve(pblock.column_metas_size()); + for (int idx = 0; idx < pblock.column_metas_size(); ++idx) { + const auto& meta = pblock.column_metas(idx); + ColumnDefinition field; + field.identifier = Field::create_field(meta.name()); + field.local_id = idx; + field.name = meta.name(); + field.type = make_nullable(DataTypeFactory::instance().create_data_type(meta)); + _file_schema.push_back(std::move(field)); + } + _schema_inited = true; + return Status::OK(); +} + +Status NativeReader::_materialize_requested_columns(const Block& source_block, + Block* file_block) const { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(_request != nullptr); + for (const auto& [file_column_id, block_position] : _request->local_positions) { + const auto source_idx = file_column_id.value(); + if (source_idx < 0 || cast_set(source_idx) >= source_block.columns()) { + return Status::InternalError("native file {} does not contain local column id {}", + _file_description->path, source_idx); + } + if (block_position.value() >= file_block->columns()) { + return Status::InternalError("native v2 request has invalid block position {}", + block_position.value()); + } + const auto& target = file_block->get_by_position(block_position.value()); + auto column = source_block.get_by_position(source_idx).column; + column = make_column_nullable_if_needed(std::move(column), target.type); + file_block->replace_by_position(block_position.value(), IColumn::mutate(std::move(column))); + } + return Status::OK(); +} + +Status NativeReader::_apply_filters(Block* file_block, size_t* rows) const { + return apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows); +} + +} // namespace doris::format::native diff --git a/be/src/format_v2/native/native_reader.h b/be/src/format_v2/native/native_reader.h new file mode 100644 index 00000000000000..3719a6afd6c4f5 --- /dev/null +++ b/be/src/format_v2/native/native_reader.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "format_v2/file_reader.h" + +namespace doris::format::native { + +// FileScannerV2 reader for Doris Native files. +// +// Native files are self-describing only through the first serialized PBlock. TableReader asks for +// schema before open(), so this reader may read and cache that first PBlock during get_schema() and +// then replay it as the first data batch after open(). +class NativeReader final : public FileReader { +public: + NativeReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile); + ~NativeReader() override; + + Status init(RuntimeState* state) override; + Status get_schema(std::vector* file_schema) const override; + std::unique_ptr create_column_mapper( + TableColumnMapperOptions options) const override; + Status open(std::shared_ptr request) override; + Status get_block(Block* file_block, size_t* rows, bool* eof) override; + Status close() override; + +private: + Status _validate_and_consume_header(); + Status _ensure_schema_loaded() const; + Status _read_next_pblock(std::string* buffer, bool* eof) const; + Status _init_schema_from_pblock(const PBlock& pblock) const; + Status _materialize_requested_columns(const Block& source_block, Block* file_block) const; + Status _apply_filters(Block* file_block, size_t* rows) const; + + RuntimeState* _runtime_state = nullptr; + mutable int64_t _current_offset = 0; + mutable int64_t _file_size = 0; + mutable bool _reader_eof = true; + mutable bool _schema_inited = false; + mutable std::vector _file_schema; + mutable std::string _first_block_buffer; + mutable bool _first_block_loaded = false; + mutable bool _first_block_consumed = false; +}; + +} // namespace doris::format::native diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp new file mode 100644 index 00000000000000..a5144d7b2017b0 --- /dev/null +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -0,0 +1,377 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/orc/orc_file_input_stream.h" + +#include +#include +#include + +#include "common/status.h" +#include "core/custom_allocator.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/tracing_file_reader.h" +#include "io/io_common.h" +#include "orc/Exceptions.hh" +#include "runtime/runtime_profile.h" +#include "util/slice.h" + +namespace doris::format::orc { +namespace { + +struct OrcMergedRangeStatistics { + int64_t copy_time = 0; + int64_t read_time = 0; + int64_t request_io = 0; + int64_t merged_io = 0; + int64_t request_bytes = 0; + int64_t merged_bytes = 0; + int64_t apply_bytes = 0; + int64_t cluster_num = 1; +}; + +class OrcMergedRangeFileReader final : public io::FileReader { +public: + OrcMergedRangeFileReader(RuntimeProfile* profile, io::FileReaderSPtr file_reader, + io::PrefetchRange range) + : _profile(profile), + _file_reader(std::move(file_reader)), + _range(range), + _size(_file_reader->size()) { + _statistics.apply_bytes += _range.end_offset - _range.start_offset; + if (_profile != nullptr) { + const char* profile_name = "MergedSmallIO"; + ADD_TIMER_WITH_LEVEL(_profile, profile_name, 1); + _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", profile_name, 1); + _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", profile_name, 1); + _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, + profile_name, 1); + _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedIO", TUnit::UNIT, + profile_name, 1); + _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, + profile_name, 1); + _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES, + profile_name, 1); + _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ApplyBytes", TUnit::BYTES, + profile_name, 1); + _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OverReadBytes", TUnit::BYTES, + profile_name, 1); + _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ClusterNum", TUnit::UNIT, + profile_name, 1); + } + } + + Status close() override { + _closed = true; + return Status::OK(); + } + + const io::Path& path() const override { return _file_reader->path(); } + size_t size() const override { return _size; } + bool closed() const override { return _closed; } + int64_t mtime() const override { return _file_reader->mtime(); } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext* io_ctx) override { + ++_statistics.request_io; + _statistics.request_bytes += static_cast(result.size); + *bytes_read = 0; + if (result.size == 0) { + return Status::OK(); + } + if (offset < _range.start_offset || offset + result.size > _range.end_offset) { + return Status::IOError("ORC stripe read [{}, {}) is outside merged range [{}, {})", + offset, offset + result.size, _range.start_offset, + _range.end_offset); + } + + RETURN_IF_ERROR(_load(io_ctx)); + { + SCOPED_RAW_TIMER(&_statistics.copy_time); + std::memcpy(result.data, _cache.get() + offset - _range.start_offset, result.size); + } + *bytes_read = result.size; + return Status::OK(); + } + + void _collect_profile_before_close() override { + if (_profile == nullptr) { + return; + } + COUNTER_UPDATE(_copy_time, _statistics.copy_time); + COUNTER_UPDATE(_read_time, _statistics.read_time); + COUNTER_UPDATE(_request_io, _statistics.request_io); + COUNTER_UPDATE(_merged_io, _statistics.merged_io); + COUNTER_UPDATE(_request_bytes, _statistics.request_bytes); + COUNTER_UPDATE(_merged_bytes, _statistics.merged_bytes); + COUNTER_UPDATE(_apply_bytes, _statistics.apply_bytes); + COUNTER_UPDATE(_over_read_bytes, + std::max(_statistics.merged_bytes - _statistics.request_bytes, 0)); + COUNTER_UPDATE(_cluster_num, _statistics.cluster_num); + } + +private: + Status _load(const io::IOContext* io_ctx) { + if (_loaded) { + return Status::OK(); + } + + const size_t range_size = _range.end_offset - _range.start_offset; + _cache = make_unique_buffer(range_size); + size_t total_read = 0; + { + SCOPED_RAW_TIMER(&_statistics.read_time); + while (total_read < range_size) { + size_t loop_read = 0; + RETURN_IF_ERROR(_file_reader->read_at( + _range.start_offset + total_read, + Slice(_cache.get() + total_read, range_size - total_read), &loop_read, + io_ctx)); + ++_statistics.merged_io; + _statistics.merged_bytes += static_cast(loop_read); + if (loop_read == 0) { + return Status::IOError("Short read for ORC merged range [{}, {})", + _range.start_offset, _range.end_offset); + } + total_read += loop_read; + } + } + _loaded = true; + return Status::OK(); + } + + RuntimeProfile* _profile = nullptr; + io::FileReaderSPtr _file_reader; + io::PrefetchRange _range; + size_t _size = 0; + bool _closed = false; + bool _loaded = false; + DorisUniqueBufferPtr _cache; + OrcMergedRangeStatistics _statistics; + + RuntimeProfile::Counter* _copy_time = nullptr; + RuntimeProfile::Counter* _read_time = nullptr; + RuntimeProfile::Counter* _request_io = nullptr; + RuntimeProfile::Counter* _merged_io = nullptr; + RuntimeProfile::Counter* _request_bytes = nullptr; + RuntimeProfile::Counter* _merged_bytes = nullptr; + RuntimeProfile::Counter* _apply_bytes = nullptr; + RuntimeProfile::Counter* _over_read_bytes = nullptr; + RuntimeProfile::Counter* _cluster_num = nullptr; +}; + +class OrcStripeInputStream final : public ::orc::InputStream { +public: + OrcStripeInputStream(std::string file_name, io::FileReaderSPtr file_reader, + const io::IOContext* io_ctx, uint64_t natural_read_size) + : _file_name(std::move(file_name)), + _file_reader(std::move(file_reader)), + _io_ctx(io_ctx), + _natural_read_size(natural_read_size) {} + + uint64_t getLength() const override { return _file_reader->size(); } + uint64_t getNaturalReadSize() const override { return _natural_read_size; } + + void read(void* buf, uint64_t length, uint64_t offset) override { + uint64_t bytes_read = 0; + auto* out = static_cast(buf); + while (bytes_read < length) { + if (_io_ctx != nullptr && _io_ctx->should_stop) { + throw ::orc::ParseError("stop"); + } + size_t loop_read = 0; + Status st = _file_reader->read_at( + static_cast(offset + bytes_read), + Slice(out + bytes_read, static_cast(length - bytes_read)), &loop_read, + _io_ctx); + if (!st.ok()) { + throw ::orc::ParseError("Failed to read " + _file_name + ": " + + st.to_string_no_stack()); + } + if (loop_read == 0) { + break; + } + bytes_read += loop_read; + } + if (bytes_read != length) { + throw ::orc::ParseError("Short read from " + _file_name); + } + } + + const std::string& getName() const override { return _file_name; } + +private: + std::string _file_name; + io::FileReaderSPtr _file_reader; + const io::IOContext* _io_ctx = nullptr; + uint64_t _natural_read_size = 0; +}; + +struct StripeStreamRange { + ::orc::StreamId stream_id; + io::PrefetchRange range; +}; + +} // namespace + +OrcFileInputStream::OrcFileInputStream(std::string file_name, io::FileReaderSPtr file_reader, + const io::IOContext* io_ctx, RuntimeProfile* profile, + OrcFileInputStreamOptions options) + : _file_name(std::move(file_name)), + _file_reader(std::move(file_reader)), + _default_reader(io_ctx != nullptr && io_ctx->file_reader_stats != nullptr + ? std::make_shared( + _file_reader, io_ctx->file_reader_stats) + : _file_reader), + _io_ctx(io_ctx), + _profile(profile), + _options(options) { + DORIS_CHECK_GT(_options.natural_read_size, 0); + DORIS_CHECK_GE(_options.once_max_read_bytes, 0); + DORIS_CHECK_GE(_options.max_merge_distance_bytes, 0); +} + +OrcFileInputStream::~OrcFileInputStream() { + _flush_active_clusters(); +} + +uint64_t OrcFileInputStream::getLength() const { + return _default_reader->size(); +} + +uint64_t OrcFileInputStream::getNaturalReadSize() const { + return _options.natural_read_size; +} + +void OrcFileInputStream::read(void* buf, uint64_t length, uint64_t offset) { + OrcStripeInputStream(_file_name, _default_reader, _io_ctx, _options.natural_read_size) + .read(buf, length, offset); +} + +const std::string& OrcFileInputStream::getName() const { + return _file_name; +} + +void OrcFileInputStream::beforeReadStripe( + std::unique_ptr<::orc::StripeInformation> current_stripe_information, + const std::vector& selected_columns, + std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams) { + _flush_active_clusters(); + _active_stripe_streams.clear(); + + std::vector small_streams; + std::vector<::orc::StreamId> direct_streams; + uint64_t offset = current_stripe_information->getOffset(); + for (uint64_t stream_index = 0; stream_index < current_stripe_information->getNumberOfStreams(); + ++stream_index) { + auto stream = current_stripe_information->getStreamInformation(stream_index); + const uint64_t column_id = stream->getColumnId(); + if (column_id >= selected_columns.size()) { + throw ::orc::ParseError( + fmt::format("Invalid ORC stream column id {} in {}, selected column count {}", + column_id, _file_name, selected_columns.size())); + } + const uint64_t length = stream->getLength(); + if (selected_columns[column_id]) { + ::orc::StreamId stream_id(column_id, stream->getKind()); + if (length == 0 || std::cmp_greater(length, _options.once_max_read_bytes)) { + direct_streams.push_back(stream_id); + } else { + small_streams.push_back({stream_id, io::PrefetchRange(offset, offset + length)}); + } + } + offset += length; + } + + for (const auto& stream_id : direct_streams) { + _add_direct_stream(stream_id, streams); + } + if (small_streams.empty()) { + return; + } + + std::sort(small_streams.begin(), small_streams.end(), + [](const StripeStreamRange& left, const StripeStreamRange& right) { + return left.range.start_offset < right.range.start_offset; + }); + std::vector small_ranges; + small_ranges.reserve(small_streams.size()); + for (const auto& stream : small_streams) { + small_ranges.push_back(stream.range); + } + const auto merged_ranges = io::PrefetchRange::merge_adjacent_seq_ranges( + small_ranges, _options.max_merge_distance_bytes, _options.once_max_read_bytes); + + size_t stream_index = 0; + for (const auto& merged_range : merged_ranges) { + std::vector> cluster_streams; + while (stream_index < small_streams.size() && + small_streams[stream_index].range.start_offset < merged_range.end_offset) { + DORIS_CHECK_LE(small_streams[stream_index].range.end_offset, merged_range.end_offset); + cluster_streams.emplace_back(small_streams[stream_index].stream_id, + small_streams[stream_index].range); + ++stream_index; + } + DORIS_CHECK(!cluster_streams.empty()); + if (cluster_streams.size() == 1) { + _add_direct_stream(cluster_streams.front().first, streams); + } else { + _add_clustered_streams(cluster_streams, merged_range, streams); + } + } + DORIS_CHECK_EQ(stream_index, small_streams.size()); +} + +void OrcFileInputStream::_flush_active_clusters() { + for (const auto& reader : _active_cluster_readers) { + reader->collect_profile_before_close(); + } + _active_cluster_readers.clear(); +} + +void OrcFileInputStream::_add_direct_stream( + const ::orc::StreamId& stream_id, + std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams) { + auto stream = std::make_shared(_file_name, _default_reader, _io_ctx, + _options.natural_read_size); + streams.emplace(stream_id, stream); + _active_stripe_streams.push_back(std::move(stream)); +} + +void OrcFileInputStream::_add_clustered_streams( + const std::vector>& cluster_streams, + const io::PrefetchRange& cluster_range, + std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams) { + io::FileReaderSPtr cluster_reader = + std::make_shared(_profile, _file_reader, cluster_range); + if (_io_ctx != nullptr && _io_ctx->file_reader_stats != nullptr) { + cluster_reader = std::make_shared(std::move(cluster_reader), + _io_ctx->file_reader_stats); + } + _active_cluster_readers.push_back(cluster_reader); + for (const auto& [stream_id, range] : cluster_streams) { + DORIS_CHECK_GE(range.start_offset, cluster_range.start_offset); + DORIS_CHECK_LE(range.end_offset, cluster_range.end_offset); + auto stream = std::make_shared(_file_name, cluster_reader, _io_ctx, + _options.natural_read_size); + streams.emplace(stream_id, stream); + _active_stripe_streams.push_back(std::move(stream)); + } +} + +} // namespace doris::format::orc diff --git a/be/src/format_v2/orc/orc_file_input_stream.h b/be/src/format_v2/orc/orc_file_input_stream.h new file mode 100644 index 00000000000000..3ced728389004f --- /dev/null +++ b/be/src/format_v2/orc/orc_file_input_stream.h @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "io/fs/buffered_reader.h" + +namespace doris { + +class RuntimeProfile; + +namespace io { +struct IOContext; +} + +namespace format::orc { + +struct OrcFileInputStreamOptions { + uint64_t natural_read_size = 8L * 1024L * 1024L; + int64_t once_max_read_bytes = 8L * 1024L * 1024L; + int64_t max_merge_distance_bytes = 1L * 1024L * 1024L; +}; + +class OrcFileInputStream final : public ::orc::InputStream { +public: + OrcFileInputStream(std::string file_name, io::FileReaderSPtr file_reader, + const io::IOContext* io_ctx, RuntimeProfile* profile, + OrcFileInputStreamOptions options); + ~OrcFileInputStream() override; + + uint64_t getLength() const override; + uint64_t getNaturalReadSize() const override; + void read(void* buf, uint64_t length, uint64_t offset) override; + const std::string& getName() const override; + + void beforeReadStripe(std::unique_ptr<::orc::StripeInformation> current_stripe_information, + const std::vector& selected_columns, + std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& + streams) override; + +private: + void _flush_active_clusters(); + void _add_direct_stream( + const ::orc::StreamId& stream_id, + std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams); + void _add_clustered_streams( + const std::vector>& cluster_streams, + const io::PrefetchRange& cluster_range, + std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams); + + std::string _file_name; + io::FileReaderSPtr _file_reader; + io::FileReaderSPtr _default_reader; + const io::IOContext* _io_ctx = nullptr; + RuntimeProfile* _profile = nullptr; + OrcFileInputStreamOptions _options; + + std::vector> _active_stripe_streams; + std::vector _active_cluster_readers; +}; + +} // namespace format::orc +} // namespace doris diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp new file mode 100644 index 00000000000000..b728c6fbedb700 --- /dev/null +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -0,0 +1,2403 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/orc/orc_reader.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/consts.h" +#include "common/exception.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_date_time.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/types.h" +#include "core/value/timestamptz_value.h" +#include "core/value/vdatetime_value.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/column_mapper.h" +#include "format_v2/orc/orc_file_input_stream.h" +#include "format_v2/orc/orc_search_argument.h" +#include "format_v2/timestamp_statistics.h" +#include "io/fs/file_reader.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_profile.h" +#include "storage/index/zone_map/zone_map_index.h" +#include "storage/segment/condition_cache.h" +#include "storage/utils.h" +#include "util/slice.h" +#include "util/timezone_utils.h" + +namespace doris::format::orc { + +bool detail::valid_statistics_bounds(const Field& min_value, const Field& max_value) { + DORIS_CHECK(min_value.get_type() == max_value.get_type()); + if (min_value.get_type() == TYPE_FLOAT && + (std::isnan(min_value.get()) || std::isnan(max_value.get()))) { + return false; + } + if (min_value.get_type() == TYPE_DOUBLE && + (std::isnan(min_value.get()) || std::isnan(max_value.get()))) { + return false; + } + return !(max_value < min_value); +} + +namespace { + +constexpr uint64_t DEFAULT_ORC_READ_BATCH_SIZE = 4096; +constexpr int DECIMAL_PRECISION_FOR_HIVE11 = BeConsts::MAX_DECIMAL128_PRECISION; +constexpr int DECIMAL_SCALE_FOR_HIVE11 = 10; +constexpr const char* ORC_LIST_ELEMENT_NAME = "element"; +constexpr const char* ORC_MAP_KEY_NAME = "key"; +constexpr const char* ORC_MAP_VALUE_NAME = "value"; +constexpr const char* ORC_ICEBERG_ID_ATTRIBUTE = "iceberg.id"; + +bool set_validated_zone_map(Field min_value, Field max_value, segment_v2::ZoneMap* zone_map) { + DORIS_CHECK(zone_map != nullptr); + if (!detail::valid_statistics_bounds(min_value, max_value)) { + return false; + } + zone_map->min_value = std::move(min_value); + zone_map->max_value = std::move(max_value); + return true; +} + +uint64_t orc_metric_value(const std::atomic& metric) { + return metric.load(std::memory_order_relaxed); +} + +template +struct OrcReadRowCountMetric { + static uint64_t value(const Metrics&) { return 0; } +}; + +template +struct OrcReadRowCountMetric().ReadRowCount)>> { + static uint64_t value(const Metrics& metrics) { return orc_metric_value(metrics.ReadRowCount); } +}; + +uint64_t orc_read_row_count(const ::orc::ReaderMetrics& metrics) { + return OrcReadRowCountMetric<::orc::ReaderMetrics>::value(metrics); +} + +bool is_orc_stop(const io::IOContext* io_ctx, const std::exception& e) { + return io_ctx != nullptr && io_ctx->should_stop && std::string_view(e.what()) == "stop"; +} + +bool is_hour_offset_timezone(std::string_view timezone) { + return timezone.size() == 6 && (timezone[0] == '+' || timezone[0] == '-') && + std::isdigit(static_cast(timezone[1])) && + std::isdigit(static_cast(timezone[2])) && timezone[3] == ':' && + timezone[4] == '0' && timezone[5] == '0'; +} + +Status set_orc_reader_timezone(const std::string& timezone, + ::orc::RowReaderOptions* row_reader_options) { + if (timezone == "CST") { + row_reader_options->setTimezoneName("Asia/Shanghai"); + return Status::OK(); + } + + if (!timezone.empty() && (timezone[0] == '+' || timezone[0] == '-') && + is_hour_offset_timezone(timezone)) { + const int hour = (timezone[1] - '0') * 10 + timezone[2] - '0'; + row_reader_options->setTimezoneName( + hour == 0 ? "Etc/GMT" + : fmt::format("Etc/GMT{}{}", timezone[0] == '+' ? '-' : '+', hour)); + return Status::OK(); + } + + row_reader_options->setTimezoneName(timezone.empty() ? "UTC" : timezone); + return Status::OK(); +} + +// selected_rows is a source-row remap produced by ORC lazy callback: +// predicate columns are decoded first, then surviving row ids drive follower decodes. +size_t decode_row_count(size_t rows, const std::vector* selected_rows) { + if (selected_rows == nullptr) { + return rows; + } + return selected_rows->size(); +} + +size_t source_row_at(size_t row, const std::vector* selected_rows) { + if (selected_rows == nullptr) { + return row; + } + return (*selected_rows)[row]; +} + +size_t trim_right_spaces(const char* value, size_t length) { + while (length > 0 && value[length - 1] == ' ') { + --length; + } + return length; +} + +Int128 to_int128(::orc::Int128 value) { + const auto high_bits = static_cast<__uint128_t>(static_cast(value.getHighBits())); + const auto low_bits = static_cast<__uint128_t>(value.getLowBits()); + return static_cast((high_bits << 64) | low_bits); +} + +// ORC nested projection is type-id based. These helpers translate Doris' +// LocalColumnIndex tree into the ORC type ids expected by includeTypes(). +Status get_projection_child_index(const format::LocalColumnIndex& child, int32_t child_count, + const std::string& column_name, int32_t* child_idx) { + DORIS_CHECK(child_idx != nullptr); + *child_idx = child.local_id(); + if (*child_idx < 0 || *child_idx >= child_count) { + return Status::InvalidArgument("Invalid ORC projection child index {} for column {}", + *child_idx, column_name); + } + return Status::OK(); +} + +void collect_type_and_descendant_ids(const ::orc::Type& type, std::set* const type_ids) { + DORIS_CHECK(type_ids != nullptr); + type_ids->insert(type.getColumnId()); + for (uint64_t child_idx = 0; child_idx < type.getSubtypeCount(); ++child_idx) { + const auto* child_type = type.getSubtype(child_idx); + DORIS_CHECK(child_type != nullptr); + collect_type_and_descendant_ids(*child_type, type_ids); + } +} + +Status collect_projected_type_ids(const ::orc::Type& type, + const format::LocalColumnIndex& projection, + std::set* const type_ids); + +Status collect_projected_map_type_ids(const ::orc::Type& type, + const format::LocalColumnIndex& projection, + std::set* const type_ids); + +Status collect_projected_map_type_ids(const ::orc::Type& type, + const format::LocalColumnIndex& projection, + std::set* const type_ids) { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::MAP); + DORIS_CHECK(type.getSubtypeCount() == 2); + type_ids->insert(type.getColumnId()); + if (projection.project_all_children) { + collect_type_and_descendant_ids(type, type_ids); + return Status::OK(); + } + if (projection.children.empty()) { + return Status::NotSupported("ORC MAP projection for column {} contains no children", + projection.local_id()); + } + + bool selected_key = false; + bool selected_value = false; + for (const auto& key_value_projection : projection.children) { + int32_t key_value_idx = 0; + RETURN_IF_ERROR( + get_projection_child_index(key_value_projection, 2, "orc_map", &key_value_idx)); + const auto* child_type = type.getSubtype(static_cast(key_value_idx)); + DORIS_CHECK(child_type != nullptr); + RETURN_IF_ERROR(collect_projected_type_ids(*child_type, key_value_projection, type_ids)); + selected_key = selected_key || key_value_idx == 0; + selected_value = selected_value || key_value_idx == 1; + } + if (!selected_key || !selected_value) { + return Status::NotSupported("ORC MAP projection must include both key and value"); + } + return Status::OK(); +} + +Status collect_projected_type_ids(const ::orc::Type& type, + const format::LocalColumnIndex& projection, + std::set* const type_ids) { + DORIS_CHECK(type_ids != nullptr); + type_ids->insert(type.getColumnId()); + if (projection.project_all_children) { + collect_type_and_descendant_ids(type, type_ids); + return Status::OK(); + } + if (projection.children.empty()) { + return Status::NotSupported("ORC projection contains no children"); + } + if (type.getKind() == ::orc::TypeKind::MAP) { + return collect_projected_map_type_ids(type, projection, type_ids); + } + if (type.getKind() != ::orc::TypeKind::STRUCT && type.getKind() != ::orc::TypeKind::LIST) { + return Status::InvalidArgument("Cannot project children from non-complex ORC type {}", + static_cast(type.getKind())); + } + + const auto child_count = static_cast(type.getSubtypeCount()); + for (const auto& child_projection : projection.children) { + int32_t child_idx = 0; + RETURN_IF_ERROR(get_projection_child_index(child_projection, child_count, "orc_complex", + &child_idx)); + const auto* child_type = type.getSubtype(static_cast(child_idx)); + DORIS_CHECK(child_type != nullptr); + RETURN_IF_ERROR(collect_projected_type_ids(*child_type, child_projection, type_ids)); + } + return Status::OK(); +} + +bool is_row_position_column(format::LocalColumnId file_column_id) { + return file_column_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID); +} + +bool is_global_rowid_column(format::LocalColumnId file_column_id) { + return file_column_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID); +} + +bool is_virtual_column(format::LocalColumnId file_column_id) { + return is_row_position_column(file_column_id) || is_global_rowid_column(file_column_id); +} + +format::ColumnDefinition nullable_global_rowid_column_definition() { + auto field = format::global_rowid_column_definition(); + field.type = make_nullable(field.type); + return field; +} + +const format::LocalColumnIndex* find_projection( + const std::vector& projections, + format::LocalColumnId file_column_id) { + const auto it = std::find_if(projections.begin(), projections.end(), + [&](const format::LocalColumnIndex& projection) { + return projection.column_id() == file_column_id; + }); + return it == projections.end() ? nullptr : &*it; +} + +bool local_column_ids_are_unique(const std::vector& projections) { + std::set column_ids; + for (const auto& projection : projections) { + if (!column_ids.insert(projection.column_id()).second) { + return false; + } + } + return true; +} + +const format::LocalColumnIndex* find_request_projection(const format::FileScanRequest& request, + format::LocalColumnId file_column_id) { + if (const auto* projection = find_projection(request.predicate_columns, file_column_id); + projection != nullptr) { + return projection; + } + return find_projection(request.non_predicate_columns, file_column_id); +} + +bool has_pruned_projection(const format::LocalColumnIndex& projection) { + return !projection.project_all_children; +} + +Status collect_lazy_filter_type_ids(const ::orc::Type& root_type, + const std::vector& projections, + std::set* const type_ids) { + DORIS_CHECK(type_ids != nullptr); + for (const auto& projection : projections) { + const auto file_column_id = projection.column_id(); + if (is_virtual_column(file_column_id)) { + continue; + } + const auto* type = root_type.getSubtype(static_cast(file_column_id.value())); + DORIS_CHECK(type != nullptr); + if (!has_pruned_projection(projection)) { + collect_type_and_descendant_ids(*type, type_ids); + continue; + } + RETURN_IF_ERROR(collect_projected_type_ids(*type, projection, type_ids)); + } + return Status::OK(); +} + +// Stripe pruning maps ORC stripe statistics into Doris ZoneMap semantics. Missing +// or unsupported statistics are treated conservatively and never prune. +bool set_integer_zone_map(const ::orc::Type& type, const ::orc::ColumnStatistics& statistics, + segment_v2::ZoneMap* zone_map) { + const auto* integer_statistics = + dynamic_cast(&statistics); + if (integer_statistics == nullptr || !integer_statistics->hasMinimum() || + !integer_statistics->hasMaximum()) { + return false; + } + switch (type.getKind()) { + case ::orc::TypeKind::BYTE: + return set_validated_zone_map( + Field::create_field(cast_set(integer_statistics->getMinimum())), + Field::create_field(cast_set(integer_statistics->getMaximum())), + zone_map); + case ::orc::TypeKind::SHORT: + return set_validated_zone_map(Field::create_field( + cast_set(integer_statistics->getMinimum())), + Field::create_field( + cast_set(integer_statistics->getMaximum())), + zone_map); + case ::orc::TypeKind::INT: + return set_validated_zone_map( + Field::create_field(cast_set(integer_statistics->getMinimum())), + Field::create_field(cast_set(integer_statistics->getMaximum())), + zone_map); + case ::orc::TypeKind::LONG: + return set_validated_zone_map( + Field::create_field(integer_statistics->getMinimum()), + Field::create_field(integer_statistics->getMaximum()), zone_map); + default: + return false; + } +} + +bool set_boolean_zone_map(const ::orc::ColumnStatistics& statistics, + segment_v2::ZoneMap* zone_map) { + const auto* boolean_statistics = + dynamic_cast(&statistics); + if (boolean_statistics == nullptr || !boolean_statistics->hasCount()) { + return false; + } + const bool has_false = boolean_statistics->getFalseCount() > 0; + const bool has_true = boolean_statistics->getTrueCount() > 0; + if (!has_false && !has_true) { + return false; + } + return set_validated_zone_map( + Field::create_field(static_cast(has_false ? 0 : 1)), + Field::create_field(static_cast(has_true ? 1 : 0)), zone_map); +} + +bool set_floating_zone_map(const ::orc::Type& type, const ::orc::ColumnStatistics& statistics, + segment_v2::ZoneMap* zone_map) { + const auto* double_statistics = dynamic_cast(&statistics); + if (double_statistics == nullptr || !double_statistics->hasMinimum() || + !double_statistics->hasMaximum()) { + return false; + } + if (type.getKind() == ::orc::TypeKind::FLOAT) { + return set_validated_zone_map(Field::create_field(static_cast( + double_statistics->getMinimum())), + Field::create_field(static_cast( + double_statistics->getMaximum())), + zone_map); + } + if (type.getKind() == ::orc::TypeKind::DOUBLE) { + return set_validated_zone_map( + Field::create_field(double_statistics->getMinimum()), + Field::create_field(double_statistics->getMaximum()), zone_map); + } + return false; +} + +bool set_string_zone_map(const ::orc::Type& type, const ::orc::ColumnStatistics& statistics, + segment_v2::ZoneMap* zone_map) { + const auto* string_statistics = dynamic_cast(&statistics); + if (string_statistics == nullptr || !string_statistics->hasMinimum() || + !string_statistics->hasMaximum()) { + return false; + } + const auto build_field = [&](const std::string& value) { + if (type.getKind() != ::orc::TypeKind::CHAR) { + return Field::create_field(value); + } + return Field::create_field( + std::string(value.data(), trim_right_spaces(value.data(), value.size()))); + }; + return set_validated_zone_map(build_field(string_statistics->getMinimum()), + build_field(string_statistics->getMaximum()), zone_map); +} + +bool set_date_zone_map(const ::orc::ColumnStatistics& statistics, segment_v2::ZoneMap* zone_map) { + const auto* date_statistics = dynamic_cast(&statistics); + if (date_statistics == nullptr || !date_statistics->hasMinimum() || + !date_statistics->hasMaximum()) { + return false; + } + auto& date_dict = date_day_offset_dict::get(); + return set_validated_zone_map( + Field::create_field(date_dict[date_statistics->getMinimum()]), + Field::create_field(date_dict[date_statistics->getMaximum()]), zone_map); +} + +DateV2Value datetime_v2_from_orc_millis(int64_t millis, int32_t nanos_tail, + const cctz::time_zone& timezone) { + int64_t seconds = millis / 1000; + int64_t millis_remainder = millis % 1000; + if (millis_remainder < 0) { + --seconds; + millis_remainder += 1000; + } + const auto extra_nanos = std::max(nanos_tail, 0); + const auto microseconds = cast_set(millis_remainder * 1000 + extra_nanos / 1000); + DateV2Value value; + value.from_unixtime(seconds, timezone); + value.set_microsecond(microseconds); + return value; +} + +TimestampTzValue timestamp_tz_from_orc_millis(int64_t millis, int32_t nanos_tail) { + static const auto utc_time_zone = cctz::utc_time_zone(); + return TimestampTzValue(datetime_v2_from_orc_millis(millis, nanos_tail, utc_time_zone)); +} + +bool set_timestamp_zone_map(const ::orc::ColumnStatistics& statistics, + const cctz::time_zone& timezone, bool use_timestamp_tz, + segment_v2::ZoneMap* zone_map) { + const auto* timestamp_statistics = + dynamic_cast(&statistics); + if (timestamp_statistics == nullptr || !timestamp_statistics->hasMinimum() || + !timestamp_statistics->hasMaximum()) { + return false; + } + const auto min_endpoint = + std::pair(timestamp_statistics->getMinimum(), timestamp_statistics->getMinimumNanos()); + const auto max_endpoint = + std::pair(timestamp_statistics->getMaximum(), timestamp_statistics->getMaximumNanos()); + if (min_endpoint > max_endpoint) { + return false; + } + if (use_timestamp_tz) { + return set_validated_zone_map( + Field::create_field( + timestamp_tz_from_orc_millis(timestamp_statistics->getMinimum(), + timestamp_statistics->getMinimumNanos())), + Field::create_field( + timestamp_tz_from_orc_millis(timestamp_statistics->getMaximum(), + timestamp_statistics->getMaximumNanos())), + zone_map); + } + if (!format::utc_timestamp_range_is_monotonic( + format::floor_epoch_seconds(timestamp_statistics->getMinimum(), 1000), + format::floor_epoch_seconds(timestamp_statistics->getMaximum(), 1000), timezone)) { + return false; + } + return set_validated_zone_map(Field::create_field(datetime_v2_from_orc_millis( + timestamp_statistics->getMinimum(), + timestamp_statistics->getMinimumNanos(), timezone)), + Field::create_field(datetime_v2_from_orc_millis( + timestamp_statistics->getMaximum(), + timestamp_statistics->getMaximumNanos(), timezone)), + zone_map); +} + +int32_t decimal_scale_for_orc_type(const ::orc::Type& type) { + return type.getPrecision() == 0 ? DECIMAL_SCALE_FOR_HIVE11 : cast_set(type.getScale()); +} + +std::optional decimal_value_at_scale(const ::orc::Decimal& decimal, + int32_t target_scale) { + if (decimal.scale == target_scale) { + return Decimal128V3(to_int128(decimal.value)); + } + if (decimal.scale < target_scale) { + bool overflow = false; + const auto scaled = ::orc::scaleUpInt128ByPowerOfTen( + decimal.value, target_scale - decimal.scale, overflow); + if (overflow) { + return std::nullopt; + } + return Decimal128V3(to_int128(scaled)); + } + + const auto scale_diff = decimal.scale - target_scale; + const auto scaled = ::orc::scaleDownInt128ByPowerOfTen(decimal.value, scale_diff); + bool overflow = false; + const auto restored = ::orc::scaleUpInt128ByPowerOfTen(scaled, scale_diff, overflow); + if (overflow || restored != decimal.value) { + return std::nullopt; + } + return Decimal128V3(to_int128(scaled)); +} + +bool set_decimal_zone_map(const ::orc::Type& type, const ::orc::ColumnStatistics& statistics, + segment_v2::ZoneMap* zone_map) { + const auto* decimal_statistics = + dynamic_cast(&statistics); + if (decimal_statistics == nullptr || !decimal_statistics->hasMinimum() || + !decimal_statistics->hasMaximum()) { + return false; + } + const auto min = decimal_statistics->getMinimum(); + const auto max = decimal_statistics->getMaximum(); + const auto expected_scale = decimal_scale_for_orc_type(type); + const auto min_value = decimal_value_at_scale(min, expected_scale); + const auto max_value = decimal_value_at_scale(max, expected_scale); + if (!min_value.has_value() || !max_value.has_value()) { + return false; + } + return set_validated_zone_map(Field::create_field(*min_value), + Field::create_field(*max_value), zone_map); +} + +bool build_zone_map_from_orc_statistics(const ::orc::Type& type, + const ::orc::ColumnStatistics& statistics, + const cctz::time_zone& timezone, + bool enable_mapping_timestamp_tz, + segment_v2::ZoneMap* zone_map) { + DORIS_CHECK(zone_map != nullptr); + zone_map->has_null = statistics.hasNull(); + zone_map->has_not_null = statistics.getNumberOfValues() > 0; + if (!zone_map->has_not_null) { + return true; + } + switch (type.getKind()) { + case ::orc::TypeKind::BOOLEAN: + return set_boolean_zone_map(statistics, zone_map); + case ::orc::TypeKind::BYTE: + case ::orc::TypeKind::SHORT: + case ::orc::TypeKind::INT: + case ::orc::TypeKind::LONG: + return set_integer_zone_map(type, statistics, zone_map); + case ::orc::TypeKind::FLOAT: + case ::orc::TypeKind::DOUBLE: + return set_floating_zone_map(type, statistics, zone_map); + case ::orc::TypeKind::STRING: + case ::orc::TypeKind::VARCHAR: + case ::orc::TypeKind::CHAR: + return set_string_zone_map(type, statistics, zone_map); + case ::orc::TypeKind::DATE: + return set_date_zone_map(statistics, zone_map); + case ::orc::TypeKind::TIMESTAMP: + return set_timestamp_zone_map(statistics, timezone, false, zone_map); + case ::orc::TypeKind::TIMESTAMP_INSTANT: + return set_timestamp_zone_map(statistics, timezone, enable_mapping_timestamp_tz, zone_map); + case ::orc::TypeKind::DECIMAL: + return set_decimal_zone_map(type, statistics, zone_map); + default: + return false; + } +} + +Status find_projected_minmax_leaf_in_type(const ::orc::Type& type, + const format::LocalColumnIndex& projection, + const ::orc::Type** leaf_type) { + DORIS_CHECK(leaf_type != nullptr); + if (projection.project_all_children || projection.children.empty()) { + if (type.getSubtypeCount() > 0) { + return Status::NotSupported( + "ORC aggregate pushdown only supports primitive column kind {}", + static_cast(type.getKind())); + } + *leaf_type = &type; + return Status::OK(); + } + if (projection.children.size() != 1) { + return Status::NotSupported( + "ORC aggregate pushdown only supports a single nested leaf under column kind {}", + static_cast(type.getKind())); + } + if (type.getKind() != ::orc::TypeKind::STRUCT) { + return Status::NotSupported( + "ORC aggregate pushdown only supports struct nested leaf projection, got kind {}", + static_cast(type.getKind())); + } + const auto& child_projection = projection.children[0]; + if (child_projection.local_id() < 0 || + child_projection.local_id() >= static_cast(type.getSubtypeCount())) { + return Status::InvalidArgument("Invalid ORC aggregate child local id {} for kind {}", + child_projection.local_id(), + static_cast(type.getKind())); + } + const auto* child_type = type.getSubtype(static_cast(child_projection.local_id())); + DORIS_CHECK(child_type != nullptr); + return find_projected_minmax_leaf_in_type(*child_type, child_projection, leaf_type); +} + +Status find_projected_minmax_leaf(const ::orc::Type& root_type, + const format::LocalColumnIndex& projection, + const ::orc::Type** leaf_type) { + DORIS_CHECK(leaf_type != nullptr); + if (root_type.getKind() != ::orc::TypeKind::STRUCT) { + return Status::NotSupported("ORC aggregate pushdown requires top-level struct schema"); + } + const auto file_column_id = projection.column_id(); + if (!file_column_id.is_valid() || + file_column_id.value() >= static_cast(root_type.getSubtypeCount())) { + return Status::InvalidArgument("Invalid ORC aggregate column id {}", + file_column_id.value()); + } + const auto* column_type = root_type.getSubtype(static_cast(file_column_id.value())); + DORIS_CHECK(column_type != nullptr); + return find_projected_minmax_leaf_in_type(*column_type, projection, leaf_type); +} + +} // namespace + +class OrcReader::OrcFilterImpl final : public ::orc::ORCFilter { +public: + explicit OrcFilterImpl(OrcReader* reader) : _reader(reader) {} + + void filter(::orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t size, + void* arg) const override { + THROW_IF_ERROR(_reader->_filter_orc_batch(data, sel, size, arg)); + } + +private: + OrcReader* _reader = nullptr; +}; + +// Per-open mutable ORC state. close() publishes counters first, then resets this +// object so the reader can be opened again without carrying stale scan state. +struct OrcReaderScanState { + struct StripeRange { + uint64_t first_stripe = 0; + uint64_t last_stripe = 0; + uint64_t offset = 0; + uint64_t length = 0; + }; + + std::unique_ptr<::orc::Reader> reader; + const ::orc::Type* root_type = nullptr; + ::orc::ReaderMetrics reader_metrics; + ::orc::RowReaderOptions row_reader_options; // projection + filter + SARG + stripe range + std::string timezone = TimezoneUtils::default_time_zone; + cctz::time_zone timezone_obj; + std::unique_ptr<::orc::RowReader> row_reader; + const ::orc::Type* selected_type = nullptr; + std::unique_ptr<::orc::ColumnVectorBatch> batch; + + std::vector read_columns; + std::map column_to_selected_batch_index; + + uint64_t current_batch_first_row = 0; + uint64_t row_reader_range_first_row = 0; + uint64_t row_reader_range_end_row = 0; + uint64_t row_reader_range_rows = 0; + uint64_t condition_cache_next_row = 0; + std::shared_ptr condition_cache_ctx; + + std::vector orc_lazy_selected_rows; + size_t orc_lazy_input_rows = 0; + bool enable_lazy_materialization = true; + bool enable_filter_by_min_max = true; + bool orc_lazy_read_enabled = false; + bool orc_lazy_selection_valid = false; + + std::vector selected_stripe_ranges; + size_t current_stripe_range = 0; + bool stripe_pruning_applied = false; + + bool row_reader_created = false; +}; + +OrcReader::OrcReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + std::optional global_rowid_context, + bool enable_mapping_timestamp_tz) + : FileReader(system_properties, file_description, io_ctx, profile), + _global_rowid_context(std::move(global_rowid_context)), + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + +OrcReader::~OrcReader() = default; + +// Expose ORC pruning and lazy-read statistics in RuntimeProfile. These counters +// are the quickest way to confirm whether SARG/stripe pruning actually fired. +void OrcReader::_init_profile() { + if (_profile == nullptr) { + return; + } + + static const char* orc_profile = "OrcReader"; + ADD_TIMER_WITH_LEVEL(_profile, orc_profile, 1); + _orc_profile.reader_call = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReaderCall", TUnit::UNIT, orc_profile, 1); + _orc_profile.reader_inclusive_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "ReaderInclusiveLatencyUs", TUnit::UNIT, orc_profile, 1); + _orc_profile.decompression_call = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "DecompressionCall", + TUnit::UNIT, orc_profile, 1); + _orc_profile.decompression_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "DecompressionLatencyUs", TUnit::UNIT, orc_profile, 1); + _orc_profile.decoding_call = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "DecodingCall", TUnit::UNIT, orc_profile, 1); + _orc_profile.decoding_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "DecodingLatencyUs", + TUnit::UNIT, orc_profile, 1); + _orc_profile.byte_decoding_call = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ByteDecodingCall", TUnit::UNIT, orc_profile, 1); + _orc_profile.byte_decoding_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "ByteDecodingLatencyUs", TUnit::UNIT, orc_profile, 1); + _orc_profile.io_count = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "IOCount", TUnit::UNIT, orc_profile, 1); + _orc_profile.io_blocking_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "IOBlockingLatencyUs", TUnit::UNIT, orc_profile, 1); + _orc_profile.selected_row_group_count = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "SelectedRowGroupCount", TUnit::UNIT, orc_profile, 1); + _orc_profile.evaluated_row_group_count = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "EvaluatedRowGroupCount", TUnit::UNIT, orc_profile, 1); + _orc_profile.read_row_count = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReadRowCount", TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsFiltered", + TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); + _orc_profile.read_row_groups = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsReadNum", TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredRowsByGroup", + TUnit::UNIT, orc_profile, 1); + _orc_profile.lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "FilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_bytes = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); + _orc_profile.open_file_num = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); +} + +void OrcReader::_collect_profile() const { + if (_state == nullptr) { + return; + } + + const auto& reader_metrics = _state->reader_metrics; + const uint64_t read_row_count = orc_read_row_count(reader_metrics); + if (_profile != nullptr) { + COUNTER_UPDATE(_orc_profile.reader_call, orc_metric_value(reader_metrics.ReaderCall)); + COUNTER_UPDATE(_orc_profile.reader_inclusive_latency_us, + orc_metric_value(reader_metrics.ReaderInclusiveLatencyUs)); + COUNTER_UPDATE(_orc_profile.decompression_call, + orc_metric_value(reader_metrics.DecompressionCall)); + COUNTER_UPDATE(_orc_profile.decompression_latency_us, + orc_metric_value(reader_metrics.DecompressionLatencyUs)); + COUNTER_UPDATE(_orc_profile.decoding_call, orc_metric_value(reader_metrics.DecodingCall)); + COUNTER_UPDATE(_orc_profile.decoding_latency_us, + orc_metric_value(reader_metrics.DecodingLatencyUs)); + COUNTER_UPDATE(_orc_profile.byte_decoding_call, + orc_metric_value(reader_metrics.ByteDecodingCall)); + COUNTER_UPDATE(_orc_profile.byte_decoding_latency_us, + orc_metric_value(reader_metrics.ByteDecodingLatencyUs)); + COUNTER_UPDATE(_orc_profile.io_count, orc_metric_value(reader_metrics.IOCount)); + COUNTER_UPDATE(_orc_profile.io_blocking_latency_us, + orc_metric_value(reader_metrics.IOBlockingLatencyUs)); + int64_t selected_row_group_count = orc_metric_value(reader_metrics.SelectedRowGroupCount); + int64_t evaluated_row_group_count = orc_metric_value(reader_metrics.EvaluatedRowGroupCount); + if (_state->stripe_pruning_applied) { + selected_row_group_count = _reader_statistics.read_row_groups; + evaluated_row_group_count = + _reader_statistics.filtered_row_groups + _reader_statistics.read_row_groups; + } + COUNTER_UPDATE(_orc_profile.selected_row_group_count, selected_row_group_count); + COUNTER_UPDATE(_orc_profile.evaluated_row_group_count, evaluated_row_group_count); + COUNTER_UPDATE(_orc_profile.read_row_count, read_row_count); + COUNTER_UPDATE(_orc_profile.filtered_row_groups, _reader_statistics.filtered_row_groups); + COUNTER_UPDATE(_orc_profile.filtered_row_groups_by_min_max, + _reader_statistics.filtered_row_groups_by_min_max); + COUNTER_UPDATE(_orc_profile.read_row_groups, _reader_statistics.read_row_groups); + COUNTER_UPDATE(_orc_profile.filtered_group_rows, _reader_statistics.filtered_group_rows); + COUNTER_UPDATE(_orc_profile.lazy_read_filtered_rows, + _reader_statistics.lazy_read_filtered_rows); + COUNTER_UPDATE(_orc_profile.filtered_bytes, _reader_statistics.filtered_bytes); + COUNTER_UPDATE(_orc_profile.open_file_num, _reader_statistics.open_file_num); + } + if (_io_ctx != nullptr && _io_ctx->file_reader_stats != nullptr) { + _io_ctx->file_reader_stats->read_rows += read_row_count; + } +} + +format::ColumnDefinition OrcReader::row_position_column_definition() { + return format::row_position_column_definition(); +} + +Status OrcReader::init(RuntimeState* state) { + RETURN_IF_ERROR(format::FileReader::init(state)); + _state = std::make_unique(); + TimezoneUtils::find_cctz_time_zone(_state->timezone, _state->timezone_obj); + if (state != nullptr) { + _state->enable_lazy_materialization = state->query_options().enable_orc_lazy_mat; + _state->enable_filter_by_min_max = state->query_options().enable_orc_filter_by_min_max; + _state->timezone = state->timezone(); + _state->timezone_obj = state->timezone_obj(); + } + + ::orc::ReaderOptions options; + options.setMemoryPool(*ExecEnv::GetInstance()->orc_memory_pool()); + options.setReaderMetrics(&_state->reader_metrics); + + OrcFileInputStreamOptions input_stream_options; + const auto natural_read_size_mb = config::orc_natural_read_size_mb; + if (natural_read_size_mb <= 0 || natural_read_size_mb > 1024) { + return Status::InvalidArgument( + "Invalid orc_natural_read_size_mb {}, valid range is [1, 1024]", + natural_read_size_mb); + } + input_stream_options.natural_read_size = static_cast(natural_read_size_mb) << 20; + if (state != nullptr) { + input_stream_options.once_max_read_bytes = state->query_options().orc_once_max_read_bytes; + input_stream_options.max_merge_distance_bytes = + state->query_options().orc_max_merge_distance_bytes; + } + auto input_stream = std::make_unique( + _file_description->path, _file_reader, _io_ctx.get(), _profile, input_stream_options); + try { + _state->reader = ::orc::createReader(std::move(input_stream), options); + _state->root_type = &_state->reader->getType(); + } catch (const std::exception& e) { + if (is_orc_stop(_io_ctx.get(), e)) { + return Status::EndOfFile("stop"); + } + return Status::InternalError("Failed to open ORC file {}: {}", _file_description->path, + e.what()); + } + return Status::OK(); +} + +void OrcReader::_extend_condition_cache_context_for_current_range() { + if (_state->condition_cache_ctx == nullptr || + _state->condition_cache_ctx->filter_result == nullptr || + _state->condition_cache_ctx->is_hit) { + return; + } + const auto end_granule = (_state->row_reader_range_first_row + _state->row_reader_range_rows + + ConditionCacheContext::GRANULE_SIZE - 1) / + ConditionCacheContext::GRANULE_SIZE; + DORIS_CHECK(end_granule > static_cast(_state->condition_cache_ctx->base_granule)); + const auto required_granules = static_cast( + end_granule - static_cast(_state->condition_cache_ctx->base_granule)); + if (_state->condition_cache_ctx->filter_result->size() < required_granules) { + _state->condition_cache_ctx->filter_result->resize(required_granules, false); + } + _state->condition_cache_ctx->num_granules = + std::max(_state->condition_cache_ctx->num_granules, required_granules); +} + +void OrcReader::set_condition_cache_context(std::shared_ptr ctx) { + DORIS_CHECK(_state != nullptr); + _state->condition_cache_ctx = std::move(ctx); + if (_state->condition_cache_ctx != nullptr && + _state->condition_cache_ctx->filter_result != nullptr && + !_state->condition_cache_ctx->is_hit) { + _state->condition_cache_ctx->base_granule = static_cast( + _state->row_reader_range_first_row / ConditionCacheContext::GRANULE_SIZE); + _state->condition_cache_ctx->num_granules = 0; + _extend_condition_cache_context_for_current_range(); + } +} + +int64_t OrcReader::get_total_rows() const { + DORIS_CHECK(_state != nullptr); + if (_state->stripe_pruning_applied && _state->selected_stripe_ranges.empty()) { + return 0; + } + if (_state->row_reader != nullptr) { + return cast_set(_state->row_reader_range_rows); + } + if (_state->reader != nullptr) { + return cast_set(_state->reader->getNumberOfRows()); + } + return 0; +} + +DataTypePtr OrcReader::_convert_to_doris_type(const ::orc::Type& type) const { + DataTypePtr data_type; + switch (type.getKind()) { + case ::orc::TypeKind::BOOLEAN: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::BYTE: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::SHORT: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::INT: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::LONG: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::FLOAT: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::DOUBLE: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::STRING: + case ::orc::TypeKind::BINARY: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::VARCHAR: + data_type = std::make_shared(cast_set(type.getMaximumLength()), + PrimitiveType::TYPE_VARCHAR); + break; + case ::orc::TypeKind::CHAR: + data_type = std::make_shared(cast_set(type.getMaximumLength()), + PrimitiveType::TYPE_CHAR); + break; + case ::orc::TypeKind::DATE: + data_type = std::make_shared(); + break; + case ::orc::TypeKind::TIMESTAMP: + data_type = std::make_shared(6); + break; + case ::orc::TypeKind::TIMESTAMP_INSTANT: + if (_enable_mapping_timestamp_tz) { + data_type = std::make_shared(6); + } else { + data_type = std::make_shared(6); + } + break; + case ::orc::TypeKind::DECIMAL: + data_type = std::make_shared>( + type.getPrecision() == 0 ? DECIMAL_PRECISION_FOR_HIVE11 + : cast_set(type.getPrecision()), + type.getPrecision() == 0 ? DECIMAL_SCALE_FOR_HIVE11 + : cast_set(type.getScale())); + break; + case ::orc::TypeKind::LIST: + data_type = _convert_list_to_doris_type(type); + break; + case ::orc::TypeKind::MAP: + data_type = _convert_map_to_doris_type(type); + break; + case ::orc::TypeKind::STRUCT: + data_type = _convert_struct_to_doris_type(type); + break; + default: + throw doris::Exception( + Status::NotSupported("ORC type {} is not supported by new ORC reader", + static_cast(type.getKind()))); + } + return make_nullable(data_type); +} + +DataTypePtr OrcReader::_convert_list_to_doris_type(const ::orc::Type& type) const { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::LIST); + DORIS_CHECK(type.getSubtypeCount() == 1); + const auto* element_type = type.getSubtype(0); + DORIS_CHECK(element_type != nullptr); + return std::make_shared(_convert_to_doris_type(*element_type)); +} + +DataTypePtr OrcReader::_convert_map_to_doris_type(const ::orc::Type& type) const { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::MAP); + DORIS_CHECK(type.getSubtypeCount() == 2); + const auto* key_type = type.getSubtype(0); + const auto* value_type = type.getSubtype(1); + DORIS_CHECK(key_type != nullptr); + DORIS_CHECK(value_type != nullptr); + return std::make_shared(_convert_to_doris_type(*key_type), + _convert_to_doris_type(*value_type)); +} + +DataTypePtr OrcReader::_convert_struct_to_doris_type(const ::orc::Type& type) const { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::STRUCT); + DataTypes child_types; + Strings child_names; + child_types.reserve(type.getSubtypeCount()); + child_names.reserve(type.getSubtypeCount()); + for (uint64_t child_idx = 0; child_idx < type.getSubtypeCount(); ++child_idx) { + const auto* child_type = type.getSubtype(child_idx); + DORIS_CHECK(child_type != nullptr); + child_types.push_back(_convert_to_doris_type(*child_type)); + child_names.push_back(type.getFieldName(child_idx)); + } + return std::make_shared(child_types, child_names); +} + +Status OrcReader::_fill_schema_field(const ::orc::Type& type, int32_t local_id, + const std::string& field_name, + format::ColumnDefinition* const field) const { + if (field == nullptr) { + return Status::InvalidArgument("schema field is null"); + } + field->local_id = local_id; + field->name = field_name; + field->column_type = format::ColumnType::DATA_COLUMN; + if (type.hasAttributeKey(ORC_ICEBERG_ID_ATTRIBUTE)) { + const auto iceberg_id = type.getAttributeValue(ORC_ICEBERG_ID_ATTRIBUTE); + int32_t parsed_id = 0; + const auto* begin = iceberg_id.data(); + const auto* end = begin + iceberg_id.size(); + const auto [ptr, ec] = std::from_chars(begin, end, parsed_id); + if (ec != std::errc() || ptr != end) { + return Status::InvalidArgument("Invalid ORC Iceberg field id '{}' for column {}", + iceberg_id, field_name); + } + field->identifier = Field::create_field(parsed_id); + } + try { + field->type = _convert_to_doris_type(type); + } catch (const doris::Exception& e) { + return e.to_status(); + } + field->children.clear(); + switch (type.getKind()) { + case ::orc::TypeKind::STRUCT: + return _fill_struct_schema_children(type, field); + case ::orc::TypeKind::LIST: + return _fill_list_schema_children(type, field); + case ::orc::TypeKind::MAP: + return _fill_map_schema_children(type, field); + default: + break; + } + return Status::OK(); +} + +Status OrcReader::_fill_struct_schema_children(const ::orc::Type& type, + format::ColumnDefinition* const field) const { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::STRUCT); + field->children.reserve(type.getSubtypeCount()); + for (uint64_t child_idx = 0; child_idx < type.getSubtypeCount(); ++child_idx) { + const auto* child_type = type.getSubtype(child_idx); + DORIS_CHECK(child_type != nullptr); + const auto child_name = type.getFieldName(child_idx); + format::ColumnDefinition child_field; + RETURN_IF_ERROR(_fill_schema_field(*child_type, static_cast(child_idx), child_name, + &child_field)); + field->children.push_back(std::move(child_field)); + } + return Status::OK(); +} + +Status OrcReader::_fill_list_schema_children(const ::orc::Type& type, + format::ColumnDefinition* const field) const { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::LIST); + DORIS_CHECK(type.getSubtypeCount() == 1); + const auto* element_type = type.getSubtype(0); + DORIS_CHECK(element_type != nullptr); + + format::ColumnDefinition element_field; + RETURN_IF_ERROR(_fill_schema_field(*element_type, 0, ORC_LIST_ELEMENT_NAME, &element_field)); + field->children.push_back(std::move(element_field)); + return Status::OK(); +} + +Status OrcReader::_fill_map_schema_children(const ::orc::Type& type, + format::ColumnDefinition* const field) const { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::MAP); + DORIS_CHECK(type.getSubtypeCount() == 2); + const auto* key_type = type.getSubtype(0); + const auto* value_type = type.getSubtype(1); + DORIS_CHECK(key_type != nullptr); + DORIS_CHECK(value_type != nullptr); + + format::ColumnDefinition key_field; + RETURN_IF_ERROR(_fill_schema_field(*key_type, 0, ORC_MAP_KEY_NAME, &key_field)); + format::ColumnDefinition value_field; + RETURN_IF_ERROR(_fill_schema_field(*value_type, 1, ORC_MAP_VALUE_NAME, &value_field)); + + field->children.push_back(std::move(key_field)); + field->children.push_back(std::move(value_field)); + return Status::OK(); +} + +Status OrcReader::get_schema(std::vector* const file_schema) const { + if (file_schema == nullptr) { + return Status::InvalidArgument("file_schema is null"); + } + if (_state == nullptr || _state->root_type == nullptr) { + return Status::Uninitialized("OrcReader is not open"); + } + if (_state->root_type->getKind() != ::orc::TypeKind::STRUCT) { + return Status::NotSupported("ORC reader only supports top-level struct schema"); + } + file_schema->clear(); + const auto extra_columns = _global_rowid_context.has_value() ? 1 : 0; + file_schema->reserve(_state->root_type->getSubtypeCount() + extra_columns); + for (uint64_t child_idx = 0; child_idx < _state->root_type->getSubtypeCount(); ++child_idx) { + const auto* child_type = _state->root_type->getSubtype(child_idx); + DORIS_CHECK(child_type != nullptr); + const auto child_name = _state->root_type->getFieldName(child_idx); + format::ColumnDefinition field; + RETURN_IF_ERROR(_fill_schema_field(*child_type, static_cast(child_idx), child_name, + &field)); + file_schema->push_back(std::move(field)); + } + if (_global_rowid_context.has_value()) { + file_schema->push_back(nullable_global_rowid_column_definition()); + } + return Status::OK(); +} + +std::unique_ptr OrcReader::create_column_mapper( + format::TableColumnMapperOptions options) const { + return std::make_unique(std::move(options)); +} + +Status OrcReader::open(std::shared_ptr request) { + if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { + return Status::Uninitialized("OrcReader is not open"); + } + RETURN_IF_ERROR(format::FileReader::open(std::move(request))); + + if (_request->local_positions.empty()) { + size_t next_position = 0; + for (const auto& projection : _request->predicate_columns) { + if (_request->local_positions + .emplace(projection.column_id(), format::LocalIndex(next_position)) + .second) { + ++next_position; + } + } + for (const auto& projection : _request->non_predicate_columns) { + if (_request->local_positions + .emplace(projection.column_id(), format::LocalIndex(next_position)) + .second) { + ++next_position; + } + } + } + + _state->read_columns.clear(); + _state->read_columns.reserve(_request->predicate_columns.size() + + _request->non_predicate_columns.size()); + for (const auto& projection : _request->predicate_columns) { + _state->read_columns.push_back(projection.column_id()); + } + for (const auto& projection : _request->non_predicate_columns) { + _state->read_columns.push_back(projection.column_id()); + } + DCHECK(local_column_ids_are_unique(_request->predicate_columns)); + DCHECK(local_column_ids_are_unique(_request->non_predicate_columns)); + std::sort(_state->read_columns.begin(), _state->read_columns.end()); + _state->read_columns.erase( + std::unique(_state->read_columns.begin(), _state->read_columns.end()), + _state->read_columns.end()); + + _state->orc_lazy_read_enabled = _can_apply_orc_lazy_callback(); + + RETURN_IF_ERROR(_configure_row_reader_projection()); + RETURN_IF_ERROR(set_orc_reader_timezone(_state->timezone, &_state->row_reader_options)); + _state->row_reader_options.setEnableLazyDecoding(_state->orc_lazy_read_enabled); + _state->row_reader_options.setUseTightNumericVector(false); + + RETURN_IF_ERROR(_init_search_argument_from_local_filters()); + + // Seed the split byte range so this scanner only reads its own stripes. When SARG + // pruning applies, _apply_current_stripe_range() overwrites this with the (already + // split-constrained) stripe ranges; otherwise this seeded range governs selection. + _apply_split_range(); + + RETURN_IF_ERROR(_select_stripe_ranges_by_statistics()); + if (_state->stripe_pruning_applied && _state->selected_stripe_ranges.empty()) { + _eof = true; + return Status::OK(); + } + _apply_current_stripe_range(); + + RETURN_IF_ERROR(_create_row_reader()); + _eof = get_total_rows() == 0; + return Status::OK(); +} + +bool OrcReader::_can_apply_orc_lazy_callback() const { + if (!_state->enable_lazy_materialization) { + return false; + } + if (!_filter_has_row_level_predicates() || _request->predicate_columns.empty() || + _request->non_predicate_columns.empty()) { + return false; + } + bool has_physical_read_column = false; + for (const auto file_column_id : _state->read_columns) { + if (is_virtual_column(file_column_id)) { + continue; + } + has_physical_read_column = true; + if (find_request_projection(*_request, file_column_id) == nullptr) { + return false; + } + } + if (!has_physical_read_column) { + return false; + } + std::set decoded_columns; + for (const auto& projection : _request->predicate_columns) { + const auto file_column_id = projection.column_id(); + // Virtual row columns need the real batch start from row_reader->getRowNumber(). + if (is_virtual_column(file_column_id)) { + return false; + } + decoded_columns.insert(file_column_id); + } + if (decoded_columns.empty()) { + return false; + } + return _can_filter_with_decoded_columns(decoded_columns); +} + +Status OrcReader::_configure_row_reader_projection() { + const auto num_fields = static_cast(_state->root_type->getSubtypeCount()); + bool has_complex_projection = false; + for (const auto file_column_id : _state->read_columns) { + if (is_virtual_column(file_column_id)) { + DORIS_CHECK(_request->local_positions.contains(file_column_id)); + continue; + } + DORIS_CHECK(file_column_id.is_valid() && file_column_id.value() < num_fields); + DORIS_CHECK(_request->local_positions.contains(file_column_id)); + const auto* projection = find_request_projection(*_request, file_column_id); + DORIS_CHECK(projection != nullptr); + has_complex_projection = has_complex_projection || has_pruned_projection(*projection); + } + if (!has_complex_projection) { + std::list include_columns; + for (const auto file_column_id : _state->read_columns) { + if (is_virtual_column(file_column_id)) { + continue; + } + include_columns.push_back(static_cast(file_column_id.value())); + } + _state->row_reader_options.include(include_columns); + if (_state->orc_lazy_read_enabled) { + std::list filter_columns; + for (const auto& projection : _request->predicate_columns) { + if (is_row_position_column(projection.column_id())) { + continue; + } + DORIS_CHECK(!is_virtual_column(projection.column_id())); + // ORC RowReader lazy filter uses column names (or type ids) to mark leaders. + // Passing field indexes does not activate the callback in this ORC build. + filter_columns.push_back( + _state->root_type->getFieldName(projection.column_id().value())); + } + if (filter_columns.empty()) { + DORIS_CHECK(!include_columns.empty()); + filter_columns.push_back(_state->root_type->getFieldName(include_columns.front())); + } + _state->row_reader_options.filter(filter_columns); + } + return Status::OK(); + } + + std::set include_type_ids; + include_type_ids.insert(_state->root_type->getColumnId()); + for (const auto file_column_id : _state->read_columns) { + if (is_virtual_column(file_column_id)) { + continue; + } + const auto* type = + _state->root_type->getSubtype(static_cast(file_column_id.value())); + DORIS_CHECK(type != nullptr); + const auto* projection = find_request_projection(*_request, file_column_id); + DORIS_CHECK(projection != nullptr); + if (!has_pruned_projection(*projection)) { + collect_type_and_descendant_ids(*type, &include_type_ids); + continue; + } + RETURN_IF_ERROR(collect_projected_type_ids(*type, *projection, &include_type_ids)); + } + std::list include_type_id_list(include_type_ids.begin(), include_type_ids.end()); + _state->row_reader_options.includeTypes(include_type_id_list); + if (_state->orc_lazy_read_enabled) { + std::set filter_type_ids; + RETURN_IF_ERROR(collect_lazy_filter_type_ids( + *_state->root_type, _request->predicate_columns, &filter_type_ids)); + DORIS_CHECK(!filter_type_ids.empty()); + std::list filter_type_id_list(filter_type_ids.begin(), filter_type_ids.end()); + _state->row_reader_options.filterTypes(filter_type_id_list); + } + return Status::OK(); +} + +Status OrcReader::_init_search_argument_from_local_filters() { + if (!_state->enable_filter_by_min_max || _request->conjuncts.empty()) { + return Status::OK(); + } + + try { + auto builder = ::orc::SearchArgumentFactory::newBuilder(); + bool has_pushdown = false; + builder->startAnd(); + for (const auto& conjunct : _request->conjuncts) { + if (conjunct == nullptr) { + continue; + } + has_pushdown = + build_orc_search_argument(*_request, *_state->root_type, _state->timezone_obj, + conjunct->root(), builder) || + has_pushdown; + } + if (!has_pushdown) { + return Status::OK(); + } + builder->end(); + _state->row_reader_options.searchArgument(builder->build()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to build ORC search argument: {}", e.what()); + } + return Status::OK(); +} + +void OrcReader::_split_byte_window(uint64_t* start, uint64_t* end) const { + const int64_t range_start = _file_description->range_start_offset; + const int64_t range_size = _file_description->range_size; + DORIS_CHECK(range_start >= 0); + *start = static_cast(range_start); + if (range_size < 0) { + // Unset sentinel: read the whole file. + *end = std::numeric_limits::max(); + return; + } + DORIS_CHECK(range_size <= std::numeric_limits::max() - range_start); + const int64_t range_end = range_start + range_size; + DORIS_CHECK(range_end >= range_start); + *end = static_cast(range_end); +} + +Status OrcReader::_collect_split_stripes(std::vector* stripe_indices) const { + DORIS_CHECK(stripe_indices != nullptr); + stripe_indices->clear(); + const auto stripe_count = _state->reader->getNumberOfStripes(); + stripe_indices->reserve(stripe_count); + + uint64_t split_start = 0; + uint64_t split_end = std::numeric_limits::max(); + _split_byte_window(&split_start, &split_end); + + for (uint64_t stripe_index = 0; stripe_index < stripe_count; ++stripe_index) { + std::unique_ptr<::orc::StripeInformation> stripe_information; + try { + stripe_information = _state->reader->getStripe(stripe_index); + } catch (const std::exception& e) { + return Status::InternalError("Failed to read ORC stripe info: {}", e.what()); + } + DORIS_CHECK(stripe_information != nullptr); + const auto stripe_offset = stripe_information->getOffset(); + if (stripe_offset >= split_start && stripe_offset < split_end) { + stripe_indices->push_back(stripe_index); + } + } + return Status::OK(); +} + +void OrcReader::_apply_split_range() { + uint64_t start = 0; + uint64_t end = std::numeric_limits::max(); + _split_byte_window(&start, &end); + if (start == 0 && end == std::numeric_limits::max()) { + // Whole file: keep ORC library defaults and avoid uint64 overflow on (start + length). + return; + } + _state->row_reader_options.range(start, end - start); +} + +// ORC RowReader ranges are continuous, so non-adjacent surviving stripes are +// compacted into separate ranges. +Status OrcReader::_select_stripe_ranges_by_statistics() { + _state->selected_stripe_ranges.clear(); + _state->current_stripe_range = 0; + _state->stripe_pruning_applied = false; + const bool has_search_argument = _state->row_reader_options.getSearchArgument() != nullptr; + if (!has_search_argument) { + return Status::OK(); + } + + std::vector split_stripes; + RETURN_IF_ERROR(_collect_split_stripes(&split_stripes)); + if (split_stripes.empty()) { + return Status::OK(); + } + + std::vector sarg_needed_stripes; + try { + sarg_needed_stripes = _state->reader->getNeedReadStripes(_state->row_reader_options); + } catch (const std::exception& e) { + return Status::InternalError("Failed to evaluate ORC search argument: {}", e.what()); + } + + std::vector selected_stripes; + selected_stripes.reserve(split_stripes.size()); + int64_t filtered_stripes = 0; + int64_t filtered_rows = 0; + int64_t filtered_bytes = 0; + for (const auto stripe_index : split_stripes) { + bool drop = false; + if (stripe_index < sarg_needed_stripes.size() && sarg_needed_stripes[stripe_index] == 0) { + drop = true; + } + if (!drop) { + selected_stripes.push_back(stripe_index); + continue; + } + + ++filtered_stripes; + try { + const auto stripe = _state->reader->getStripe(stripe_index); + filtered_rows += cast_set(stripe->getNumberOfRows()); + filtered_bytes += cast_set(stripe->getLength()); + } catch (const std::exception&) { + } + } + + if (filtered_stripes == 0) { + return Status::OK(); + } + + _state->stripe_pruning_applied = true; + _reader_statistics.filtered_row_groups = cast_set(filtered_stripes); + _reader_statistics.filtered_row_groups_by_min_max = cast_set(filtered_stripes); + _reader_statistics.filtered_group_rows = filtered_rows; + _reader_statistics.filtered_bytes = filtered_bytes; + _reader_statistics.read_row_groups = cast_set(selected_stripes.size()); + if (selected_stripes.empty()) { + return Status::OK(); + } + + auto append_stripe_range = [&](uint64_t first_stripe, uint64_t last_stripe) -> Status { + DORIS_CHECK(first_stripe < last_stripe); + try { + const auto first = _state->reader->getStripe(first_stripe); + const auto last = _state->reader->getStripe(last_stripe - 1); + const auto offset = first->getOffset(); + const auto end_offset = last->getOffset() + last->getLength(); + DORIS_CHECK(end_offset > offset); + _state->selected_stripe_ranges.push_back(OrcReaderScanState::StripeRange { + .first_stripe = first_stripe, + .last_stripe = last_stripe, + .offset = offset, + .length = end_offset - offset, + }); + } catch (const std::exception& e) { + return Status::InternalError("Failed to build ORC stripe read range: {}", e.what()); + } + return Status::OK(); + }; + + uint64_t range_first = selected_stripes.front(); + uint64_t previous = range_first; + for (size_t idx = 1; idx < selected_stripes.size(); ++idx) { + const auto stripe_index = selected_stripes[idx]; + if (stripe_index == previous + 1) { + previous = stripe_index; + continue; + } + RETURN_IF_ERROR(append_stripe_range(range_first, previous + 1)); + range_first = stripe_index; + previous = stripe_index; + } + RETURN_IF_ERROR(append_stripe_range(range_first, previous + 1)); + return Status::OK(); +} + +void OrcReader::_apply_current_stripe_range() { + if (!_state->stripe_pruning_applied || _state->selected_stripe_ranges.empty()) { + return; + } + DORIS_CHECK(_state->current_stripe_range < _state->selected_stripe_ranges.size()); + const auto& stripe_range = _state->selected_stripe_ranges[_state->current_stripe_range]; + _state->row_reader_options.range(stripe_range.offset, stripe_range.length); +} + +Status OrcReader::_advance_to_next_stripe_range(bool* advanced) { + DORIS_CHECK(advanced != nullptr); + *advanced = false; + if (!_state->stripe_pruning_applied || _state->selected_stripe_ranges.empty() || + _state->current_stripe_range + 1 >= _state->selected_stripe_ranges.size()) { + return Status::OK(); + } + ++_state->current_stripe_range; + _apply_current_stripe_range(); + RETURN_IF_ERROR(_create_row_reader()); + *advanced = true; + return Status::OK(); +} + +Status OrcReader::_create_row_reader() { + try { + if (_state->orc_lazy_read_enabled && _orc_filter == nullptr) { + _orc_filter = std::make_unique(this); + } + _state->row_reader = _state->reader->createRowReader( + _state->row_reader_options, + _state->orc_lazy_read_enabled ? _orc_filter.get() : nullptr); + _state->selected_type = &_state->row_reader->getSelectedType(); + DORIS_CHECK(_state->selected_type->getKind() == ::orc::TypeKind::STRUCT); + _state->batch = _state->row_reader->createRowBatch(DEFAULT_ORC_READ_BATCH_SIZE); + _state->orc_lazy_selection_valid = false; + _state->orc_lazy_selected_rows.clear(); + _state->orc_lazy_input_rows = 0; + const uint64_t file_total_rows = _state->reader->getNumberOfRows(); + const auto initial_row_number = _state->row_reader->getRowNumber(); + _state->row_reader_range_rows = _state->row_reader->getNumberOfRows(); + if (initial_row_number == std::numeric_limits::max()) { + _state->row_reader_range_first_row = 0; + } else if (initial_row_number >= file_total_rows) { + _state->row_reader_range_first_row = file_total_rows; + } else { + _state->row_reader_range_first_row = initial_row_number + 1; + } + DORIS_CHECK(_state->row_reader_range_first_row <= file_total_rows); + DORIS_CHECK(_state->row_reader_range_rows <= + file_total_rows - _state->row_reader_range_first_row); + _state->row_reader_range_end_row = + _state->row_reader_range_first_row + _state->row_reader_range_rows; + _state->condition_cache_next_row = _state->row_reader_range_first_row; + _extend_condition_cache_context_for_current_range(); + _state->column_to_selected_batch_index.clear(); + size_t physical_read_column_count = 0; + for (const auto file_column_id : _state->read_columns) { + physical_read_column_count += !is_virtual_column(file_column_id); + } + for (uint64_t selected_idx = 0; selected_idx < _state->selected_type->getSubtypeCount(); + ++selected_idx) { + const auto field_name = _state->selected_type->getFieldName(selected_idx); + for (const auto file_column_id : _state->read_columns) { + if (is_virtual_column(file_column_id)) { + continue; + } + if (field_name == _state->root_type->getFieldName(file_column_id.value())) { + _state->column_to_selected_batch_index.emplace( + file_column_id, static_cast(selected_idx)); + break; + } + } + } + DORIS_CHECK(_state->column_to_selected_batch_index.size() == physical_read_column_count); + _state->row_reader_created = true; + } catch (const std::exception& e) { + return Status::InternalError("Failed to create ORC row reader: {}", e.what()); + } + return Status::OK(); +} + +void OrcReader::_skip_condition_cache_false_granules(size_t* rows, bool* eof) { + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + if (_state->condition_cache_ctx == nullptr || !_state->condition_cache_ctx->is_hit) { + return; + } + DORIS_CHECK(_state->condition_cache_ctx->filter_result != nullptr); + const auto base_granule = _state->condition_cache_ctx->base_granule; + const auto& cache = *_state->condition_cache_ctx->filter_result; + constexpr uint64_t granule_size = ConditionCacheContext::GRANULE_SIZE; + const uint64_t file_total_rows = _state->reader->getNumberOfRows(); + DORIS_CHECK(_state->condition_cache_next_row <= file_total_rows); + + const auto current_granule = + static_cast(_state->condition_cache_next_row / granule_size); + const auto cache_idx_offset = current_granule - base_granule; + if (cache_idx_offset < 0 || static_cast(cache_idx_offset) >= cache.size()) { + return; + } + size_t cache_idx = static_cast(cache_idx_offset); + while (cache_idx < cache.size() && !cache[cache_idx]) { + ++cache_idx; + } + if (cache_idx >= cache.size()) { + if (_state->row_reader_range_end_row <= _state->condition_cache_next_row) { + return; + } + const auto last_range_granule = + static_cast((_state->row_reader_range_end_row - 1) / granule_size); + const auto last_cache_idx_offset = last_range_granule - base_granule; + if (last_cache_idx_offset < 0 || + static_cast(last_cache_idx_offset) >= cache.size()) { + return; + } + _state->row_reader->seekToRow(_state->row_reader_range_end_row); + if (_io_ctx != nullptr) { + _io_ctx->condition_cache_filtered_rows += + _state->row_reader_range_end_row - _state->condition_cache_next_row; + } + _state->condition_cache_next_row = _state->row_reader_range_end_row; + return; + } + + const auto target_granule_offset = base_granule + static_cast(cache_idx); + DORIS_CHECK(target_granule_offset >= 0); + const auto target_granule = static_cast(target_granule_offset); + const uint64_t target_row = target_granule * granule_size; + if (target_row >= _state->row_reader_range_end_row) { + if (_state->row_reader_range_end_row > _state->condition_cache_next_row) { + _state->row_reader->seekToRow(_state->row_reader_range_end_row); + if (_io_ctx != nullptr) { + _io_ctx->condition_cache_filtered_rows += + _state->row_reader_range_end_row - _state->condition_cache_next_row; + } + _state->condition_cache_next_row = _state->row_reader_range_end_row; + } + return; + } + if (target_row > _state->condition_cache_next_row) { + DORIS_CHECK(target_row <= file_total_rows); + _state->row_reader->seekToRow(target_row); + if (_io_ctx != nullptr) { + _io_ctx->condition_cache_filtered_rows += target_row - _state->condition_cache_next_row; + } + _state->condition_cache_next_row = target_row; + } +} + +void OrcReader::_mark_condition_cache_surviving_rows(const IColumn::Filter& keep_filter, + size_t rows) const { + DORIS_CHECK(keep_filter.size() == rows); + if (_state->condition_cache_ctx == nullptr || _state->condition_cache_ctx->is_hit) { + return; + } + DORIS_CHECK(_state->condition_cache_ctx->filter_result != nullptr); + const auto base_granule = _state->condition_cache_ctx->base_granule; + auto& cache = *_state->condition_cache_ctx->filter_result; + constexpr uint64_t granule_size = ConditionCacheContext::GRANULE_SIZE; + for (size_t row = 0; row < rows; ++row) { + if (keep_filter[row] == 0) { + continue; + } + const auto granule = + static_cast((_state->current_batch_first_row + row) / granule_size); + const auto cache_idx = granule - base_granule; + if (cache_idx >= 0 && static_cast(cache_idx) < cache.size()) { + cache[static_cast(cache_idx)] = true; + } + } +} + +void OrcReader::_mark_condition_cache_selected_rows( + size_t rows, const std::vector* selected_rows) const { + if (_state->condition_cache_ctx == nullptr || _state->condition_cache_ctx->is_hit) { + return; + } + DORIS_CHECK(_state->condition_cache_ctx->filter_result != nullptr); + const auto base_granule = _state->condition_cache_ctx->base_granule; + auto& cache = *_state->condition_cache_ctx->filter_result; + constexpr uint64_t granule_size = ConditionCacheContext::GRANULE_SIZE; + const auto mark_row = [&](size_t row) { + DORIS_CHECK(row < rows); + const auto granule = + static_cast((_state->current_batch_first_row + row) / granule_size); + const auto cache_idx = granule - base_granule; + if (cache_idx >= 0 && static_cast(cache_idx) < cache.size()) { + cache[static_cast(cache_idx)] = true; + } + }; + if (selected_rows == nullptr) { + for (size_t row = 0; row < rows; ++row) { + mark_row(row); + } + return; + } + for (const auto row : *selected_rows) { + mark_row(row); + } +} + +Status OrcReader::_filter_orc_batch(::orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t size, + void* /*arg*/) { + if (!_state->orc_lazy_read_enabled || sel == nullptr || size == 0) { + data.numElements = size; + return Status::OK(); + } + if (size > DEFAULT_ORC_READ_BATCH_SIZE) { + return Status::InvalidArgument("ORC lazy filter batch size {} exceeds {}", size, + DEFAULT_ORC_READ_BATCH_SIZE); + } + + auto* struct_batch = dynamic_cast<::orc::StructVectorBatch*>(&data); + if (struct_batch == nullptr) { + return Status::InternalError("New ORC lazy filter expects struct row batch"); + } + + std::vector> position_to_column( + _request->local_positions.size()); + for (const auto& [file_column_id, local_position] : _request->local_positions) { + const auto position = local_position.value(); + if (position >= position_to_column.size()) { + return Status::InvalidArgument("ORC scan local position {} is out of range {}", + position, position_to_column.size()); + } + if (position_to_column[position].has_value()) { + return Status::InvalidArgument("ORC scan local position {} is duplicated", position); + } + position_to_column[position] = file_column_id; + } + + Block filter_block; + filter_block.reserve(position_to_column.size()); + for (size_t position = 0; position < position_to_column.size(); ++position) { + if (!position_to_column[position].has_value()) { + return Status::InvalidArgument("ORC scan local positions are not dense at {}", + position); + } + const auto file_column_id = *position_to_column[position]; + if (is_row_position_column(file_column_id)) { + auto field = row_position_column_definition(); + filter_block.insert({field.type->create_column(), field.type, field.name}); + continue; + } + if (is_global_rowid_column(file_column_id)) { + auto field = nullable_global_rowid_column_definition(); + filter_block.insert({field.type->create_column(), field.type, field.name}); + continue; + } + const auto batch_index_it = _state->column_to_selected_batch_index.find(file_column_id); + DORIS_CHECK(batch_index_it != _state->column_to_selected_batch_index.end()); + const auto* selected_type = _state->selected_type->getSubtype(batch_index_it->second); + DORIS_CHECK(selected_type != nullptr); + auto column_type = _convert_to_doris_type(*selected_type); + if (column_type == nullptr) { + return Status::NotSupported("ORC type {} is not supported by new ORC reader", + static_cast(selected_type->getKind())); + } + filter_block.insert({column_type->create_column(), column_type, + _state->root_type->getFieldName(file_column_id.value())}); + } + + std::set decoded_columns; + RETURN_IF_ERROR(_decode_columns(*struct_batch, _request->predicate_columns, size, &filter_block, + &decoded_columns)); + + IColumn::Filter keep_filter(size, 1); + RETURN_IF_ERROR(_build_keep_filter(&filter_block, size, &keep_filter)); + + _state->orc_lazy_selected_rows.clear(); + _state->orc_lazy_selected_rows.reserve(size); + uint16_t selected_rows = 0; + for (uint16_t row = 0; row < size; ++row) { + if (keep_filter[row] == 0) { + continue; + } + sel[selected_rows++] = row; + _state->orc_lazy_selected_rows.push_back(row); + } + _state->orc_lazy_input_rows = size; + _state->orc_lazy_selection_valid = true; + data.numElements = selected_rows; + const auto filtered_rows = cast_set(size - selected_rows); + _reader_statistics.lazy_read_filtered_rows += filtered_rows; + return Status::OK(); +} + +Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type& selected_type, + const ::orc::ColumnVectorBatch& batch, MutableColumnPtr& column, + size_t rows, const std::vector* selected_rows) const { + DORIS_CHECK(file_type.getKind() == selected_type.getKind()); + DORIS_CHECK(column->is_nullable()); + const auto column_type = _convert_to_doris_type(selected_type); + OrcDecodedColumnView view; + view.file_type = &file_type; + view.selected_type = &selected_type; + view.batch = &batch; + view.rows = rows; + view.selected_rows = selected_rows; + view.timezone = &_state->timezone_obj; + view.enable_mapping_timestamp_tz = _enable_mapping_timestamp_tz; + return column_type->get_serde()->read_column_from_orc(*column, view); +} + +Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + if (_state == nullptr) { + return Status::Uninitialized("OrcReader is not open"); + } + *rows = 0; + file_block->clear_column_data(file_block->columns()); + if (_io_ctx != nullptr && _io_ctx->should_stop) { + *eof = true; + _eof = true; + return Status::OK(); + } + if (_eof) { + *eof = true; + return Status::OK(); + } + if (!_state->row_reader_created || _state->batch == nullptr) { + return Status::Uninitialized("OrcReader is not open"); + } + + bool has_next = false; + while (true) { + _skip_condition_cache_false_granules(rows, eof); + if (*eof) { + return Status::OK(); + } + try { + _state->orc_lazy_selection_valid = false; + _state->orc_lazy_selected_rows.clear(); + _state->orc_lazy_input_rows = 0; + has_next = _state->row_reader->next(*_state->batch); + } catch (const std::exception& e) { + if (is_orc_stop(_io_ctx.get(), e)) { + file_block->clear_column_data(file_block->columns()); + *rows = 0; + *eof = true; + _eof = true; + return Status::OK(); + } + return Status::InternalError("Orc row reader nextBatch failed. reason = {}", e.what()); + } + if (has_next) { + if (_state->orc_lazy_read_enabled && _state->orc_lazy_selection_valid && + _state->batch != nullptr && _state->batch->numElements == 0 && + _state->orc_lazy_input_rows > 0 && _state->orc_lazy_selected_rows.empty()) { + // ORC lazy can read rows but return an empty batch when the callback rejects all + // rows. Keep pulling so callers either see real rows or a clean EOF. + continue; + } + break; + } + bool advanced = false; + RETURN_IF_ERROR(_advance_to_next_stripe_range(&advanced)); + if (!advanced) { + _eof = true; + *eof = true; + return Status::OK(); + } + } + + const auto batch_rows = static_cast(_state->batch->numElements); + const auto batch_first_row = _state->row_reader->getRowNumber(); + _state->current_batch_first_row = batch_first_row; + _state->condition_cache_next_row = _state->current_batch_first_row + batch_rows; + auto* struct_batch = dynamic_cast<::orc::StructVectorBatch*>(_state->batch.get()); + if (struct_batch == nullptr) { + return Status::InternalError("New ORC reader expects struct row batch"); + } + + const bool orc_lazy_read_applied = + _state->orc_lazy_read_enabled && _state->orc_lazy_selection_valid; + if (orc_lazy_read_applied && _state->orc_lazy_input_rows != batch_rows) { + return Status::InternalError("ORC lazy filter selected from {} rows but batch has {} rows", + _state->orc_lazy_input_rows, batch_rows); + } + + std::set decoded_columns; + RETURN_IF_ERROR(_decode_columns(*struct_batch, _request->predicate_columns, batch_rows, + file_block, &decoded_columns)); + + const auto columns_decoded_before_selection = decoded_columns; + IColumn::Filter keep_filter; + size_t selected_rows = batch_rows; + std::vector selected_row_indices; + const std::vector* non_predicate_selected_rows = nullptr; + if (orc_lazy_read_applied) { + selected_rows = _state->orc_lazy_selected_rows.size(); + if (selected_rows != batch_rows) { + keep_filter.resize(batch_rows); + std::fill(keep_filter.begin(), keep_filter.end(), 0); + for (const auto row : _state->orc_lazy_selected_rows) { + DORIS_CHECK(row < batch_rows); + keep_filter[row] = 1; + } + selected_row_indices = _state->orc_lazy_selected_rows; + non_predicate_selected_rows = &selected_row_indices; + } + _mark_condition_cache_selected_rows(batch_rows, non_predicate_selected_rows); + } + + RETURN_IF_ERROR(_decode_columns(*struct_batch, _request->non_predicate_columns, batch_rows, + file_block, &decoded_columns, non_predicate_selected_rows)); + + *rows = batch_rows; + if (orc_lazy_read_applied) { + if (selected_rows != batch_rows) { + _filter_decoded_columns(file_block, keep_filter, selected_rows, + columns_decoded_before_selection); + *rows = selected_rows; + } + _state->orc_lazy_selection_valid = false; + } else { + RETURN_IF_ERROR(_filter_block(file_block, rows)); + } + *eof = false; + return Status::OK(); +} + +Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& request, + format::FileAggregateResult* result) { + DORIS_CHECK(result != nullptr); + if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { + return Status::Uninitialized("OrcReader is not open"); + } + + result->count = 0; + result->columns.clear(); + if (request.agg_type != TPushAggOp::type::COUNT && + request.agg_type != TPushAggOp::type::MINMAX) { + return Status::NotSupported("Unsupported ORC aggregate pushdown type {}", request.agg_type); + } + + std::vector selected_stripes; + if (_state->stripe_pruning_applied) { + for (const auto& stripe_range : _state->selected_stripe_ranges) { + if (stripe_range.last_stripe < stripe_range.first_stripe || + stripe_range.last_stripe > _state->reader->getNumberOfStripes()) { + return Status::InternalError("Invalid ORC stripe range {}-{}", + stripe_range.first_stripe, stripe_range.last_stripe); + } + for (uint64_t stripe_index = stripe_range.first_stripe; + stripe_index < stripe_range.last_stripe; ++stripe_index) { + selected_stripes.push_back(stripe_index); + } + } + } else { + RETURN_IF_ERROR(_collect_split_stripes(&selected_stripes)); + } + + for (const auto stripe_index : selected_stripes) { + std::unique_ptr<::orc::StripeInformation> stripe_information; + try { + stripe_information = _state->reader->getStripe(stripe_index); + } catch (const std::exception& e) { + return Status::InternalError("Failed to read ORC stripe {}: {}", stripe_index, + e.what()); + } + DORIS_CHECK(stripe_information != nullptr); + result->count += cast_set(stripe_information->getNumberOfRows()); + } + + if (request.agg_type == TPushAggOp::type::COUNT) { + if (request.columns.empty()) { + return Status::OK(); + } + if (request.columns.size() != 1) { + return Status::NotSupported("ORC COUNT pushdown only supports one count column"); + } + const auto& count_projection = request.columns[0].projection; + if (!count_projection.project_all_children || !count_projection.children.empty()) { + return Status::NotSupported( + "ORC COUNT pushdown only supports top-level column projection"); + } + if (count_projection.local_id() < 0 || + count_projection.local_id() >= + static_cast(_state->root_type->getSubtypeCount())) { + return Status::InvalidArgument("Invalid ORC COUNT aggregate column id {}", + count_projection.local_id()); + } + const auto* count_type = + _state->root_type->getSubtype(static_cast(count_projection.local_id())); + DORIS_CHECK(count_type != nullptr); + + result->count = 0; + const auto stripe_statistics_count = _state->reader->getNumberOfStripeStatistics(); + for (const auto stripe_index : selected_stripes) { + if (stripe_index >= stripe_statistics_count) { + return Status::NotSupported( + "Missing ORC stripe statistics for COUNT column kind {} in stripe {}", + static_cast(count_type->getKind()), stripe_index); + } + std::unique_ptr<::orc::StripeStatistics> stripe_statistics; + try { + stripe_statistics = _state->reader->getStripeStatistics(stripe_index); + } catch (const std::exception& e) { + return Status::InternalError("Failed to read ORC stripe statistics {}: {}", + stripe_index, e.what()); + } + if (stripe_statistics == nullptr) { + return Status::NotSupported("Missing ORC stripe statistics for stripe {}", + stripe_index); + } + const auto* column_statistics = stripe_statistics->getColumnStatistics( + cast_set(count_type->getColumnId())); + if (column_statistics == nullptr) { + return Status::NotSupported( + "Missing ORC COUNT statistics for column kind {} in stripe {}", + static_cast(count_type->getKind()), stripe_index); + } + result->count += cast_set(column_statistics->getNumberOfValues()); + } + return Status::OK(); + } + + result->columns.resize(request.columns.size()); + if (selected_stripes.empty()) { + return Status::NotSupported("No ORC stripe selected for min/max pushdown"); + } + + const auto stripe_statistics_count = _state->reader->getNumberOfStripeStatistics(); + for (size_t column_idx = 0; column_idx < request.columns.size(); ++column_idx) { + const auto& request_column = request.columns[column_idx]; + const ::orc::Type* leaf_type = nullptr; + RETURN_IF_ERROR(find_projected_minmax_leaf(*_state->root_type, request_column.projection, + &leaf_type)); + DORIS_CHECK(leaf_type != nullptr); + + auto& aggregate_column = result->columns[column_idx]; + aggregate_column.projection = request_column.projection; + for (const auto stripe_index : selected_stripes) { + if (stripe_index >= stripe_statistics_count) { + return Status::NotSupported( + "Missing ORC stripe statistics for stripe {} and column kind {}", + stripe_index, static_cast(leaf_type->getKind())); + } + std::unique_ptr<::orc::StripeStatistics> stripe_statistics; + try { + stripe_statistics = _state->reader->getStripeStatistics(stripe_index); + } catch (const std::exception& e) { + return Status::InternalError("Failed to read ORC stripe statistics {}: {}", + stripe_index, e.what()); + } + if (stripe_statistics == nullptr) { + return Status::NotSupported("Missing ORC stripe statistics for stripe {}", + stripe_index); + } + const auto* column_statistics = stripe_statistics->getColumnStatistics( + cast_set(leaf_type->getColumnId())); + if (column_statistics == nullptr) { + return Status::NotSupported( + "Missing ORC min/max statistics for column kind {} in stripe {}", + static_cast(leaf_type->getKind()), stripe_index); + } + + segment_v2::ZoneMap zone_map; + if (!build_zone_map_from_orc_statistics(*leaf_type, *column_statistics, + _state->timezone_obj, + _enable_mapping_timestamp_tz, &zone_map)) { + return Status::NotSupported( + "Missing ORC min/max statistics for column kind {} in stripe {}", + static_cast(leaf_type->getKind()), stripe_index); + } + if (!zone_map.has_not_null) { + continue; + } + if (!aggregate_column.has_min || zone_map.min_value < aggregate_column.min_value) { + aggregate_column.min_value = zone_map.min_value; + aggregate_column.has_min = true; + } + if (!aggregate_column.has_max || aggregate_column.max_value < zone_map.max_value) { + aggregate_column.max_value = zone_map.max_value; + aggregate_column.has_max = true; + } + } + if (!aggregate_column.has_min || !aggregate_column.has_max) { + return Status::NotSupported("No ORC non-null statistics selected for min/max pushdown"); + } + } + return Status::OK(); +} + +Status OrcReader::_decode_column_into_block(const ::orc::StructVectorBatch& struct_batch, + format::LocalColumnId file_column_id, size_t rows, + Block* file_block, + const std::vector* selected_rows) const { + DORIS_CHECK(file_block != nullptr); + if (is_virtual_column(file_column_id)) { + return Status::OK(); + } + const auto position_it = _request->local_positions.find(file_column_id); + DORIS_CHECK(position_it != _request->local_positions.end()); + const auto block_position = position_it->second; + DORIS_CHECK(block_position.value() < file_block->columns()); + const auto* type = _state->root_type->getSubtype(static_cast(file_column_id.value())); + DORIS_CHECK(type != nullptr); + const auto batch_index_it = _state->column_to_selected_batch_index.find(file_column_id); + DORIS_CHECK(batch_index_it != _state->column_to_selected_batch_index.end()); + const size_t selected_batch_idx = batch_index_it->second; + DORIS_CHECK(selected_batch_idx < struct_batch.fields.size()); + const auto* selected_type = _state->selected_type->getSubtype(selected_batch_idx); + DORIS_CHECK(selected_type != nullptr); + auto column = file_block->get_by_position(block_position.value()).column->assert_mutable(); + RETURN_IF_ERROR(_decode_column(*type, *selected_type, *struct_batch.fields[selected_batch_idx], + column, rows, selected_rows)); + file_block->replace_by_position(block_position.value(), std::move(column)); + return Status::OK(); +} + +Status OrcReader::_decode_columns(const ::orc::StructVectorBatch& struct_batch, + const std::vector& projections, + size_t rows, Block* file_block, + std::set* decoded_columns, + const std::vector* selected_rows) const { + DORIS_CHECK(decoded_columns != nullptr); + for (const auto& projection : projections) { + const auto file_column_id = projection.column_id(); + if (!decoded_columns->insert(file_column_id).second) { + continue; + } + if (is_row_position_column(file_column_id)) { + _fill_row_position_column(file_block, rows, selected_rows); + } else if (is_global_rowid_column(file_column_id)) { + RETURN_IF_ERROR(_fill_global_rowid_column(file_block, rows, selected_rows)); + } else { + RETURN_IF_ERROR(_decode_column_into_block(struct_batch, file_column_id, rows, + file_block, selected_rows)); + } + } + return Status::OK(); +} + +// The row-position virtual column stores the original physical ORC row number. +void OrcReader::_fill_row_position_column(Block* file_block, size_t rows, + const std::vector* selected_rows) const { + const auto position_it = + _request->local_positions.find(format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)); + if (position_it == _request->local_positions.end()) { + return; + } + DORIS_CHECK(file_block != nullptr); + const auto block_position = position_it->second; + DORIS_CHECK(block_position.value() < file_block->columns()); + auto column = file_block->get_by_position(block_position.value()).column->assert_mutable(); + const auto output_rows = decode_row_count(rows, selected_rows); + if (auto* nullable_column = check_and_get_column(*column)) { + auto& data = assert_cast(nullable_column->get_nested_column()).get_data(); + auto& null_map = nullable_column->get_null_map_data(); + const auto old_size = data.size(); + data.resize(old_size + output_rows); + null_map.resize_fill(old_size + output_rows, 0); + for (size_t row = 0; row < output_rows; ++row) { + data[old_size + row] = cast_set(_state->current_batch_first_row + + source_row_at(row, selected_rows)); + } + } else { + auto& data = assert_cast(*column).get_data(); + const auto old_size = data.size(); + data.resize(old_size + output_rows); + for (size_t row = 0; row < output_rows; ++row) { + data[old_size + row] = cast_set(_state->current_batch_first_row + + source_row_at(row, selected_rows)); + } + } + file_block->replace_by_position(block_position.value(), std::move(column)); +} + +Status OrcReader::_fill_global_rowid_column(Block* file_block, size_t rows, + const std::vector* selected_rows) const { + const auto position_it = + _request->local_positions.find(format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)); + if (position_it == _request->local_positions.end()) { + return Status::OK(); + } + if (!_global_rowid_context.has_value()) { + return Status::InvalidArgument("ORC global row id requested without row id context"); + } + + DORIS_CHECK(file_block != nullptr); + const auto block_position = position_it->second; + DORIS_CHECK(block_position.value() < file_block->columns()); + auto column = file_block->get_by_position(block_position.value()).column->assert_mutable(); + const auto output_rows = decode_row_count(rows, selected_rows); + + ColumnString* data_column = nullptr; + ColumnUInt8::Container* null_map = nullptr; + if (auto* nullable_column = check_and_get_column(*column)) { + data_column = check_and_get_column(nullable_column->get_nested_column()); + null_map = &nullable_column->get_null_map_data(); + } else { + data_column = check_and_get_column(*column); + } + if (data_column == nullptr) { + return Status::InvalidArgument("ORC global row id column must be STRING"); + } + if (null_map != nullptr) { + null_map->resize_fill(null_map->size() + output_rows, 0); + } + + const auto& context = *_global_rowid_context; + for (size_t row = 0; row < output_rows; ++row) { + const auto row_id = cast_set(_state->current_batch_first_row + + source_row_at(row, selected_rows)); + const GlobalRowLoacationV2 location(context.version, context.backend_id, context.file_id, + row_id); + data_column->insert_data(reinterpret_cast(&location), sizeof(location)); + } + file_block->replace_by_position(block_position.value(), std::move(column)); + return Status::OK(); +} + +bool OrcReader::_can_filter_with_decoded_columns( + const std::set& decoded_columns) const { + auto expr_can_run = [&](const VExprContextSPtr& expr) { + DORIS_CHECK(expr != nullptr); + std::set block_positions; + expr->root()->collect_slot_column_ids(block_positions); + for (const auto block_position : block_positions) { + if (block_position < 0) { + return false; + } + const auto local_position = format::LocalIndex(cast_set(block_position)); + const auto position_it = std::ranges::find_if( + _request->local_positions, + [&](const auto& entry) { return entry.second == local_position; }); + if (position_it == _request->local_positions.end() || + !decoded_columns.contains(position_it->first)) { + return false; + } + } + return true; + }; + + for (const auto& conjunct : _request->conjuncts) { + if (!expr_can_run(conjunct)) { + return false; + } + } + for (const auto& delete_conjunct : _request->delete_conjuncts) { + if (!expr_can_run(delete_conjunct)) { + return false; + } + } + return true; +} + +bool OrcReader::_filter_has_row_level_predicates() const { + return !_request->conjuncts.empty() || !_request->delete_conjuncts.empty(); +} + +Status OrcReader::_build_keep_filter(Block* file_block, size_t rows, + IColumn::Filter* keep_filter) const { + DORIS_CHECK(keep_filter != nullptr); + if (!_filter_has_row_level_predicates()) { + return Status::OK(); + } + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(keep_filter->size() == rows); + if (rows == 0) { + return Status::OK(); + } + + RETURN_IF_ERROR(_execute_conjuncts(file_block, rows, keep_filter)); + RETURN_IF_ERROR(_execute_delete_conjuncts(file_block, rows, keep_filter)); + return Status::OK(); +} + +Status OrcReader::_filter_block(Block* file_block, size_t* rows) const { + if (!_filter_has_row_level_predicates()) { + return Status::OK(); + } + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + if (*rows == 0) { + return Status::OK(); + } + + IColumn::Filter keep_filter(*rows, 1); + RETURN_IF_ERROR(_build_keep_filter(file_block, *rows, &keep_filter)); + _mark_condition_cache_surviving_rows(keep_filter, *rows); + size_t selected_rows = 0; + for (const auto keep : keep_filter) { + selected_rows += keep != 0; + } + _filter_block_with_keep_filter(file_block, keep_filter, selected_rows, rows); + return Status::OK(); +} + +void OrcReader::_filter_block_with_keep_filter(Block* file_block, + const IColumn::Filter& keep_filter, + size_t selected_rows, size_t* rows) const { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + if (selected_rows == *rows) { + return; + } + _filter_requested_columns(file_block, keep_filter, selected_rows); + *rows = selected_rows; +} + +void OrcReader::_filter_decoded_columns( + Block* file_block, const IColumn::Filter& keep_filter, size_t selected_rows, + const std::set& decoded_columns) const { + DORIS_CHECK(file_block != nullptr); + for (const auto file_column_id : decoded_columns) { + const auto position_it = _request->local_positions.find(file_column_id); + DORIS_CHECK(position_it != _request->local_positions.end()); + const auto position = position_it->second.value(); + DORIS_CHECK(position < file_block->columns()); + file_block->replace_by_position( + position, + file_block->get_by_position(position).column->filter(keep_filter, selected_rows)); + } +} + +Status OrcReader::_execute_conjuncts(Block* file_block, size_t rows, + IColumn::Filter* keep_filter) const { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(keep_filter != nullptr); + for (const auto& conjunct : _request->conjuncts) { + IColumn::Filter conjunct_filter(rows, 1); + bool can_filter_all = false; + RETURN_IF_ERROR(conjunct->execute_filter(file_block, conjunct_filter.data(), rows, false, + &can_filter_all)); + if (can_filter_all) { + std::fill(keep_filter->begin(), keep_filter->end(), 0); + return Status::OK(); + } + for (size_t row = 0; row < rows; ++row) { + (*keep_filter)[row] &= conjunct_filter[row]; + } + } + return Status::OK(); +} + +Status OrcReader::_execute_delete_conjuncts(Block* file_block, size_t rows, + IColumn::Filter* keep_filter) const { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(keep_filter != nullptr); + for (const auto& delete_conjunct : _request->delete_conjuncts) { + DORIS_CHECK(delete_conjunct != nullptr); + int result_column_id = -1; + RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, + &result_column_id)); + DORIS_CHECK(result_column_id >= 0 && + result_column_id < static_cast(file_block->columns())); + const auto& delete_filter = assert_cast( + *file_block->get_by_position(result_column_id).column) + .get_data(); + DORIS_CHECK(delete_filter.size() == rows); + for (size_t row = 0; row < rows; ++row) { + (*keep_filter)[row] &= !delete_filter[row]; + } + file_block->erase(result_column_id); + } + return Status::OK(); +} + +void OrcReader::_filter_requested_columns(Block* file_block, const IColumn::Filter& keep_filter, + size_t selected_rows) const { + DORIS_CHECK(file_block != nullptr); + for (const auto& [_, block_position] : _request->local_positions) { + const auto position = block_position.value(); + DORIS_CHECK(position < file_block->columns()); + file_block->replace_by_position( + position, + file_block->get_by_position(position).column->filter(keep_filter, selected_rows)); + } +} + +Status OrcReader::close() { + _collect_profile(); + if (_state != nullptr) { + _state = std::make_unique(); + } + return format::FileReader::close(); +} + +} // namespace doris::format::orc diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h new file mode 100644 index 00000000000000..adba20fbce0b10 --- /dev/null +++ b/be/src/format_v2/orc/orc_reader.h @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column.h" +#include "format_v2/file_reader.h" +#include "runtime/runtime_profile.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris::format::orc { + +struct OrcReaderScanState; + +namespace detail { +bool valid_statistics_bounds(const Field& min_value, const Field& max_value); +} // namespace detail + +// ORC implementation of the format-v2 FileReader contract. The reader consumes +// file-local scan requests; table schema mapping is handled before open(). +class OrcReader final : public format::FileReader { +public: + OrcReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + std::optional global_rowid_context = std::nullopt, + bool enable_mapping_timestamp_tz = false); + ~OrcReader() override; + + static format::ColumnDefinition row_position_column_definition(); + + Status init(RuntimeState* state) override; + Status get_schema(std::vector* const file_schema) const override; + std::unique_ptr create_column_mapper( + format::TableColumnMapperOptions options) const override; + Status open(std::shared_ptr request) override; + Status get_block(Block* file_block, size_t* rows, bool* eof) override; + Status get_aggregate_result(const format::FileAggregateRequest& request, + format::FileAggregateResult* result) override; + void set_condition_cache_context(std::shared_ptr ctx) override; + int64_t get_total_rows() const override; + Status close() override; + + const format::FileReader::ReaderStatistics& reader_statistics() const { + return _reader_statistics; + } + +private: + struct OrcProfile { + RuntimeProfile::Counter* reader_call = nullptr; // ReaderCall + RuntimeProfile::Counter* reader_inclusive_latency_us = nullptr; // ReaderInclusiveLatencyUs + RuntimeProfile::Counter* decompression_call = nullptr; // DecompressionCall + RuntimeProfile::Counter* decompression_latency_us = nullptr; // DecompressionLatencyUs + RuntimeProfile::Counter* decoding_call = nullptr; // DecodingCall + RuntimeProfile::Counter* decoding_latency_us = nullptr; // DecodingLatencyUs + RuntimeProfile::Counter* byte_decoding_call = nullptr; // ByteDecodingCall + RuntimeProfile::Counter* byte_decoding_latency_us = nullptr; // ByteDecodingLatencyUs + RuntimeProfile::Counter* io_count = nullptr; // IOCount + RuntimeProfile::Counter* io_blocking_latency_us = nullptr; // IOBlockingLatencyUs + RuntimeProfile::Counter* selected_row_group_count = nullptr; // SelectedRowGroupCount + RuntimeProfile::Counter* evaluated_row_group_count = nullptr; // EvaluatedRowGroupCount + RuntimeProfile::Counter* read_row_count = nullptr; // ReadRowCount + RuntimeProfile::Counter* filtered_row_groups = nullptr; // RowGroupsFiltered + RuntimeProfile::Counter* filtered_row_groups_by_min_max = nullptr; + RuntimeProfile::Counter* filtered_group_rows = nullptr; // FilteredRowsByGroup + RuntimeProfile::Counter* filtered_bytes = nullptr; + RuntimeProfile::Counter* read_row_groups = nullptr; // RowGroupsReadNum + RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; + RuntimeProfile::Counter* open_file_num = nullptr; + }; + + class OrcFilterImpl; + + void _init_profile() override; + void _collect_profile() const; + + DataTypePtr _convert_to_doris_type(const ::orc::Type& type) const; + DataTypePtr _convert_list_to_doris_type(const ::orc::Type& type) const; + DataTypePtr _convert_map_to_doris_type(const ::orc::Type& type) const; + DataTypePtr _convert_struct_to_doris_type(const ::orc::Type& type) const; + Status _fill_schema_field(const ::orc::Type& type, int32_t local_id, + const std::string& field_name, + format::ColumnDefinition* const field) const; + Status _fill_struct_schema_children(const ::orc::Type& type, + format::ColumnDefinition* const field) const; + Status _fill_list_schema_children(const ::orc::Type& type, + format::ColumnDefinition* const field) const; + Status _fill_map_schema_children(const ::orc::Type& type, + format::ColumnDefinition* const field) const; + + // RowReader setup, SARG pruning, and ORC lazy callback. + Status _configure_row_reader_projection(); + bool _can_apply_orc_lazy_callback() const; + Status _init_search_argument_from_local_filters(); + // Translate the split's byte range (_file_description->range_start_offset/range_size) + // into an ORC [start, end) window. A negative range_size (unset sentinel) yields the + // whole file: {0, UINT64_MAX}. + void _split_byte_window(uint64_t* start, uint64_t* end) const; + // Collect stripes whose offset falls in the current split window, matching ORC + // RowReader's own range inclusion test. + Status _collect_split_stripes(std::vector* stripe_indices) const; + // Seed row_reader_options with the split byte window so each split only reads its own + // stripes. Skips the call for a whole-file window to keep ORC library defaults. + void _apply_split_range(); + Status _select_stripe_ranges_by_statistics(); + void _apply_current_stripe_range(); + Status _advance_to_next_stripe_range(bool* advanced); + Status _create_row_reader(); + + Status _filter_orc_batch(::orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t size, + void* arg); + + Status _decode_column(const ::orc::Type& file_type, const ::orc::Type& selected_type, + const ::orc::ColumnVectorBatch& batch, MutableColumnPtr& column, + size_t rows, const std::vector* selected_rows = nullptr) const; + Status _decode_column_into_block(const ::orc::StructVectorBatch& struct_batch, + format::LocalColumnId file_column_id, size_t rows, + Block* file_block, + const std::vector* selected_rows = nullptr) const; + Status _decode_columns(const ::orc::StructVectorBatch& struct_batch, + const std::vector& projections, size_t rows, + Block* file_block, std::set* decoded_columns, + const std::vector* selected_rows = nullptr) const; + + void _fill_row_position_column(Block* file_block, size_t rows, + const std::vector* selected_rows = nullptr) const; + Status _fill_global_rowid_column(Block* file_block, size_t rows, + const std::vector* selected_rows = nullptr) const; + + // Row-level filtering on decoded Doris columns. + bool _can_filter_with_decoded_columns( + const std::set& decoded_columns) const; + bool _filter_has_row_level_predicates() const; + Status _build_keep_filter(Block* file_block, size_t rows, IColumn::Filter* keep_filter) const; + Status _filter_block(Block* file_block, size_t* rows) const; + void _extend_condition_cache_context_for_current_range(); + void _skip_condition_cache_false_granules(size_t* rows, bool* eof); + void _mark_condition_cache_surviving_rows(const IColumn::Filter& keep_filter, + size_t rows) const; + void _mark_condition_cache_selected_rows(size_t rows, + const std::vector* selected_rows) const; + Status _execute_conjuncts(Block* file_block, size_t rows, IColumn::Filter* keep_filter) const; + Status _execute_delete_conjuncts(Block* file_block, size_t rows, + IColumn::Filter* keep_filter) const; + void _filter_block_with_keep_filter(Block* file_block, const IColumn::Filter& keep_filter, + size_t selected_rows, size_t* rows) const; + void _filter_decoded_columns(Block* file_block, const IColumn::Filter& keep_filter, + size_t selected_rows, + const std::set& decoded_columns) const; + void _filter_requested_columns(Block* file_block, const IColumn::Filter& keep_filter, + size_t selected_rows) const; + + std::unique_ptr _orc_filter; + std::unique_ptr _state; + OrcProfile _orc_profile; // RuntimeProfile counters + std::optional _global_rowid_context; + bool _enable_mapping_timestamp_tz = false; +}; + +} // namespace doris::format::orc diff --git a/be/src/format_v2/orc/orc_search_argument.cpp b/be/src/format_v2/orc/orc_search_argument.cpp new file mode 100644 index 00000000000000..4766fa8304587a --- /dev/null +++ b/be/src/format_v2/orc/orc_search_argument.cpp @@ -0,0 +1,1432 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/orc/orc_search_argument.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "core/column/column.h" +#include "core/data_type/data_type_date_time.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/field.h" +#include "core/types.h" +#include "core/value/vdatetime_value.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "exprs/vtopn_pred.h" +#include "format_v2/timestamp_statistics.h" + +namespace doris::format::orc { +namespace { + +// SARG conversion is intentionally conservative: only direct slots, +// struct_element chains, and whitelisted schema-evolution casts become ORC +// SearchArgument predicates. Everything else stays as row-level filtering. +// Supported ORC predicate domains are LONG, FLOAT, STRING, DATE, DECIMAL, +// TIMESTAMP, and BOOLEAN. +std::optional<::orc::PredicateDataType> predicate_type_for_orc_type(const ::orc::Type& type) { + switch (type.getKind()) { + case ::orc::TypeKind::BYTE: + case ::orc::TypeKind::SHORT: + case ::orc::TypeKind::INT: + case ::orc::TypeKind::LONG: + return ::orc::PredicateDataType::LONG; + case ::orc::TypeKind::FLOAT: + case ::orc::TypeKind::DOUBLE: + return ::orc::PredicateDataType::FLOAT; + case ::orc::TypeKind::STRING: + case ::orc::TypeKind::BINARY: + case ::orc::TypeKind::VARCHAR: + return ::orc::PredicateDataType::STRING; + case ::orc::TypeKind::DATE: + return ::orc::PredicateDataType::DATE; + case ::orc::TypeKind::DECIMAL: + return ::orc::PredicateDataType::DECIMAL; + case ::orc::TypeKind::TIMESTAMP: + case ::orc::TypeKind::TIMESTAMP_INSTANT: + return ::orc::PredicateDataType::TIMESTAMP; + case ::orc::TypeKind::BOOLEAN: + return ::orc::PredicateDataType::BOOLEAN; + default: + return std::nullopt; + } +} + +struct OrcSargColumn { + uint64_t column_id = 0; + ::orc::PredicateDataType predicate_type = ::orc::PredicateDataType::LONG; + const ::orc::Type* orc_type = nullptr; +}; + +struct OrcSargComparison { + OrcSargColumn column; + ::orc::Literal literal; + TExprOpcode::type normalized_op = TExprOpcode::INVALID_OPCODE; +}; + +std::optional file_column_id_for_slot_position( + const format::FileScanRequest& request, int slot_column_id) { + if (slot_column_id < 0) { + return std::nullopt; + } + // Localized VSlotRef::column_id() points to the file block position, not the file schema id. + const auto slot_position = cast_set(slot_column_id); + for (const auto& [file_column_id, local_position] : request.local_positions) { + if (local_position.value() == slot_position) { + return file_column_id; + } + } + return std::nullopt; +} + +const ::orc::Type* orc_type_for_slot(const format::FileScanRequest& request, + const ::orc::Type& root_type, const VExprSPtr& slot_expr) { + if (slot_expr == nullptr || !slot_expr->is_slot_ref()) { + return nullptr; + } + const auto* slot_ref = dynamic_cast(slot_expr.get()); + if (slot_ref == nullptr || slot_ref->column_id() < 0) { + return nullptr; + } + const auto file_column_id = file_column_id_for_slot_position(request, slot_ref->column_id()); + if (!file_column_id.has_value() || !file_column_id->is_valid()) { + return nullptr; + } + const auto file_column_position = cast_set(file_column_id->value()); + if (file_column_position >= root_type.getSubtypeCount()) { + return nullptr; + } + return root_type.getSubtype(file_column_position); +} + +std::optional sarg_column_for_orc_type(const ::orc::Type* orc_type) { + if (orc_type == nullptr) { + return std::nullopt; + } + const auto predicate_type = predicate_type_for_orc_type(*orc_type); + if (!predicate_type.has_value()) { + return std::nullopt; + } + return OrcSargColumn { + .column_id = orc_type->getColumnId(), + .predicate_type = *predicate_type, + .orc_type = orc_type, + }; +} + +const ::orc::Type* orc_type_for_sarg_source_expr(const format::FileScanRequest& request, + const ::orc::Type& root_type, + const VExprSPtr& expr); + +std::optional literal_field_for_sarg(const VExprSPtr& literal_expr) { + if (literal_expr == nullptr || !literal_expr->is_literal()) { + return std::nullopt; + } + const auto* literal = dynamic_cast(literal_expr.get()); + if (literal == nullptr || literal->get_column_ptr().get() == nullptr || + literal->get_column_ptr()->is_null_at(0)) { + return std::nullopt; + } + Field field; + literal->get_column_ptr()->get(0, field); + return field; +} + +bool is_struct_element_expr(const VExprSPtr& expr) { + return expr != nullptr && expr->children().size() == 2 && + expr->fn().name.function_name == "struct_element"; +} + +// struct_element can address a child by field name or 1-based ordinal. +std::optional struct_child_index(const ::orc::Type& struct_type, + const VExprSPtr& selector_expr) { + if (struct_type.getKind() != ::orc::TypeKind::STRUCT) { + return std::nullopt; + } + const auto field = literal_field_for_sarg(selector_expr); + if (!field.has_value()) { + return std::nullopt; + } + switch (field->get_type()) { + case TYPE_STRING: + case TYPE_CHAR: + case TYPE_VARCHAR: { + const auto child_name = std::string(field->as_string_view()); + for (uint64_t child_idx = 0; child_idx < struct_type.getSubtypeCount(); ++child_idx) { + if (struct_type.getFieldName(child_idx) == child_name) { + return child_idx; + } + } + return std::nullopt; + } + case TYPE_TINYINT: + if (field->get() <= 0) { + return std::nullopt; + } + return cast_set(field->get() - 1); + case TYPE_SMALLINT: + if (field->get() <= 0) { + return std::nullopt; + } + return cast_set(field->get() - 1); + case TYPE_INT: + if (field->get() <= 0) { + return std::nullopt; + } + return cast_set(field->get() - 1); + case TYPE_BIGINT: + if (field->get() <= 0) { + return std::nullopt; + } + return cast_set(field->get() - 1); + default: + return std::nullopt; + } +} + +const ::orc::Type* orc_type_for_struct_element(const format::FileScanRequest& request, + const ::orc::Type& root_type, + const VExprSPtr& expr) { + if (!is_struct_element_expr(expr)) { + return nullptr; + } + const auto* parent_type = + orc_type_for_sarg_source_expr(request, root_type, expr->children()[0]); + if (parent_type == nullptr || parent_type->getKind() != ::orc::TypeKind::STRUCT) { + return nullptr; + } + const auto child_idx = struct_child_index(*parent_type, expr->children()[1]); + if (!child_idx.has_value() || *child_idx >= parent_type->getSubtypeCount()) { + return nullptr; + } + return parent_type->getSubtype(*child_idx); +} + +const ::orc::Type* orc_type_for_sarg_source_expr(const format::FileScanRequest& request, + const ::orc::Type& root_type, + const VExprSPtr& expr) { + if (is_struct_element_expr(expr)) { + return orc_type_for_struct_element(request, root_type, expr); + } + return orc_type_for_slot(request, root_type, expr); +} + +std::optional sarg_column_for_source_expr(const format::FileScanRequest& request, + const ::orc::Type& root_type, + const VExprSPtr& expr) { + return sarg_column_for_orc_type(orc_type_for_sarg_source_expr(request, root_type, expr)); +} + +// Safe cast checks protect pruning correctness. A cast is SARGable only when the +// comparison in ORC's domain is equivalent to the original Doris expression. +std::optional orc_integer_width(const ::orc::Type& type) { + switch (type.getKind()) { + case ::orc::TypeKind::BYTE: + return 8; + case ::orc::TypeKind::SHORT: + return 16; + case ::orc::TypeKind::INT: + return 32; + case ::orc::TypeKind::LONG: + return 64; + default: + return std::nullopt; + } +} + +std::optional signed_integer_width(PrimitiveType type) { + switch (type) { + case TYPE_TINYINT: + return 8; + case TYPE_SMALLINT: + return 16; + case TYPE_INT: + return 32; + case TYPE_BIGINT: + return 64; + default: + return std::nullopt; + } +} + +std::optional orc_floating_width(const ::orc::Type& type) { + switch (type.getKind()) { + case ::orc::TypeKind::FLOAT: + return 32; + case ::orc::TypeKind::DOUBLE: + return 64; + default: + return std::nullopt; + } +} + +std::optional floating_width(PrimitiveType type) { + switch (type) { + case TYPE_FLOAT: + return 32; + case TYPE_DOUBLE: + return 64; + default: + return std::nullopt; + } +} + +std::optional floating_exact_integer_width(PrimitiveType type) { + switch (type) { + case TYPE_FLOAT: + return 24; + case TYPE_DOUBLE: + return 53; + default: + return std::nullopt; + } +} + +struct DecimalPrecisionScale { + UInt32 precision = 0; + UInt32 scale = 0; +}; + +std::optional decimal_precision_scale(const DataTypePtr& type) { + if (type == nullptr || !is_decimal(type->get_primitive_type())) { + return std::nullopt; + } + const auto precision = type->get_precision(); + const auto scale = type->get_scale(); + if (precision == 0 || scale > precision) { + return std::nullopt; + } + return DecimalPrecisionScale {.precision = precision, .scale = scale}; +} + +bool is_safe_widening_cast(const OrcSargColumn& column, const VExprSPtr& cast_expr) { + if (column.orc_type == nullptr || cast_expr == nullptr || cast_expr->data_type() == nullptr) { + return false; + } + const auto target_type = remove_nullable(cast_expr->data_type()); + if (target_type == nullptr) { + return false; + } + const auto target_primitive_type = target_type->get_primitive_type(); + if (const auto source_width = orc_integer_width(*column.orc_type)) { + const auto target_width = signed_integer_width(target_primitive_type); + return target_width.has_value() && *target_width >= *source_width; + } + if (const auto source_width = orc_floating_width(*column.orc_type)) { + const auto target_width = floating_width(target_primitive_type); + return target_width.has_value() && *target_width >= *source_width; + } + return false; +} + +bool is_safe_date_schema_evolution_cast(const OrcSargColumn& column, const VExprSPtr& cast_expr) { + if (column.orc_type == nullptr || cast_expr == nullptr || cast_expr->data_type() == nullptr || + column.orc_type->getKind() != ::orc::TypeKind::DATE) { + return false; + } + const auto target_type = remove_nullable(cast_expr->data_type()); + if (target_type == nullptr) { + return false; + } + const auto target_primitive_type = target_type->get_primitive_type(); + return target_primitive_type == TYPE_DATE || target_primitive_type == TYPE_DATEV2 || + target_primitive_type == TYPE_DATETIME || target_primitive_type == TYPE_DATETIMEV2; +} + +bool is_safe_string_schema_evolution_cast(const OrcSargColumn& column, const VExprSPtr& cast_expr) { + if (column.orc_type == nullptr || cast_expr == nullptr || cast_expr->data_type() == nullptr) { + return false; + } + const auto orc_kind = column.orc_type->getKind(); + switch (column.orc_type->getKind()) { + case ::orc::TypeKind::STRING: + case ::orc::TypeKind::BINARY: + case ::orc::TypeKind::VARCHAR: + break; + default: + return false; + } + if (cast_expr->children().size() != 1 || cast_expr->children()[0] == nullptr || + cast_expr->children()[0]->data_type() == nullptr) { + return false; + } + const auto source_type = remove_nullable(cast_expr->children()[0]->data_type()); + if (source_type == nullptr) { + return false; + } + const auto source_primitive_type = source_type->get_primitive_type(); + const bool source_is_string_like = + source_primitive_type == TYPE_STRING || source_primitive_type == TYPE_VARCHAR || + (orc_kind == ::orc::TypeKind::BINARY && source_primitive_type == TYPE_VARBINARY); + if (!source_is_string_like) { + return false; + } + const auto target_type = remove_nullable(cast_expr->data_type()); + if (target_type == nullptr) { + return false; + } + return target_type->get_primitive_type() == TYPE_STRING; +} + +bool is_safe_decimal_schema_evolution_cast(const OrcSargColumn& column, + const VExprSPtr& cast_expr) { + if (column.orc_type == nullptr || column.orc_type->getKind() != ::orc::TypeKind::DECIMAL || + cast_expr == nullptr || cast_expr->data_type() == nullptr || + cast_expr->children().size() != 1 || cast_expr->children()[0] == nullptr || + cast_expr->children()[0]->data_type() == nullptr) { + return false; + } + + const auto source_type = remove_nullable(cast_expr->children()[0]->data_type()); + const auto target_type = remove_nullable(cast_expr->data_type()); + const auto source_decimal = decimal_precision_scale(source_type); + const auto target_decimal = decimal_precision_scale(target_type); + if (!source_decimal.has_value() || !target_decimal.has_value()) { + return false; + } + if (source_decimal->precision != cast_set(column.orc_type->getPrecision()) || + source_decimal->scale != cast_set(column.orc_type->getScale())) { + return false; + } + + const auto source_integer_digits = source_decimal->precision - source_decimal->scale; + const auto target_integer_digits = target_decimal->precision - target_decimal->scale; + return target_decimal->scale >= source_decimal->scale && + target_integer_digits >= source_integer_digits; +} + +bool is_safe_integer_to_floating_cast(const OrcSargColumn& column, const VExprSPtr& cast_expr) { + if (column.orc_type == nullptr || cast_expr == nullptr || cast_expr->data_type() == nullptr) { + return false; + } + const auto source_width = orc_integer_width(*column.orc_type); + if (!source_width.has_value()) { + return false; + } + const auto target_type = remove_nullable(cast_expr->data_type()); + if (target_type == nullptr) { + return false; + } + const auto target_width = floating_exact_integer_width(target_type->get_primitive_type()); + return target_width.has_value() && *source_width <= *target_width; +} + +bool is_safe_cast_for_sarg(const OrcSargColumn& column, const VExprSPtr& cast_expr) { + return is_safe_widening_cast(column, cast_expr) || + is_safe_date_schema_evolution_cast(column, cast_expr) || + is_safe_string_schema_evolution_cast(column, cast_expr) || + is_safe_decimal_schema_evolution_cast(column, cast_expr) || + is_safe_integer_to_floating_cast(column, cast_expr); +} + +std::optional sarg_column_for_slot_or_safe_cast( + const format::FileScanRequest& request, const ::orc::Type& root_type, + const VExprSPtr& expr) { + auto column = sarg_column_for_source_expr(request, root_type, expr); + if (column.has_value()) { + return column; + } + if (expr == nullptr || expr->node_type() != TExprNodeType::CAST_EXPR || + expr->children().size() != 1) { + return std::nullopt; + } + column = sarg_column_for_source_expr(request, root_type, expr->children()[0]); + if (!column.has_value() || !is_safe_cast_for_sarg(*column, expr)) { + return std::nullopt; + } + return column; +} + +// Literal conversion preserves ORC PredicateDataType rules. Unsupported +// literal/type pairs simply make the surrounding predicate non-SARGable. +std::optional<::orc::Literal> make_long_literal(const Field& field) { + switch (field.get_type()) { + case TYPE_TINYINT: + return ::orc::Literal(static_cast(field.get())); + case TYPE_SMALLINT: + return ::orc::Literal(static_cast(field.get())); + case TYPE_INT: + return ::orc::Literal(static_cast(field.get())); + case TYPE_BIGINT: + return ::orc::Literal(static_cast(field.get())); + default: + return std::nullopt; + } +} + +std::optional<::orc::Literal> make_float_literal(const Field& field) { + switch (field.get_type()) { + case TYPE_FLOAT: + return ::orc::Literal(static_cast(field.get())); + case TYPE_DOUBLE: + return ::orc::Literal(field.get()); + default: + return std::nullopt; + } +} + +std::optional<::orc::Literal> make_string_literal(const Field& field) { + if (!is_string_type(field.get_type()) && !is_varbinary(field.get_type())) { + return std::nullopt; + } + const auto value = field.as_string_view(); + return ::orc::Literal(value.data(), value.size()); +} + +std::optional<::orc::Literal> make_bool_literal(const Field& field) { + if (field.get_type() != TYPE_BOOLEAN) { + return std::nullopt; + } + return ::orc::Literal(field.get() != 0); +} + +std::optional<::orc::Literal> make_date_literal(const Field& field) { + static const cctz::time_zone utc0 = cctz::utc_time_zone(); + switch (field.get_type()) { + case TYPE_DATE: { + const auto& date = field.get(); + const cctz::civil_day civil_date(date.year(), date.month(), date.day()); + const auto day_offset = + cctz::convert(civil_date, utc0).time_since_epoch().count() / (24 * 60 * 60); + return ::orc::Literal(::orc::PredicateDataType::DATE, day_offset); + } + case TYPE_DATEV2: { + const auto& date = field.get(); + const cctz::civil_day civil_date(date.year(), date.month(), date.day()); + const auto day_offset = + cctz::convert(civil_date, utc0).time_since_epoch().count() / (24 * 60 * 60); + return ::orc::Literal(::orc::PredicateDataType::DATE, day_offset); + } + case TYPE_DATETIME: { + const auto& datetime = field.get(); + if (datetime.hour() != 0 || datetime.minute() != 0 || datetime.second() != 0) { + return std::nullopt; + } + const cctz::civil_day civil_date(datetime.year(), datetime.month(), datetime.day()); + const auto day_offset = + cctz::convert(civil_date, utc0).time_since_epoch().count() / (24 * 60 * 60); + return ::orc::Literal(::orc::PredicateDataType::DATE, day_offset); + } + case TYPE_DATETIMEV2: { + const auto& datetime = field.get(); + if (datetime.hour() != 0 || datetime.minute() != 0 || datetime.second() != 0 || + datetime.microsecond() != 0) { + return std::nullopt; + } + const cctz::civil_day civil_date(datetime.year(), datetime.month(), datetime.day()); + const auto day_offset = + cctz::convert(civil_date, utc0).time_since_epoch().count() / (24 * 60 * 60); + return ::orc::Literal(::orc::PredicateDataType::DATE, day_offset); + } + default: + return std::nullopt; + } +} + +std::optional<::orc::Literal> make_date_literal(int year, int month, int day) { + static const cctz::time_zone utc0 = cctz::utc_time_zone(); + const cctz::civil_day civil_date(year, month, day); + const auto day_offset = + cctz::convert(civil_date, utc0).time_since_epoch().count() / (24 * 60 * 60); + return ::orc::Literal(::orc::PredicateDataType::DATE, day_offset); +} + +struct DateTimeLiteralParts { + int year = 0; + int month = 0; + int day = 0; + bool has_time = false; +}; + +std::optional date_time_literal_parts(const Field& field) { + switch (field.get_type()) { + case TYPE_DATETIME: { + const auto& datetime = field.get(); + return DateTimeLiteralParts { + .year = datetime.year(), + .month = datetime.month(), + .day = datetime.day(), + .has_time = + datetime.hour() != 0 || datetime.minute() != 0 || datetime.second() != 0, + }; + } + case TYPE_DATETIMEV2: { + const auto& datetime = field.get(); + return DateTimeLiteralParts { + .year = datetime.year(), + .month = datetime.month(), + .day = datetime.day(), + .has_time = datetime.hour() != 0 || datetime.minute() != 0 || + datetime.second() != 0 || datetime.microsecond() != 0, + }; + } + default: + return std::nullopt; + } +} + +std::optional<::orc::Literal> make_timestamp_literal(const Field& field, + const cctz::time_zone& timezone) { + const auto civil_year_is_monotonic = [&](const cctz::civil_second& civil_seconds) { + // Localized file predicates may already have selected one side of a repeated civil time. + // Checking the whole civil year catches that lossy representation while retaining SARG + // pruning for years and zones without a backward transition. + const auto year_start = + cctz::convert(cctz::civil_second(civil_seconds.year(), 1, 1), timezone); + const auto next_year_start = + cctz::convert(cctz::civil_second(civil_seconds.year() + 1, 1, 1), timezone); + return format::utc_timestamp_range_is_monotonic(year_start.time_since_epoch().count(), + next_year_start.time_since_epoch().count(), + timezone); + }; + switch (field.get_type()) { + case TYPE_DATETIME: { + const auto& datetime = field.get(); + const cctz::civil_second civil_seconds(datetime.year(), datetime.month(), datetime.day(), + datetime.hour(), datetime.minute(), + datetime.second()); + const auto lookup = timezone.lookup(civil_seconds); + // ORC SearchArguments accept one UTC timestamp literal. A repeated or skipped local civil + // time has no unique UTC representation, so pushing it down could prune a stripe that + // contains another valid interpretation. Keep such predicates for row-level evaluation. + if (!civil_year_is_monotonic(civil_seconds) || + lookup.kind != cctz::time_zone::civil_lookup::UNIQUE) { + return std::nullopt; + } + return ::orc::Literal(lookup.pre.time_since_epoch().count(), 0); + } + case TYPE_DATETIMEV2: { + const auto& datetime = field.get(); + const cctz::civil_second civil_seconds(datetime.year(), datetime.month(), datetime.day(), + datetime.hour(), datetime.minute(), + datetime.second()); + const auto lookup = timezone.lookup(civil_seconds); + // See the DATETIME path above. The fractional part does not disambiguate a civil time + // repeated by a backward timezone transition. + if (!civil_year_is_monotonic(civil_seconds) || + lookup.kind != cctz::time_zone::civil_lookup::UNIQUE) { + return std::nullopt; + } + const auto seconds = lookup.pre.time_since_epoch().count(); + const auto nanos = cast_set(datetime.microsecond() * 1000); + return ::orc::Literal(seconds, nanos); + } + default: + return std::nullopt; + } +} + +std::optional<::orc::Literal> make_decimal_literal(const ::orc::Type& orc_type, + const VLiteral& literal, const Field& field) { + if (orc_type.getKind() != ::orc::TypeKind::DECIMAL) { + return std::nullopt; + } + const auto& literal_type = literal.get_data_type(); + if (literal_type == nullptr) { + return std::nullopt; + } + + Int128 decimal_value = 0; + switch (field.get_type()) { + case TYPE_DECIMALV2: + decimal_value = binary_cast(field.get()); + break; + case TYPE_DECIMAL32: + decimal_value = static_cast(field.get().value); + break; + case TYPE_DECIMAL64: + decimal_value = static_cast(field.get().value); + break; + case TYPE_DECIMAL128I: + decimal_value = field.get().value; + break; + default: + return std::nullopt; + } + + const auto precision = literal_type->get_precision() == 0 + ? cast_set(orc_type.getPrecision()) + : literal_type->get_precision(); + const auto scale = literal_type->get_scale(); + return ::orc::Literal(::orc::Int128(static_cast(decimal_value >> 64), + static_cast(decimal_value)), + cast_set(precision), cast_set(scale)); +} + +std::optional<::orc::Literal> make_orc_literal(const OrcSargColumn& sarg_column, const Field& field, + const cctz::time_zone& timezone) { + switch (sarg_column.predicate_type) { + case ::orc::PredicateDataType::LONG: + return make_long_literal(field); + case ::orc::PredicateDataType::FLOAT: + return make_float_literal(field); + case ::orc::PredicateDataType::STRING: + return make_string_literal(field); + case ::orc::PredicateDataType::BOOLEAN: + return make_bool_literal(field); + case ::orc::PredicateDataType::DATE: + return make_date_literal(field); + case ::orc::PredicateDataType::TIMESTAMP: { + DORIS_CHECK(sarg_column.orc_type != nullptr); + static const cctz::time_zone utc0 = cctz::utc_time_zone(); + const auto& literal_timezone = + sarg_column.orc_type->getKind() == ::orc::TypeKind::TIMESTAMP_INSTANT ? timezone + : utc0; + return make_timestamp_literal(field, literal_timezone); + } + case ::orc::PredicateDataType::DECIMAL: + return std::nullopt; + } + return std::nullopt; +} + +std::optional<::orc::Literal> make_orc_literal(const OrcSargColumn& sarg_column, + const VExprSPtr& literal_expr, + const cctz::time_zone& timezone) { + if (literal_expr == nullptr || !literal_expr->is_literal()) { + return std::nullopt; + } + const auto* literal = dynamic_cast(literal_expr.get()); + if (literal == nullptr || literal->get_column_ptr().get() == nullptr || + literal->get_column_ptr()->is_null_at(0)) { + return std::nullopt; + } + + Field field; + literal->get_column_ptr()->get(0, field); + if (sarg_column.predicate_type == ::orc::PredicateDataType::DECIMAL) { + if (sarg_column.orc_type == nullptr) { + return std::nullopt; + } + return make_decimal_literal(*sarg_column.orc_type, *literal, field); + } + return make_orc_literal(sarg_column, field, timezone); +} + +bool is_null_literal(const VExprSPtr& literal_expr) { + if (literal_expr == nullptr || !literal_expr->is_literal()) { + return false; + } + const auto* literal = dynamic_cast(literal_expr.get()); + return literal != nullptr && literal->get_column_ptr().get() != nullptr && + literal->get_column_ptr()->is_null_at(0); +} + +// Buildability is checked before emission so the builder path can assume the +// expression shape was validated and use DORIS_CHECK for impossible branches. +bool can_build_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, + const cctz::time_zone& timezone, const VExprSPtr& expr); + +std::optional expression_for_search_argument(const VExprSPtr& expr) { + if (expr == nullptr) { + return std::nullopt; + } + + // Match legacy ORC reader behavior: lower wrapper predicates to the concrete predicate before + // checking SARG support. This keeps runtime-filter and top-N pruning from regressing in format v2. + if (expr->is_rf_wrapper()) { + if (expr->node_type() == TExprNodeType::NULL_AWARE_IN_PRED || + expr->node_type() == TExprNodeType::NULL_AWARE_BINARY_PRED) { + return std::nullopt; + } + const auto impl = expr->get_impl(); + if (impl == nullptr) { + return expr; + } + if (const auto* direct_in = dynamic_cast(impl.get()); + direct_in != nullptr) { + VExprSPtr in_expr; + if (!direct_in->get_slot_in_expr(in_expr)) { + return std::nullopt; + } + return in_expr; + } + return impl; + } + + if (expr->is_topn_filter()) { + const auto* topn_pred = dynamic_cast(expr.get()); + if (topn_pred == nullptr) { + return std::nullopt; + } + VExprSPtr binary_expr; + if (!topn_pred->get_binary_expr(binary_expr)) { + return std::nullopt; + } + return binary_expr; + } + + return expr; +} + +std::optional reverse_comparison_op(TExprOpcode::type op) { + switch (op) { + case TExprOpcode::GE: + return TExprOpcode::LE; + case TExprOpcode::GT: + return TExprOpcode::LT; + case TExprOpcode::LE: + return TExprOpcode::GE; + case TExprOpcode::LT: + return TExprOpcode::GT; + case TExprOpcode::EQ: + case TExprOpcode::NE: + return op; + default: + return std::nullopt; + } +} + +bool is_date_to_datetime_cast_for_sarg(const OrcSargColumn& column, const VExprSPtr& expr) { + if (column.orc_type == nullptr || column.orc_type->getKind() != ::orc::TypeKind::DATE || + expr == nullptr || expr->node_type() != TExprNodeType::CAST_EXPR || + expr->data_type() == nullptr) { + return false; + } + const auto target_type = remove_nullable(expr->data_type()); + if (target_type == nullptr) { + return false; + } + const auto target_type_id = target_type->get_primitive_type(); + return target_type_id == TYPE_DATETIME || target_type_id == TYPE_DATETIMEV2; +} + +bool is_integer_to_floating_cast_for_sarg(const OrcSargColumn& column, const VExprSPtr& expr) { + return expr != nullptr && expr->node_type() == TExprNodeType::CAST_EXPR && + is_safe_integer_to_floating_cast(column, expr); +} + +std::optional normalize_date_to_datetime_comparison_op(TExprOpcode::type op, + bool literal_has_time) { + if (!literal_has_time) { + return op; + } + switch (op) { + case TExprOpcode::GT: + case TExprOpcode::GE: + return TExprOpcode::GT; + case TExprOpcode::LT: + case TExprOpcode::LE: + return TExprOpcode::LE; + default: + return std::nullopt; + } +} + +std::optional floating_literal_value_for_sarg(const VExprSPtr& literal_expr) { + const auto field = literal_field_for_sarg(literal_expr); + if (!field.has_value()) { + return std::nullopt; + } + switch (field->get_type()) { + case TYPE_FLOAT: + return static_cast(field->get()); + case TYPE_DOUBLE: + return field->get(); + default: + return std::nullopt; + } +} + +std::optional double_to_int64_boundary(double value, double (*round_func)(double)) { + if (!std::isfinite(value)) { + return std::nullopt; + } + const auto rounded = round_func(value); + const auto int64_min = static_cast(std::numeric_limits::min()); + // INT64_MAX rounds to 2^63 as double, so keep the upper bound exclusive. + const auto int64_max_exclusive = std::ldexp(1.0, 63); + if (rounded < int64_min || rounded >= int64_max_exclusive) { + return std::nullopt; + } + return static_cast(rounded); +} + +std::optional double_to_integral_int64(double value) { + if (!std::isfinite(value) || std::trunc(value) != value) { + return std::nullopt; + } + return double_to_int64_boundary(value, std::trunc); +} + +struct OrcSargComparisonLiteral { + ::orc::Literal literal; + TExprOpcode::type normalized_op = TExprOpcode::INVALID_OPCODE; +}; + +std::optional make_integer_to_floating_comparison_literal( + TExprOpcode::type normalized_op, double literal_value) { + std::optional integer_literal; + TExprOpcode::type integer_op = normalized_op; + switch (normalized_op) { + case TExprOpcode::GT: + integer_literal = double_to_int64_boundary(literal_value, std::floor); + break; + case TExprOpcode::GE: + integer_literal = double_to_int64_boundary(literal_value, std::ceil); + break; + case TExprOpcode::LT: + integer_literal = double_to_int64_boundary(literal_value, std::ceil); + break; + case TExprOpcode::LE: + integer_literal = double_to_int64_boundary(literal_value, std::floor); + break; + case TExprOpcode::EQ: + case TExprOpcode::NE: + integer_literal = double_to_integral_int64(literal_value); + break; + default: + return std::nullopt; + } + if (!integer_literal.has_value()) { + return std::nullopt; + } + return OrcSargComparisonLiteral { + .literal = ::orc::Literal(*integer_literal), + .normalized_op = integer_op, + }; +} + +std::optional make_comparison_literal_for_sarg( + const OrcSargColumn& column, const VExprSPtr& source_expr, const VExprSPtr& literal_expr, + TExprOpcode::type normalized_op, const cctz::time_zone& timezone) { + if (is_date_to_datetime_cast_for_sarg(column, source_expr)) { + const auto field = literal_field_for_sarg(literal_expr); + if (!field.has_value()) { + return std::nullopt; + } + const auto parts = date_time_literal_parts(*field); + if (!parts.has_value()) { + return std::nullopt; + } + const auto adjusted_op = + normalize_date_to_datetime_comparison_op(normalized_op, parts->has_time); + if (!adjusted_op.has_value()) { + return std::nullopt; + } + const auto literal = make_date_literal(parts->year, parts->month, parts->day); + if (!literal.has_value()) { + return std::nullopt; + } + return OrcSargComparisonLiteral { + .literal = *literal, + .normalized_op = *adjusted_op, + }; + } + if (is_integer_to_floating_cast_for_sarg(column, source_expr)) { + const auto literal_value = floating_literal_value_for_sarg(literal_expr); + if (!literal_value.has_value()) { + return std::nullopt; + } + return make_integer_to_floating_comparison_literal(normalized_op, *literal_value); + } + + const auto literal = make_orc_literal(column, literal_expr, timezone); + if (!literal.has_value()) { + return std::nullopt; + } + return OrcSargComparisonLiteral { + .literal = *literal, + .normalized_op = normalized_op, + }; +} + +std::optional> make_in_literals_for_sarg( + const OrcSargColumn& column, const VExprSPtr& source_expr, + const std::vector& children, const cctz::time_zone& timezone) { + if (children.size() < 2) { + return std::nullopt; + } + + std::vector<::orc::Literal> literals; + literals.reserve(children.size() - 1); + if (is_date_to_datetime_cast_for_sarg(column, source_expr)) { + for (auto child_it = children.begin() + 1; child_it != children.end(); ++child_it) { + if (is_null_literal(*child_it)) { + continue; + } + const auto field = literal_field_for_sarg(*child_it); + if (!field.has_value()) { + return std::nullopt; + } + const auto parts = date_time_literal_parts(*field); + if (!parts.has_value()) { + return std::nullopt; + } + if (parts->has_time) { + continue; + } + const auto literal = make_date_literal(parts->year, parts->month, parts->day); + if (!literal.has_value()) { + return std::nullopt; + } + literals.push_back(*literal); + } + if (literals.empty()) { + return std::nullopt; + } + return literals; + } + if (is_integer_to_floating_cast_for_sarg(column, source_expr)) { + for (auto child_it = children.begin() + 1; child_it != children.end(); ++child_it) { + if (is_null_literal(*child_it)) { + continue; + } + const auto literal_value = floating_literal_value_for_sarg(*child_it); + if (!literal_value.has_value()) { + return std::nullopt; + } + const auto integer_literal = double_to_integral_int64(*literal_value); + if (!integer_literal.has_value()) { + continue; + } + literals.emplace_back(*integer_literal); + } + if (literals.empty()) { + return std::nullopt; + } + return literals; + } + + for (auto child_it = children.begin() + 1; child_it != children.end(); ++child_it) { + if (is_null_literal(*child_it)) { + continue; + } + auto literal = make_orc_literal(column, *child_it, timezone); + if (!literal.has_value()) { + return std::nullopt; + } + literals.push_back(*literal); + } + if (literals.empty()) { + return std::nullopt; + } + return literals; +} + +std::optional sarg_comparison_for_expr(const format::FileScanRequest& request, + const ::orc::Type& root_type, + const cctz::time_zone& timezone, + const VExprSPtr& expr) { + if (expr == nullptr || expr->children().size() != 2) { + return std::nullopt; + } + + const VExprSPtr* slot_expr = nullptr; + const VExprSPtr* literal_expr = nullptr; + auto normalized_op = expr->op(); + if (sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]).has_value() && + expr->children()[1]->is_literal()) { + slot_expr = &expr->children()[0]; + literal_expr = &expr->children()[1]; + } else if (expr->children()[0]->is_literal() && + sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[1]) + .has_value()) { + const auto reversed_op = reverse_comparison_op(expr->op()); + if (!reversed_op.has_value()) { + return std::nullopt; + } + slot_expr = &expr->children()[1]; + literal_expr = &expr->children()[0]; + normalized_op = *reversed_op; + } else { + return std::nullopt; + } + + auto sarg_column = sarg_column_for_slot_or_safe_cast(request, root_type, *slot_expr); + if (!sarg_column.has_value()) { + return std::nullopt; + } + const auto comparison_literal = make_comparison_literal_for_sarg( + *sarg_column, *slot_expr, *literal_expr, normalized_op, timezone); + if (!comparison_literal.has_value()) { + return std::nullopt; + } + return OrcSargComparison { + .column = *sarg_column, + .literal = comparison_literal->literal, + .normalized_op = comparison_literal->normalized_op, + }; +} + +bool can_build_slot_literal_predicate(const format::FileScanRequest& request, + const ::orc::Type& root_type, const cctz::time_zone& timezone, + const VExprSPtr& expr) { + if (expr == nullptr || expr->children().size() != 2) { + return false; + } + return sarg_comparison_for_expr(request, root_type, timezone, expr).has_value(); +} + +bool can_build_in_predicate(const format::FileScanRequest& request, const ::orc::Type& root_type, + const cctz::time_zone& timezone, const VExprSPtr& expr) { + if (expr == nullptr || expr->children().size() < 2) { + return false; + } + const auto sarg_column = + sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]); + if (!sarg_column.has_value()) { + return false; + } + return make_in_literals_for_sarg(*sarg_column, expr->children()[0], expr->children(), timezone) + .has_value(); +} + +bool can_build_is_null_predicate(const format::FileScanRequest& request, + const ::orc::Type& root_type, const VExprSPtr& expr) { + return expr != nullptr && expr->children().size() == 1 && + sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]).has_value(); +} + +std::optional sarg_column_for_null_safe_equal_null( + const format::FileScanRequest& request, const ::orc::Type& root_type, + const VExprSPtr& expr) { + if (expr == nullptr || expr->node_type() != TExprNodeType::NULL_AWARE_BINARY_PRED || + expr->op() != TExprOpcode::EQ_FOR_NULL || expr->children().size() != 2) { + return std::nullopt; + } + + auto column_if_other_child_is_null_literal = + [&](size_t slot_idx, size_t literal_idx) -> std::optional { + if (!is_null_literal(expr->children()[literal_idx])) { + return std::nullopt; + } + return sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[slot_idx]); + }; + auto column = column_if_other_child_is_null_literal(0, 1); + if (column.has_value()) { + return column; + } + return column_if_other_child_is_null_literal(1, 0); +} + +std::optional sarg_comparison_for_null_safe_equal_literal( + const format::FileScanRequest& request, const ::orc::Type& root_type, + const cctz::time_zone& timezone, const VExprSPtr& expr) { + if (expr == nullptr || expr->node_type() != TExprNodeType::NULL_AWARE_BINARY_PRED || + expr->op() != TExprOpcode::EQ_FOR_NULL || expr->children().size() != 2) { + return std::nullopt; + } + + auto comparison_if_other_child_is_non_null_literal = + [&](size_t slot_idx, size_t literal_idx) -> std::optional { + const auto& literal_expr = expr->children()[literal_idx]; + if (literal_expr == nullptr || !literal_expr->is_literal() || + is_null_literal(literal_expr)) { + return std::nullopt; + } + const auto& source_expr = expr->children()[slot_idx]; + auto column = sarg_column_for_slot_or_safe_cast(request, root_type, source_expr); + if (!column.has_value()) { + return std::nullopt; + } + const auto comparison_literal = make_comparison_literal_for_sarg( + *column, source_expr, literal_expr, TExprOpcode::EQ, timezone); + if (!comparison_literal.has_value() || + comparison_literal->normalized_op != TExprOpcode::EQ) { + return std::nullopt; + } + return OrcSargComparison { + .column = *column, + .literal = comparison_literal->literal, + .normalized_op = TExprOpcode::EQ, + }; + }; + auto comparison = comparison_if_other_child_is_non_null_literal(0, 1); + if (comparison.has_value()) { + return comparison; + } + return comparison_if_other_child_is_non_null_literal(1, 0); +} + +bool can_build_null_safe_equal_predicate(const format::FileScanRequest& request, + const ::orc::Type& root_type, + const cctz::time_zone& timezone, const VExprSPtr& expr) { + return sarg_column_for_null_safe_equal_null(request, root_type, expr).has_value() || + sarg_comparison_for_null_safe_equal_literal(request, root_type, timezone, expr) + .has_value(); +} + +bool contains_null_safe_equal(const VExprSPtr& expr) { + const auto sarg_expr = expression_for_search_argument(expr); + if (!sarg_expr.has_value() || *sarg_expr == nullptr) { + return false; + } + if (sarg_expr->get() != expr.get()) { + return contains_null_safe_equal(*sarg_expr); + } + if ((*sarg_expr)->op() == TExprOpcode::EQ_FOR_NULL) { + return true; + } + return std::ranges::any_of((*sarg_expr)->children(), contains_null_safe_equal); +} + +bool can_build_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, + const cctz::time_zone& timezone, const VExprSPtr& expr) { + const auto sarg_expr = expression_for_search_argument(expr); + if (!sarg_expr.has_value()) { + return false; + } + if (*sarg_expr == nullptr) { + return false; + } + if (sarg_expr->get() != expr.get()) { + return can_build_search_argument(request, root_type, timezone, *sarg_expr); + } + + switch ((*sarg_expr)->op()) { + case TExprOpcode::COMPOUND_AND: + return std::ranges::any_of((*sarg_expr)->children(), [&](const auto& child) { + return can_build_search_argument(request, root_type, timezone, child); + }); + case TExprOpcode::COMPOUND_OR: + if (contains_null_safe_equal(*sarg_expr)) { + return false; + } + return !(*sarg_expr)->children().empty() && + std::ranges::all_of((*sarg_expr)->children(), [&](const auto& child) { + return can_build_search_argument(request, root_type, timezone, child); + }); + case TExprOpcode::COMPOUND_NOT: + if (contains_null_safe_equal(*sarg_expr)) { + return false; + } + return (*sarg_expr)->children().size() == 1 && + can_build_search_argument(request, root_type, timezone, (*sarg_expr)->children()[0]); + case TExprOpcode::GE: + case TExprOpcode::GT: + case TExprOpcode::LE: + case TExprOpcode::LT: + case TExprOpcode::EQ: + case TExprOpcode::NE: + return (*sarg_expr)->node_type() != TExprNodeType::NULL_AWARE_BINARY_PRED && + can_build_slot_literal_predicate(request, root_type, timezone, *sarg_expr); + case TExprOpcode::EQ_FOR_NULL: + return can_build_null_safe_equal_predicate(request, root_type, timezone, *sarg_expr); + case TExprOpcode::FILTER_IN: + case TExprOpcode::FILTER_NOT_IN: + return (*sarg_expr)->node_type() != TExprNodeType::NULL_AWARE_IN_PRED && + can_build_in_predicate(request, root_type, timezone, *sarg_expr); + case TExprOpcode::INVALID_OPCODE: + return (*sarg_expr)->node_type() == TExprNodeType::FUNCTION_CALL && + ((*sarg_expr)->fn().name.function_name == "is_null_pred" || + (*sarg_expr)->fn().name.function_name == "is_not_null_pred") && + can_build_is_null_predicate(request, root_type, *sarg_expr); + default: + return false; + } +} + +void build_less_than(const OrcSargColumn& column, const ::orc::Literal& literal, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + builder->lessThan(column.column_id, column.predicate_type, literal); +} + +void build_less_than(const OrcSargComparison& comparison, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + build_less_than(comparison.column, comparison.literal, builder); +} + +void build_less_than_equals(const OrcSargColumn& column, const ::orc::Literal& literal, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + builder->lessThanEquals(column.column_id, column.predicate_type, literal); +} + +void build_less_than_equals(const OrcSargComparison& comparison, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + build_less_than_equals(comparison.column, comparison.literal, builder); +} + +void build_equals(const OrcSargColumn& column, const ::orc::Literal& literal, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + builder->equals(column.column_id, column.predicate_type, literal); +} + +void build_equals(const OrcSargComparison& comparison, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + build_equals(comparison.column, comparison.literal, builder); +} + +void build_comparison_predicate(const format::FileScanRequest& request, + const ::orc::Type& root_type, const VExprSPtr& expr, + const cctz::time_zone& timezone, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + const auto comparison = *sarg_comparison_for_expr(request, root_type, timezone, expr); + switch (comparison.normalized_op) { + case TExprOpcode::GE: + builder->startNot(); + build_less_than(comparison, builder); + builder->end(); + return; + case TExprOpcode::GT: + builder->startNot(); + build_less_than_equals(comparison, builder); + builder->end(); + return; + case TExprOpcode::LE: + build_less_than_equals(comparison, builder); + return; + case TExprOpcode::LT: + build_less_than(comparison, builder); + return; + case TExprOpcode::EQ: + build_equals(comparison, builder); + return; + case TExprOpcode::NE: + builder->startNot(); + build_equals(comparison, builder); + builder->end(); + return; + default: + DORIS_CHECK(false) << "Unsupported normalized ORC SARG comparison op " + << comparison.normalized_op; + } +} + +void build_in_predicate(const format::FileScanRequest& request, const ::orc::Type& root_type, + const VExprSPtr& expr, const cctz::time_zone& timezone, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + const auto sarg_column = + *sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]); + auto literals = *make_in_literals_for_sarg(sarg_column, expr->children()[0], expr->children(), + timezone); + DORIS_CHECK(!literals.empty()); + if (literals.size() == 1) { + builder->equals(sarg_column.column_id, sarg_column.predicate_type, literals.front()); + return; + } + builder->in(sarg_column.column_id, sarg_column.predicate_type, literals); +} + +void build_is_null(const format::FileScanRequest& request, const ::orc::Type& root_type, + const VExprSPtr& expr, std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + const auto sarg_column = + *sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]); + builder->isNull(sarg_column.column_id, sarg_column.predicate_type); +} + +void build_null_safe_equal(const format::FileScanRequest& request, const ::orc::Type& root_type, + const VExprSPtr& expr, const cctz::time_zone& timezone, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + const auto sarg_column = sarg_column_for_null_safe_equal_null(request, root_type, expr); + if (sarg_column.has_value()) { + builder->isNull(sarg_column->column_id, sarg_column->predicate_type); + return; + } + const auto comparison = + *sarg_comparison_for_null_safe_equal_literal(request, root_type, timezone, expr); + build_equals(comparison, builder); +} + +bool build_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, + const cctz::time_zone& timezone, const VExprSPtr& expr, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + const auto sarg_expr = expression_for_search_argument(expr); + if (!sarg_expr.has_value() || *sarg_expr == nullptr) { + return false; + } + if (sarg_expr->get() != expr.get()) { + return build_search_argument(request, root_type, timezone, *sarg_expr, builder); + } + if (!can_build_search_argument(request, root_type, timezone, *sarg_expr)) { + return false; + } + + switch ((*sarg_expr)->op()) { + case TExprOpcode::COMPOUND_AND: + builder->startAnd(); + for (const auto& child : (*sarg_expr)->children()) { + static_cast(build_search_argument(request, root_type, timezone, child, builder)); + } + builder->end(); + return true; + case TExprOpcode::COMPOUND_OR: + builder->startOr(); + for (const auto& child : (*sarg_expr)->children()) { + const auto built = build_search_argument(request, root_type, timezone, child, builder); + DORIS_CHECK(built); + } + builder->end(); + return true; + case TExprOpcode::COMPOUND_NOT: + builder->startNot(); + DORIS_CHECK(build_search_argument(request, root_type, timezone, (*sarg_expr)->children()[0], + builder)); + builder->end(); + return true; + case TExprOpcode::GE: + case TExprOpcode::GT: + case TExprOpcode::LE: + case TExprOpcode::LT: + case TExprOpcode::EQ: + case TExprOpcode::NE: + build_comparison_predicate(request, root_type, *sarg_expr, timezone, builder); + return true; + case TExprOpcode::EQ_FOR_NULL: + build_null_safe_equal(request, root_type, *sarg_expr, timezone, builder); + return true; + case TExprOpcode::FILTER_IN: + build_in_predicate(request, root_type, *sarg_expr, timezone, builder); + return true; + case TExprOpcode::FILTER_NOT_IN: + builder->startNot(); + build_in_predicate(request, root_type, *sarg_expr, timezone, builder); + builder->end(); + return true; + case TExprOpcode::INVALID_OPCODE: + if ((*sarg_expr)->fn().name.function_name == "is_null_pred") { + build_is_null(request, root_type, *sarg_expr, builder); + return true; + } + if ((*sarg_expr)->fn().name.function_name == "is_not_null_pred") { + builder->startNot(); + build_is_null(request, root_type, *sarg_expr, builder); + builder->end(); + return true; + } + return false; + default: + return false; + } +} + +} // namespace + +bool build_orc_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, + const cctz::time_zone& timezone, const VExprSPtr& expr, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { + return build_search_argument(request, root_type, timezone, expr, builder); +} + +} // namespace doris::format::orc diff --git a/be/src/format_v2/orc/orc_search_argument.h b/be/src/format_v2/orc/orc_search_argument.h new file mode 100644 index 00000000000000..ad922915e6ed50 --- /dev/null +++ b/be/src/format_v2/orc/orc_search_argument.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "exprs/vexpr_fwd.h" +#include "format_v2/file_reader.h" + +namespace orc { +class SearchArgumentBuilder; +class Type; +} // namespace orc + +namespace doris::format::orc { + +// Lower already-localized Doris file filters to ORC SearchArgument predicates. +// TableColumnMapper owns table-schema -> file-local localization; this module +// owns the ORC-specific type-id/literal lowering needed by the ORC C++ library. +bool build_orc_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, + const cctz::time_zone& timezone, const VExprSPtr& expr, + std::unique_ptr<::orc::SearchArgumentBuilder>& builder); + +} // namespace doris::format::orc diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp new file mode 100644 index 00000000000000..b42d47987a54cb --- /dev/null +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -0,0 +1,492 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_column_schema.h" + +#include + +#include +#include +#include +#include + +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/parquet_type.h" + +namespace doris::format::parquet { +namespace { + +struct SchemaBuildContext { + int32_t local_id = -1; // child ordinal in the parent node + int16_t definition_level = 0; // accumulated optional/repeated level count + int16_t repetition_level = 0; // accumulated repeated level count + int16_t nullable_definition_level = 0; // definition level of the nearest optional node + int16_t repeated_repetition_level = 0; // repetition level of the nearest repeated node + int16_t repeated_ancestor_definition_level = 0; // definition level of the nearest repeated node +}; + +enum class SchemaBuildMode { + // Normal recursive schema build. Bare repeated fields are exposed as Doris ARRAY for + // protobuf/legacy Parquet compatibility, while repeated LIST/MAP annotated groups are rejected + // because Parquet LIST/MAP outer groups are not allowed to be repeated at a top-level or struct + // field boundary. + NORMAL, + // Build the current repeated node as the already-selected element of an enclosing LIST. This + // is the compatibility path for Arrow/parquet-format legacy two-level LIST encodings where the + // repeated node itself is the array element instead of a wrapper that should be stripped. + REPEATED_NODE_AS_LIST_ELEMENT, + // Build the current repeated group as a STRUCT element of an enclosing LIST, ignoring LIST/MAP + // annotations on the repeated group itself. This keeps compatibility with the old Doris + // Parquet schema parser for Hive/legacy wrappers named "array" or "_tuple". + REPEATED_NODE_AS_STRUCT_ELEMENT, +}; + +// Result of applying Parquet LIST backward compatibility rules to the single repeated child of a +// LIST-annotated group. The repeated child can either be a physical wrapper whose only child is the +// element, or the element node itself. +struct ListElementResolution { + // Parquet node that should be exposed as Doris ARRAY element. + const ::parquet::schema::Node* element_node = nullptr; + // Level state after consuming the LIST repeated child. The parent ARRAY schema keeps this state + // to materialize offsets, empty arrays and null arrays. + SchemaBuildContext repeated_context; + // Level state used to build element_node. This equals repeated_context when the repeated child + // itself is the element, and includes the wrapper's only child when standard 3-level LIST + // encoding is stripped. + SchemaBuildContext element_context; + // Build mode for element_node. Non-NORMAL modes mean element_node is the repeated child itself, + // and the repeated level must not be interpreted as a second unrelated array at the same + // boundary. + SchemaBuildMode element_build_mode = SchemaBuildMode::NORMAL; +}; + +// Resolved repeated entry group of a MAP-annotated group. The entry wrapper is a physical Parquet +// encoding detail; Doris folds it into the parent MAP schema and exposes only direct [key, value] +// children. +struct MapEntryResolution { + const ::parquet::schema::GroupNode* entry_group = nullptr; + // Level state after consuming the repeated entry group. The parent MAP schema keeps this state + // to materialize offsets, empty maps and null maps. + SchemaBuildContext entry_context; +}; + +bool is_list_node(const ::parquet::schema::Node& node) { + const auto& logical_type = node.logical_type(); + return node.converted_type() == ::parquet::ConvertedType::LIST || + (logical_type != nullptr && logical_type->is_valid() && logical_type->is_list()); +} + +bool is_map_node(const ::parquet::schema::Node& node) { + const auto& logical_type = node.logical_type(); + return node.converted_type() == ::parquet::ConvertedType::MAP || + node.converted_type() == ::parquet::ConvertedType::MAP_KEY_VALUE || + (logical_type != nullptr && logical_type->is_valid() && logical_type->is_map()); +} + +bool has_logical_annotation(const ::parquet::schema::Node& node) { + const auto& logical_type = node.logical_type(); + return (node.converted_type() != ::parquet::ConvertedType::NONE && + node.converted_type() != ::parquet::ConvertedType::UNDEFINED) || + (logical_type != nullptr && logical_type->is_valid() && !logical_type->is_none()); +} + +bool has_structural_list_name(const std::string& list_name, const std::string& repeated_name) { + return repeated_name == "array" || repeated_name == list_name + "_tuple"; +} + +bool should_build_repeated_field_as_list(const ::parquet::schema::Node& node) { + return node.is_repeated() && !is_list_node(node) && !is_map_node(node); +} + +DataTypePtr nullable_if_needed(DataTypePtr type, const ::parquet::schema::Node& node) { + return node.is_optional() ? make_nullable(type) : type; +} + +void inherit_common_schema_state(const ::parquet::schema::Node& node, + const SchemaBuildContext& context, + ParquetColumnSchema* column_schema) { + DORIS_CHECK(column_schema != nullptr); + column_schema->local_id = context.local_id; + column_schema->parquet_field_id = node.field_id(); + column_schema->name = node.name(); + column_schema->max_definition_level = context.definition_level; + column_schema->max_repetition_level = context.repetition_level; + column_schema->nullable_definition_level = context.nullable_definition_level; + column_schema->definition_level = context.definition_level; + column_schema->repetition_level = context.repetition_level; + column_schema->repeated_ancestor_definition_level = context.repeated_ancestor_definition_level; + column_schema->repeated_repetition_level = context.repeated_repetition_level; +} + +SchemaBuildContext child_context(const SchemaBuildContext& parent, + const ::parquet::schema::Node& child_node, int32_t child_idx) { + SchemaBuildContext result = parent; + result.local_id = child_idx; + if (child_node.repetition() == ::parquet::Repetition::OPTIONAL) { + result.definition_level++; + result.nullable_definition_level = result.definition_level; + } + if (child_node.is_repeated()) { + result.repetition_level++; + result.definition_level++; + result.repeated_repetition_level = result.repetition_level; + result.repeated_ancestor_definition_level = result.definition_level; + } + return result; +} + +void propagate_child_levels(ParquetColumnSchema* column_schema) { + DORIS_CHECK(column_schema != nullptr); + for (const auto& child : column_schema->children) { + column_schema->max_definition_level = + std::max(column_schema->max_definition_level, child->max_definition_level); + column_schema->max_repetition_level = + std::max(column_schema->max_repetition_level, child->max_repetition_level); + } +} + +// Mirrors Arrow's ResolveList() compatibility rules, but only decides which Parquet node is the +// logical LIST element. The caller still builds Doris' semantic LIST->[element] schema tree. +// Important cases: +// - repeated primitive: the primitive itself is the element (legacy two-level LIST). +// - repeated group with multiple children: the group itself is a STRUCT element. +// - repeated group named "array" or "_tuple": the group itself is a STRUCT element per +// Parquet backward compatibility rules, even when it has one child or its own logical annotation. +// This also keeps v2 file-local schema aligned with Doris' old schema parser used by HDFS TVF. +// - other repeated group with a logical annotation, or whose only child is repeated: the group +// itself is the element. This preserves nested LIST/MAP and repeated fields inside struct +// elements. +// - otherwise, strip the one-child repeated wrapper as standard three-level LIST encoding. +Status resolve_list_element_node(const ::parquet::schema::GroupNode& list_group, + const SchemaBuildContext& list_context, + ListElementResolution* result) { + if (result == nullptr) { + return Status::InvalidArgument("result is null"); + } + if (list_group.field_count() != 1) { + return Status::NotSupported("Unsupported parquet LIST encoding for column {}", + list_group.name()); + } + const auto& repeated_node = *list_group.field(0); + if (!repeated_node.is_repeated()) { + return Status::NotSupported("Unsupported parquet LIST encoding for column {}", + list_group.name()); + } + result->repeated_context = child_context(list_context, repeated_node, 0); + if (repeated_node.is_primitive()) { + result->element_node = &repeated_node; + result->element_context = result->repeated_context; + result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; + return Status::OK(); + } + + const auto& repeated_group = static_cast(repeated_node); + if (repeated_group.field_count() == 0) { + return Status::NotSupported("Unsupported parquet LIST element layout for column {}", + list_group.name()); + } + const bool repeated_group_has_logical_annotation = has_logical_annotation(repeated_group); + if (repeated_group.field_count() > 1 || + has_structural_list_name(list_group.name(), repeated_group.name())) { + result->element_node = &repeated_node; + result->element_context = result->repeated_context; + result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT; + return Status::OK(); + } + if (repeated_group_has_logical_annotation) { + result->element_node = &repeated_node; + result->element_context = result->repeated_context; + result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; + return Status::OK(); + } + + const auto& only_child = *repeated_group.field(0); + if (only_child.is_repeated()) { + result->element_node = &repeated_node; + result->element_context = result->repeated_context; + result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; + return Status::OK(); + } + + result->element_node = &only_child; + result->element_context = child_context(result->repeated_context, only_child, 0); + return Status::OK(); +} + +// Resolves the repeated entry group of a MAP/MAP_KEY_VALUE node. Unlike LIST, MAP has no supported +// two-level form in this reader: Doris requires a repeated group with exactly key and value +// children, then folds that physical entry group out of ParquetColumnSchema. Some external writers +// emit optional MAP keys even though standard Parquet MAP keys are required; keep the key's +// definition levels and expose it as nullable for compatibility with the old reader. +Status resolve_map_entry_group(const ::parquet::schema::GroupNode& map_group, + const SchemaBuildContext& map_context, MapEntryResolution* result) { + if (result == nullptr) { + return Status::InvalidArgument("result is null"); + } + if (map_group.field_count() != 1) { + return Status::NotSupported("Unsupported parquet MAP encoding for column {}", + map_group.name()); + } + const auto& entry_node = *map_group.field(0); + if (!entry_node.is_repeated()) { + return Status::NotSupported("Unsupported parquet MAP encoding for column {}", + map_group.name()); + } + if (entry_node.is_primitive()) { + return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", + map_group.name()); + } + const auto& entry_group = static_cast(entry_node); + if (entry_group.field_count() != 2) { + return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", + map_group.name()); + } + // The Parquet logical MAP spec requires key to be REQUIRED. Some legacy/Hive-written files + // still mark the key field OPTIONAL even when all actual keys are non-null, for example: + // optional group t_map_varchar (MAP) { + // repeated group key_value { + // optional binary key (STRING); + // optional binary value (STRING); + // } + // } + // Accept that schema here so compatible files can be read. MapColumnReader validates the + // materialized key column and rejects data that really contains null map keys. + result->entry_group = &entry_group; + result->entry_context = child_context(map_context, entry_node, 0); + return Status::OK(); +} + +Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, + const ::parquet::schema::Node& node, + const SchemaBuildContext& context, + std::unique_ptr* result, + SchemaBuildMode mode); + +// Builds a semantic ARRAY schema for a bare repeated field. Arrow handles this in +// NodeToSchemaField()/GroupToSchemaField(); Doris needs the same compatibility behavior because +// protobuf and old parquet writers often encode repeated fields without a LIST annotation. +// Example: +// optional group event { +// repeated group links { +// optional binary url (UTF8); +// optional int32 rank; +// } +// } +// Doris exposes event.links as ARRAY>, not STRUCT. This keeps v2's +// file-local schema aligned with the old schema parser used by HDFS TVF schema fetching. +// When the repeated field appears inside an already resolved LIST element, only the nested repeated +// child should be wrapped: +// optional group a (LIST) { +// repeated group element { +// repeated int32 items; +// } +// } +// The outer LIST element is the repeated "element" group, and its repeated "items" child should be +// represented as a field of type ARRAY inside the struct element. +Status build_repeated_field_as_list_schema(const ::parquet::SchemaDescriptor& schema, + const ::parquet::schema::Node& repeated_node, + const SchemaBuildContext& repeated_context, + std::unique_ptr* result) { + if (result == nullptr) { + return Status::InvalidArgument("result is null"); + } + auto list_schema = std::make_unique(); + inherit_common_schema_state(repeated_node, repeated_context, list_schema.get()); + list_schema->kind = ParquetColumnSchemaKind::LIST; + list_schema->definition_level = repeated_context.definition_level; + list_schema->repetition_level = repeated_context.repetition_level; + list_schema->repeated_repetition_level = repeated_context.repeated_repetition_level; + + std::unique_ptr element_child; + RETURN_IF_ERROR(build_node_schema_with_mode(schema, repeated_node, repeated_context, + &element_child, + SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT)); + element_child->name = "element"; + list_schema->type = std::make_shared(element_child->type); + list_schema->children.push_back(std::move(element_child)); + propagate_child_levels(list_schema.get()); + *result = std::move(list_schema); + return Status::OK(); +} + +// Recursively builds ParquetColumnSchema for the given schema node and its children in Parquet +// file's metadata. NORMAL mode exposes bare repeated fields as ARRAY for legacy compatibility. +// REPEATED_NODE_AS_LIST_ELEMENT mode means the current repeated node was already selected as an +// enclosing LIST element, so only its nested bare repeated children should be wrapped. +Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, + const ::parquet::schema::Node& node, + const SchemaBuildContext& context, + std::unique_ptr* result, + SchemaBuildMode mode) { + if (result == nullptr) { + return Status::InvalidArgument("result is null"); + } + if (mode == SchemaBuildMode::NORMAL && should_build_repeated_field_as_list(node)) { + return build_repeated_field_as_list_schema(schema, node, context, result); + } + + auto column_schema = std::make_unique(); + inherit_common_schema_state(node, context, column_schema.get()); + + if (node.is_primitive()) { + const int leaf_column_id = schema.ColumnIndex(node); + if (leaf_column_id < 0) { + return Status::InvalidArgument("Cannot find leaf column id for parquet column {}", + node.name()); + } + column_schema->kind = ParquetColumnSchemaKind::PRIMITIVE; + column_schema->leaf_column_id = leaf_column_id; + column_schema->descriptor = schema.Column(leaf_column_id); + if (column_schema->descriptor != nullptr) { + column_schema->max_definition_level = column_schema->descriptor->max_definition_level(); + column_schema->max_repetition_level = column_schema->descriptor->max_repetition_level(); + } + column_schema->type_descriptor = resolve_parquet_type(column_schema->descriptor); + column_schema->type = column_schema->type_descriptor.doris_type; + if (column_schema->type == nullptr) { + if (!column_schema->type_descriptor.unsupported_reason.empty()) { + return Status::NotSupported("Unsupported parquet column '{}': {}", node.name(), + column_schema->type_descriptor.unsupported_reason); + } + return Status::NotSupported("Unsupported parquet column type for column {}", + node.name()); + } + column_schema->type = node.is_optional() + ? make_nullable(remove_nullable(column_schema->type)) + : remove_nullable(column_schema->type); + *result = std::move(column_schema); + return Status::OK(); + } + + const auto& group = static_cast(node); + if (is_list_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { + if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { + return Status::NotSupported("Unsupported repeated parquet LIST column {}", node.name()); + } + column_schema->kind = ParquetColumnSchemaKind::LIST; + ListElementResolution list_element; + RETURN_IF_ERROR(resolve_list_element_node(group, context, &list_element)); + column_schema->definition_level = list_element.repeated_context.definition_level; + column_schema->repetition_level = list_element.repeated_context.repetition_level; + column_schema->repeated_repetition_level = + list_element.repeated_context.repeated_repetition_level; + std::unique_ptr child; + RETURN_IF_ERROR(build_node_schema_with_mode(schema, *list_element.element_node, + list_element.element_context, &child, + list_element.element_build_mode)); + child->name = "element"; + column_schema->type = + nullable_if_needed(std::make_shared(child->type), node); + column_schema->children.push_back(std::move(child)); + propagate_child_levels(column_schema.get()); + *result = std::move(column_schema); + return Status::OK(); + } + + if (is_map_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { + if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { + return Status::NotSupported("Unsupported repeated parquet MAP column {}", node.name()); + } + column_schema->kind = ParquetColumnSchemaKind::MAP; + MapEntryResolution map_entry; + RETURN_IF_ERROR(resolve_map_entry_group(group, context, &map_entry)); + column_schema->definition_level = map_entry.entry_context.definition_level; + column_schema->repetition_level = map_entry.entry_context.repetition_level; + column_schema->repeated_repetition_level = + map_entry.entry_context.repeated_repetition_level; + for (int child_idx = 0; child_idx < map_entry.entry_group->field_count(); ++child_idx) { + std::unique_ptr child; + RETURN_IF_ERROR(build_node_schema_with_mode( + schema, *map_entry.entry_group->field(child_idx), + child_context(map_entry.entry_context, *map_entry.entry_group->field(child_idx), + child_idx), + &child, SchemaBuildMode::NORMAL)); + child->name = child_idx == 0 ? "key" : "value"; + column_schema->children.push_back(std::move(child)); + } + if (column_schema->children.size() != 2) { + return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", + node.name()); + } + auto key_type = make_nullable(column_schema->children[0]->type); + auto value_type = make_nullable(column_schema->children[1]->type); + column_schema->type = + nullable_if_needed(std::make_shared(key_type, value_type), node); + propagate_child_levels(column_schema.get()); + *result = std::move(column_schema); + return Status::OK(); + } + + column_schema->kind = ParquetColumnSchemaKind::STRUCT; + DataTypes child_types; + Strings child_names; + child_types.reserve(group.field_count()); + child_names.reserve(group.field_count()); + for (int child_idx = 0; child_idx < group.field_count(); ++child_idx) { + const auto& child_node = *group.field(child_idx); + std::unique_ptr child; + const auto child_ctx = child_context(context, child_node, child_idx); + if (should_build_repeated_field_as_list(child_node)) { + RETURN_IF_ERROR( + build_repeated_field_as_list_schema(schema, child_node, child_ctx, &child)); + } else { + RETURN_IF_ERROR(build_node_schema_with_mode(schema, child_node, child_ctx, &child, + SchemaBuildMode::NORMAL)); + } + child_types.push_back(make_nullable(child->type)); + child_names.push_back(child->name); + column_schema->children.push_back(std::move(child)); + } + column_schema->type = + nullable_if_needed(std::make_shared(child_types, child_names), node); + propagate_child_levels(column_schema.get()); + *result = std::move(column_schema); + return Status::OK(); +} + +Status build_node_schema(const ::parquet::SchemaDescriptor& schema, + const ::parquet::schema::Node& node, const SchemaBuildContext& context, + std::unique_ptr* result) { + return build_node_schema_with_mode(schema, node, context, result, SchemaBuildMode::NORMAL); +} + +} // namespace + +Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, + std::vector>* fields) { + if (fields == nullptr) { + return Status::InvalidArgument("fields is null"); + } + fields->clear(); + const auto* root = schema.group_node(); + if (root == nullptr) { + return Status::InvalidArgument("Parquet schema root is null"); + } + fields->reserve(root->field_count()); + for (int field_idx = 0; field_idx < root->field_count(); ++field_idx) { + std::unique_ptr field; + SchemaBuildContext context; + RETURN_IF_ERROR(build_node_schema( + schema, *root->field(field_idx), + child_context(context, *root->field(field_idx), field_idx), &field)); + fields->push_back(std::move(field)); + } + return Status::OK(); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_column_schema.h b/be/src/format_v2/parquet/parquet_column_schema.h new file mode 100644 index 00000000000000..1fb7262aabde6f --- /dev/null +++ b/be/src/format_v2/parquet/parquet_column_schema.h @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/parquet_type.h" + +namespace parquet { +class ColumnDescriptor; +class SchemaDescriptor; +} // namespace parquet + +namespace doris::format::parquet { + +enum class ParquetColumnSchemaKind { + PRIMITIVE, // primitive leaf -> ScalarColumnReader + STRUCT, // struct -> StructColumnReader + LIST, // array -> ListColumnReader + MAP, // map -> MapColumnReader +}; + +// ============================================================================ +// ============================================================================ +// ============================================================================ +struct ParquetColumnSchema { + int local_id = -1; + + int parquet_field_id = -1; + + std::string name; + + DataTypePtr type = nullptr; + + int leaf_column_id = -1; + + ParquetTypeDescriptor type_descriptor {}; + + ParquetColumnSchemaKind kind = ParquetColumnSchemaKind::PRIMITIVE; + + const ::parquet::ColumnDescriptor* descriptor = nullptr; + + // ======== Dremel Levels ======== + + int16_t max_definition_level = 0; + int16_t max_repetition_level = 0; + + int16_t nullable_definition_level = 0; + + int16_t definition_level = 0; + int16_t repetition_level = 0; + + int16_t repeated_ancestor_definition_level = 0; + + int16_t repeated_repetition_level = 0; + + std::vector> children {}; +}; + +Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, + std::vector>* fields); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp new file mode 100644 index 00000000000000..d80dc58181d4f6 --- /dev/null +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -0,0 +1,616 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_file_context.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/config.h" +#include "io/cache/cached_remote_file_reader.h" +#include "io/file_factory.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader.h" +#include "io/fs/tracing_file_reader.h" +#include "io/io_common.h" +#include "storage/cache/page_cache.h" +#include "util/slice.h" + +namespace doris::format::parquet { + +namespace detail { + +std::vector plan_page_cache_range_read( + int64_t position, int64_t nbytes, const std::vector& cached_ranges) { + if (position < 0 || nbytes <= 0) { + return {}; + } + + std::vector ranges; + ranges.reserve(cached_ranges.size()); + const int64_t request_end = position + nbytes; + for (const auto& range : cached_ranges) { + if (range.size > 0 && range.offset < request_end && position < range.end_offset()) { + ranges.push_back(range); + } + } + std::sort(ranges.begin(), ranges.end(), [](const auto& lhs, const auto& rhs) { + if (lhs.offset != rhs.offset) { + return lhs.offset < rhs.offset; + } + return lhs.size > rhs.size; + }); + + std::vector plan; + int64_t cursor = position; + while (cursor < request_end) { + // At each cursor position, choose the cached range that already covers the cursor and + // extends farthest to the right. This handles both adjacent ranges and overlapping + // ranges. If no range covers the current cursor, there is a gap and the request must + // miss as a whole. + auto best = ranges.end(); + int64_t best_end = cursor; + for (auto it = ranges.begin(); it != ranges.end(); ++it) { + const int64_t cached_end = it->end_offset(); + if (it->offset <= cursor && cursor < cached_end && cached_end > best_end) { + best = it; + best_end = cached_end; + } + } + if (best == ranges.end()) { + return {}; + } + const int64_t copy_size = std::min(best_end, request_end) - cursor; + ParquetPageCacheReadPlanEntry entry; + entry.cached_range = *best; + entry.copy_offset_in_cache = cursor - best->offset; + entry.output_offset = cursor - position; + entry.copy_size = copy_size; + plan.push_back(entry); + cursor += copy_size; + } + return plan; +} + +std::vector valid_prefetch_ranges( + const std::vector& ranges) { + std::vector valid_ranges; + valid_ranges.reserve(ranges.size()); + for (const auto& range : ranges) { + if (range.offset < 0 || range.size <= 0 || + range.offset > std::numeric_limits::max() - range.size) { + continue; + } + valid_ranges.push_back(range); + } + return valid_ranges; +} + +size_t average_prefetch_range_size(const std::vector& ranges) { + const auto valid_ranges = valid_prefetch_ranges(ranges); + if (valid_ranges.empty()) { + return 0; + } + size_t total_size = 0; + for (const auto& range : valid_ranges) { + total_size += static_cast(range.size); + } + return total_size / valid_ranges.size(); +} + +bool should_use_merge_range_reader(const std::vector& ranges, + size_t avg_io_size, bool is_in_memory_reader) { + return !is_in_memory_reader && !valid_prefetch_ranges(ranges).empty() && + avg_io_size < io::MergeRangeFileReader::SMALL_IO; +} + +} // namespace detail + +namespace { + +// StoragePageCache only supports exact-key lookup. Keep lightweight range metadata here so later +// Arrow ReadAt requests can reuse cached bytes when their requested ranges are subsets of, or are +// fully covered by, previously cached ranges. Stale metadata is pruned on lookup. +std::mutex cached_page_range_index_mutex; +std::unordered_map> cached_page_range_index; +constexpr size_t MAX_CACHED_PAGE_RANGE_FILES = 4096; +constexpr size_t MAX_CACHED_PAGE_RANGES_PER_FILE = 65536; + +void register_cached_page_range(const std::string& file_key, int64_t position, int64_t nbytes) { + DORIS_CHECK(nbytes > 0); + std::lock_guard lock(cached_page_range_index_mutex); + if (cached_page_range_index.find(file_key) == cached_page_range_index.end() && + cached_page_range_index.size() >= MAX_CACHED_PAGE_RANGE_FILES) { + cached_page_range_index.erase(cached_page_range_index.begin()); + } + auto& ranges = cached_page_range_index[file_key]; + auto it = std::find_if(ranges.begin(), ranges.end(), [&](const ParquetPageCacheRange& range) { + return range.offset == position && range.size == nbytes; + }); + if (it == ranges.end()) { + if (ranges.size() >= MAX_CACHED_PAGE_RANGES_PER_FILE) { + ranges.erase(ranges.begin()); + } + ranges.push_back(ParquetPageCacheRange {position, nbytes}); + } +} + +void unregister_cached_page_range(const std::string& file_key, + const ParquetPageCacheRange& stale_range) { + std::lock_guard lock(cached_page_range_index_mutex); + auto it = cached_page_range_index.find(file_key); + if (it == cached_page_range_index.end()) { + return; + } + auto& ranges = it->second; + ranges.erase(std::remove_if(ranges.begin(), ranges.end(), + [&](const ParquetPageCacheRange& range) { + return range.offset == stale_range.offset && + range.size == stale_range.size; + }), + ranges.end()); + if (ranges.empty()) { + cached_page_range_index.erase(it); + } +} + +std::vector cached_page_ranges_for_file(const std::string& file_key) { + std::lock_guard lock(cached_page_range_index_mutex); + auto it = cached_page_range_index.find(file_key); + if (it == cached_page_range_index.end()) { + return {}; + } + return it->second; +} + +std::string build_page_cache_file_key(const io::FileReader& file_reader, + const io::FileDescription& file_description) { + const int64_t mtime = + file_description.mtime != 0 ? file_description.mtime : file_reader.mtime(); + if (mtime == 0) { + // StoragePageCache is process-global. A key with only path + unknown mtime can outlive a + // rewritten local test file, or any external file whose version was not propagated. Disable + // v2 parquet page cache until the scan descriptor carries a stable object version. + return {}; + } + const int64_t file_size = file_description.file_size >= 0 + ? file_description.file_size + : static_cast(file_reader.size()); + return fmt::format("{}::{}::mtime={}::size={}", file_description.fs_name, + file_reader.path().native(), mtime, file_size); +} + +class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { +public: + DorisRandomAccessFile(io::FileReaderSPtr file_reader, io::IOContext* io_ctx, + bool enable_page_cache, std::string page_cache_file_key) + : _file_reader(std::move(file_reader)), + _base_file_reader(_file_reader), + _io_ctx(io_ctx), + _enable_page_cache(enable_page_cache), + _page_cache_file_key(std::move(page_cache_file_key)) { + DORIS_CHECK(_file_reader != nullptr); + if (auto tracing_reader = std::dynamic_pointer_cast(_file_reader)) { + _file_reader_stats = tracing_reader->stats(); + _base_file_reader = tracing_reader->inner_reader(); + } + DORIS_CHECK(_base_file_reader != nullptr); + set_mode(arrow::io::FileMode::READ); + } + + arrow::Status Close() override { + if (!_closed) { + collect_active_merge_range_profile(); + _closed = true; + } + return arrow::Status::OK(); + } + + bool closed() const override { return _closed; } + + arrow::Result Tell() const override { return _pos; } + + arrow::Status Seek(int64_t position) override { + if (position < 0) { + return arrow::Status::Invalid("negative seek position"); + } + _pos = position; + return arrow::Status::OK(); + } + + arrow::Result GetSize() override { + if (!_file_reader) { + return arrow::Status::IOError("Doris file reader is not open"); + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + return arrow::Status::IOError("stop"); + } + return static_cast(_file_reader->size()); + } + + arrow::Result Read(int64_t nbytes, void* out) override { + ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(_pos, nbytes, out)); + _pos += bytes_read; + return bytes_read; + } + + arrow::Result> Read(int64_t nbytes) override { + ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); + ARROW_ASSIGN_OR_RAISE(auto bytes_read, Read(nbytes, buffer->mutable_data())); + ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); + buffer->ZeroPadding(); + return buffer; + } + + arrow::Result ReadAt(int64_t position, int64_t nbytes, void* out) override { + if (!_file_reader) { + return arrow::Status::IOError("Doris file reader is not open"); + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + return arrow::Status::IOError("stop"); + } + if (position < 0 || nbytes < 0) { + return arrow::Status::Invalid("negative read position or length"); + } + if (try_read_from_page_cache(position, nbytes, out)) { + return nbytes; + } + size_t bytes_read = 0; + Status st = _file_reader->read_at( + static_cast(position), + Slice(static_cast(out), static_cast(nbytes)), &bytes_read, + _io_ctx); + if (!st.ok()) { + return arrow::Status::IOError(st.to_string_no_stack()); + } + insert_page_cache(position, nbytes, out, bytes_read); + return static_cast(bytes_read); + } + + arrow::Result> ReadAt(int64_t position, + int64_t nbytes) override { + ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); + ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(position, nbytes, buffer->mutable_data())); + ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); + buffer->ZeroPadding(); + return buffer; + } + + void register_page_cache_ranges(std::vector ranges) { + std::lock_guard lock(_page_cache_mutex); + _page_cache_ranges = std::move(ranges); + } + + void prefetch_ranges(const std::vector& ranges, + const io::IOContext* io_ctx) { + auto cached_reader = cached_remote_file_reader(); + if (cached_reader == nullptr) { + return; + } + const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : _io_ctx; + for (const auto& range : ranges) { + if (range.offset < 0 || range.size <= 0) { + continue; + } + cached_reader->prefetch_range(static_cast(range.offset), + static_cast(range.size), prefetch_io_ctx); + } + } + + bool set_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size) { + reset_active_file_reader(); + const auto valid_ranges = detail::valid_prefetch_ranges(ranges); + if (!detail::should_use_merge_range_reader( + valid_ranges, avg_io_size, + typeid_cast(_base_file_reader.get()) != nullptr)) { + return false; + } + + std::vector random_access_ranges; + random_access_ranges.reserve(valid_ranges.size()); + for (const auto& range : valid_ranges) { + random_access_ranges.emplace_back(static_cast(range.offset), + static_cast(range.end_offset())); + } + + // This mirrors the v1 parquet reader: when projected column chunks in a row group are + // small random IOs, make the actual ReadAt path range-aware. Arrow still drives decoding, + // but every page read below this point sees MergeRangeFileReader instead of the raw remote + // reader, so adjacent small requests can be coalesced and served from merge buffers. + // Example: a row group projects leaf chunks [1MB, 1.5MB) and [1.6MB, 2MB). Arrow later + // issues page reads inside those chunks; MergeRangeFileReader can fetch a wider slice once + // and satisfy the following ReadAt calls from its boxes, reducing remote request count. + _merge_range_active = true; + set_active_file_reader(std::make_shared( + profile, _base_file_reader, random_access_ranges, merge_read_slice_size)); + return true; + } + + void reset_random_access_ranges() { reset_active_file_reader(); } + + ParquetPageCacheStats page_cache_stats() const { + std::lock_guard lock(_page_cache_mutex); + return _page_cache_stats; + } + +private: + bool page_cache_enabled() const { + return _enable_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr && !_page_cache_file_key.empty(); + } + + bool range_in_page_cache_scope(int64_t position, int64_t nbytes) const { + if (nbytes <= 0) { + return false; + } + const int64_t end = position + nbytes; + for (const auto& range : _page_cache_ranges) { + const int64_t range_end = range.offset + range.size; + if (position >= range.offset && end <= range_end) { + return true; + } + } + return false; + } + + StoragePageCache::CacheKey page_cache_key(int64_t position, int64_t nbytes) const { + return StoragePageCache::CacheKey(_page_cache_file_key, + static_cast(position + nbytes), position); + } + + bool copy_cached_range(const ParquetPageCacheRange& cached_range, int64_t copy_position, + int64_t copy_size, void* out, int64_t output_offset) { + PageCacheHandle handle; + if (!StoragePageCache::instance()->lookup( + page_cache_key(cached_range.offset, cached_range.size), &handle, + segment_v2::DATA_PAGE)) { + unregister_cached_page_range(_page_cache_file_key, cached_range); + return false; + } + Slice cached = handle.data(); + const int64_t cache_offset = copy_position - cached_range.offset; + DORIS_CHECK(cache_offset >= 0); + DORIS_CHECK(cached.size >= static_cast(cache_offset + copy_size)); + memcpy(static_cast(out) + output_offset, cached.data + cache_offset, + static_cast(copy_size)); + return true; + } + + bool try_read_from_cached_ranges(int64_t position, int64_t nbytes, void* out) { + auto plan = detail::plan_page_cache_range_read( + position, nbytes, cached_page_ranges_for_file(_page_cache_file_key)); + if (plan.empty()) { + return false; + } + for (const auto& entry : plan) { + if (!copy_cached_range(entry.cached_range, + entry.cached_range.offset + entry.copy_offset_in_cache, + entry.copy_size, out, entry.output_offset)) { + return false; + } + } + return true; + } + + bool try_read_from_page_cache(int64_t position, int64_t nbytes, void* out) { + std::lock_guard lock(_page_cache_mutex); + if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes)) { + return false; + } + ++_page_cache_stats.read_count; + // Fast path: Arrow issues the same ReadAt(offset, size) again, so the exact + // StoragePageCache key matches. + // Fallback path: Arrow may read a different but related byte range on another scan. + // Examples: + // - Current request [120, 150) can be served from cached [100, 200) by copying the + // 30-byte subset starting at cached offset 20. + // - Current request [100, 260) can be served by stitching cached [100, 180) and + // [180, 260). If any middle span is missing, it is a miss and the file reader fills + // the whole request from storage. + if (!copy_cached_range(ParquetPageCacheRange {position, nbytes}, position, nbytes, out, + 0) && + !try_read_from_cached_ranges(position, nbytes, out)) { + ++_page_cache_stats.miss_count; + return false; + } + ++_page_cache_stats.hit_count; + ++_page_cache_stats.compressed_hit_count; + return true; + } + + void insert_page_cache(int64_t position, int64_t nbytes, const void* data, size_t bytes_read) { + std::lock_guard lock(_page_cache_mutex); + if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes) || + bytes_read != static_cast(nbytes)) { + return; + } + auto* page = new DataPage(bytes_read, true, segment_v2::DATA_PAGE); + memcpy(page->data(), data, bytes_read); + PageCacheHandle handle; + StoragePageCache::instance()->insert(page_cache_key(position, nbytes), page, &handle, + segment_v2::DATA_PAGE); + register_cached_page_range(_page_cache_file_key, position, nbytes); + ++_page_cache_stats.write_count; + ++_page_cache_stats.compressed_write_count; + } + + void set_active_file_reader(io::FileReaderSPtr reader) { + DORIS_CHECK(reader != nullptr); + _file_reader = _file_reader_stats != nullptr + ? std::make_shared(std::move(reader), + _file_reader_stats) + : std::move(reader); + } + + void reset_active_file_reader() { + collect_active_merge_range_profile(); + _merge_range_active = false; + set_active_file_reader(_base_file_reader); + } + + void collect_active_merge_range_profile() { + if (_merge_range_active && _file_reader != nullptr) { + // MergeRangeFileReader writes its MergedSmallIO counters only from + // collect_profile_before_close(). v2 replaces the active reader for every row group, + // so collect before overwriting it; Close() handles the final row group. Example: + // RG0 installs a merge reader, RG1 calls set_random_access_ranges() and resets the + // active reader first, so RG0's RequestIO/MergedIO counters must be flushed here. + _file_reader->collect_profile_before_close(); + } + } + + std::shared_ptr cached_remote_file_reader() { + if (_merge_range_active) { + return nullptr; + } + auto reader = _file_reader; + if (reader == nullptr) { + return nullptr; + } + // FileReader::init wraps the physical reader with TracingFileReader when scan IO stats are + // enabled. Prefetch should target the physical cached reader below that tracing wrapper, + // otherwise v2 scans with profiling would silently lose prefetch. + if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { + reader = tracing_reader->inner_reader(); + } + return std::dynamic_pointer_cast(reader); + } + + io::FileReaderSPtr _file_reader; + io::FileReaderSPtr _base_file_reader; + io::FileReaderStats* _file_reader_stats = nullptr; + io::IOContext* _io_ctx = nullptr; + int64_t _pos = 0; + bool _closed = false; + bool _enable_page_cache = false; + bool _merge_range_active = false; + std::string _page_cache_file_key; + mutable std::mutex _page_cache_mutex; + std::vector _page_cache_ranges; + ParquetPageCacheStats _page_cache_stats; +}; + +} // namespace + +Status arrow_status_to_doris_status(const arrow::Status& status) { + if (status.ok()) { + return Status::OK(); + } + if (status.IsIOError()) { + return Status::IOError(status.ToString()); + } + if (status.IsInvalid()) { + return Status::InvalidArgument(status.ToString()); + } + return Status::InternalError(status.ToString()); +} + +Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, + bool enable_page_cache, + const io::FileDescription& file_description) { + DORIS_CHECK(input_file_reader != nullptr); + auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); + arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, + enable_page_cache, + std::move(page_cache_file_key)); + try { + // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. + this->file_reader = ::parquet::ParquetFileReader::Open( + arrow_file, ::parquet::default_reader_properties()); + metadata = this->file_reader->metadata(); + schema = metadata != nullptr ? metadata->schema() : nullptr; + } catch (const ::parquet::ParquetException& e) { + if (io_ctx != nullptr && io_ctx->should_stop && + std::string_view(e.what()).find("stop") != std::string_view::npos) { + return Status::EndOfFile("stop"); + } + return Status::Corruption("Failed to open parquet file: {}", e.what()); + } catch (const std::exception& e) { + if (io_ctx != nullptr && io_ctx->should_stop && + std::string_view(e.what()).find("stop") != std::string_view::npos) { + return Status::EndOfFile("stop"); + } + return Status::InternalError("Failed to open parquet file: {}", e.what()); + } + + if (metadata == nullptr || schema == nullptr) { + return Status::Corruption("Failed to read parquet metadata"); + } + return Status::OK(); +} + +void ParquetFileContext::register_page_cache_ranges(std::vector ranges) { + DORIS_CHECK(arrow_file != nullptr); + static_cast(arrow_file.get()) + ->register_page_cache_ranges(std::move(ranges)); +} + +void ParquetFileContext::prefetch_ranges(const std::vector& ranges, + const io::IOContext* io_ctx) { + DORIS_CHECK(arrow_file != nullptr); + static_cast(arrow_file.get())->prefetch_ranges(ranges, io_ctx); +} + +bool ParquetFileContext::set_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size) { + DORIS_CHECK(arrow_file != nullptr); + return static_cast(arrow_file.get()) + ->set_random_access_ranges(ranges, avg_io_size, profile, merge_read_slice_size); +} + +void ParquetFileContext::reset_random_access_ranges() { + DORIS_CHECK(arrow_file != nullptr); + static_cast(arrow_file.get())->reset_random_access_ranges(); +} + +ParquetPageCacheStats ParquetFileContext::page_cache_stats() const { + if (arrow_file == nullptr) { + return {}; + } + return static_cast(arrow_file.get())->page_cache_stats(); +} + +Status ParquetFileContext::close() { + if (file_reader != nullptr) { + try { + file_reader->Close(); + } catch (const std::exception&) { + } + } + if (arrow_file != nullptr) { + static_cast(arrow_status_to_doris_status(arrow_file->Close())); + } + file_reader.reset(); + arrow_file.reset(); + return Status::OK(); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h new file mode 100644 index 00000000000000..0dca52244957d7 --- /dev/null +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader.h" + +namespace doris::io { +struct FileDescription; +struct IOContext; +} // namespace doris::io + +namespace doris { +class RuntimeProfile; +} // namespace doris + +namespace doris::format::parquet { + +struct ParquetPageCacheRange { + int64_t offset = 0; + int64_t size = 0; + + int64_t end_offset() const { return offset + size; } +}; + +struct ParquetPageCacheReadPlanEntry { + // The exact cached StoragePageCache entry. The final cache key is still exact-range based: + // file key + cached_range.end_offset() + cached_range.offset. + ParquetPageCacheRange cached_range; + // Byte offset inside cached_range to start copying from. + int64_t copy_offset_in_cache = 0; + // Byte offset inside the current ReadAt output buffer to start writing to. + int64_t output_offset = 0; + int64_t copy_size = 0; +}; + +struct ParquetPageCacheStats { + int64_t read_count = 0; + int64_t write_count = 0; + int64_t compressed_write_count = 0; + int64_t hit_count = 0; + int64_t miss_count = 0; + int64_t compressed_hit_count = 0; +}; + +namespace detail { + +// Build the copy plan for a ReadAt(position, nbytes) request from the range metadata of +// previously cached entries. +// StoragePageCache cannot do range lookup by itself; it can only lookup an exact key. The +// caller therefore keeps lightweight cached range metadata and uses this function to decide +// which exact cache entries to fetch and which byte spans to copy. +// Examples: +// 1. Subset hit: +// request [120, 150), cached [100, 200) -> copy 30 bytes from cached offset 20. +// 2. Superset hit covered by multiple cached entries: +// request [100, 260), cached [100, 180) and [180, 260) +// -> two copies: [100, 180) to output offset 0, [180, 260) to output offset 80. +// 3. Partial overlap is a miss: +// request [100, 260), cached [100, 180) only -> empty plan, caller reads from file. +std::vector plan_page_cache_range_read( + int64_t position, int64_t nbytes, const std::vector& cached_ranges); + +// Keep only byte ranges that are safe to hand to FileReader implementations. Parquet metadata is +// expected to contain non-negative offsets and positive compressed sizes, but tests and corrupted +// footers can still feed invalid values. Example: [100, 64) is kept, while [-1, 64), [100, 0) and +// an offset+size overflow are ignored. +std::vector valid_prefetch_ranges( + const std::vector& ranges); + +// Average projected column-chunk size for one row group. The v1 parquet path uses this value to +// decide whether a row group is dominated by small random IOs; v2 uses the same signal before +// installing MergeRangeFileReader. Example: chunks of 512KB and 1MB average below SMALL_IO and are +// good merge-reader candidates, while two 8MB chunks should stay on the raw random-access reader. +size_t average_prefetch_range_size(const std::vector& ranges); + +// Decide whether Arrow ReadAt() should be routed through MergeRangeFileReader for the current row +// group. This is intentionally stricter than the background warm-up path: +// - no valid projected chunks -> nothing to merge; +// - in-memory file readers already avoid remote random IO; +// - average chunk size >= MergeRangeFileReader::SMALL_IO would make merged reading wasteful. +bool should_use_merge_range_reader(const std::vector& ranges, + size_t avg_io_size, bool is_in_memory_reader); + +} // namespace detail + +struct ParquetFileContext { + std::shared_ptr arrow_file; // Arrow wrapper for Doris FileReader + std::unique_ptr<::parquet::ParquetFileReader> file_reader; // Arrow Parquet file parser + std::shared_ptr<::parquet::FileMetaData> metadata; // footer metadata (RowGroup information) + const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema + + Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, + const io::FileDescription& file_description); + // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page + // decoding, so v2 caches the serialized bytes read inside these ranges and excludes + // footer/metadata reads that happen before registration. + void register_page_cache_ranges(std::vector ranges); + // Best-effort asynchronous warm-up for Parquet column chunks. This only has an effect when + // the underlying Doris file reader is a CachedRemoteFileReader; other readers keep the same + // random-access behavior and simply skip prefetch. + void prefetch_ranges(const std::vector& ranges, + const io::IOContext* io_ctx); + // Switch the active reader used by Arrow ReadAt() to v1's MergeRangeFileReader when the current + // row group's projected column chunks are small random IOs. This is the real v1-compatible + // prefetch path: subsequent Arrow page reads go through the merged reader instead of merely + // warming file cache in the background. Returns true when merge-range reading is active. + bool set_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size); + // Restore Arrow ReadAt() to the base Doris file reader and flush any active merge-reader + // counters. Row-group setup uses this before dictionary-page probes, because those probes are + // a separate pass over the column chunk from the later Arrow RecordReader data-page stream. + void reset_random_access_ranges(); + ParquetPageCacheStats page_cache_stats() const; + Status close(); +}; + +Status arrow_status_to_doris_status(const arrow::Status& status); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp new file mode 100644 index 00000000000000..d41ff295ec4b3e --- /dev/null +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_profile.h" + +#include "format_v2/parquet/parquet_statistics.h" + +namespace doris::format::parquet { + +void ParquetProfile::init(RuntimeProfile* profile) { + if (profile == nullptr) { + return; + } + + static const char* parquet_profile = "ParquetReader"; + ADD_TIMER_WITH_LEVEL(profile, parquet_profile, 1); + + filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, + parquet_profile, 1); + filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, parquet_profile, 1); + filtered_row_groups_by_dictionary = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "RowGroupsFilteredByDictionary", TUnit::UNIT, parquet_profile, 1); + filtered_row_groups_by_bloom_filter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "RowGroupsFilteredByBloomFilter", TUnit::UNIT, parquet_profile, 1); + filtered_row_groups_by_page_index = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "RowGroupsFilteredByPageIndex", TUnit::UNIT, parquet_profile, 1); + to_read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsReadNum", TUnit::UNIT, + parquet_profile, 1); + total_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsTotalNum", TUnit::UNIT, + parquet_profile, 1); + selected_row_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRowRanges", TUnit::UNIT, + parquet_profile, 1); + filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByGroup", TUnit::UNIT, + parquet_profile, 1); + filtered_page_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByPage", TUnit::UNIT, + parquet_profile, 1); + pages_skipped_by_data_page_filter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "PagesSkippedByDataPageFilter", TUnit::UNIT, parquet_profile, 1); + data_page_filter_skip_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DataPageFilterSkipBytes", + TUnit::BYTES, parquet_profile, 1); + selected_rows = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); + rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", + TUnit::UNIT, parquet_profile, 1); + total_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); + empty_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptySelectionBatches", + TUnit::UNIT, parquet_profile, 1); + range_gap_skipped_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RangeGapSkippedRows", + TUnit::UNIT, parquet_profile, 1); + reader_read_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderReadRows", TUnit::UNIT, + parquet_profile, 1); + reader_skip_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSkipRows", TUnit::UNIT, + parquet_profile, 1); + reader_select_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSelectRows", TUnit::UNIT, + parquet_profile, 1); + arrow_read_records_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowReadRecordsTime", parquet_profile, 1); + materialization_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); + lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", + TUnit::UNIT, parquet_profile, 1); + filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, + parquet_profile, 1); + raw_rows_read = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1); + column_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ColumnReadTime", parquet_profile, 1); + parse_meta_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseMetaTime", parquet_profile, 1); + parse_footer_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseFooterTime", parquet_profile, 1); + file_reader_create_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); + open_file_num = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); + page_index_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, 1); + page_index_filter_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexFilterTime", parquet_profile, 1); + read_page_index_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexReadTime", parquet_profile, 1); + parse_page_index_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexParseTime", parquet_profile, 1); + expr_zonemap_unusable = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ExprZoneMapUnusableEvals", + TUnit::UNIT, parquet_profile, 1); + in_zonemap_point_check = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InZoneMapPointCheckCount", + TUnit::UNIT, parquet_profile, 1); + in_zonemap_range_only = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InZoneMapRangeOnlyCount", + TUnit::UNIT, parquet_profile, 1); + row_group_filter_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); + file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); + file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); + decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, + parquet_profile, 1); + page_read_counter = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageReadCount", TUnit::UNIT, parquet_profile, 1); + page_cache_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheWriteCount", + TUnit::UNIT, parquet_profile, 1); + page_cache_compressed_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "PageCacheCompressedWriteCount", TUnit::UNIT, parquet_profile, 1); + page_cache_decompressed_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "PageCacheDecompressedWriteCount", TUnit::UNIT, parquet_profile, 1); + page_cache_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheHitCount", TUnit::UNIT, + parquet_profile, 1); + page_cache_missing_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheMissingCount", + TUnit::UNIT, parquet_profile, 1); + page_cache_compressed_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "PageCacheCompressedHitCount", TUnit::UNIT, parquet_profile, 1); + page_cache_decompressed_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "PageCacheDecompressedHitCount", TUnit::UNIT, parquet_profile, 1); + decode_header_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageHeaderDecodeTime", parquet_profile, 1); + read_page_header_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageHeaderReadTime", parquet_profile, 1); + decode_value_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeValueTime", parquet_profile, 1); + decode_dict_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeDictTime", parquet_profile, 1); + decode_level_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeLevelTime", parquet_profile, 1); + decode_null_map_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeNullMapTime", parquet_profile, 1); + skip_page_header_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SkipPageHeaderNum", TUnit::UNIT, + parquet_profile, 1); + parse_page_header_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ParsePageHeaderNum", TUnit::UNIT, + parquet_profile, 1); + predicate_filter_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateFilterTime", parquet_profile, 1); + dict_filter_rewrite_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); + dict_filter_expr_rewrite_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterExprRewriteTime", parquet_profile, 1); + dict_filter_read_dict_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterReadDictTime", parquet_profile, 1); + dict_filter_build_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterBuildTime", parquet_profile, 1); + dict_filter_candidate_columns = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "DictFilterCandidateColumns", TUnit::UNIT, parquet_profile, 1); + dict_filter_columns = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DictFilterColumns", TUnit::UNIT, + parquet_profile, 1); + dict_filter_unsupported_columns = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "DictFilterUnsupportedColumns", TUnit::UNIT, parquet_profile, 1); + dict_filter_read_failures = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DictFilterReadFailures", + TUnit::UNIT, parquet_profile, 1); + rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", + TUnit::UNIT, parquet_profile, 1); + convert_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ConvertTime", parquet_profile, 1); + bloom_filter_read_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "BloomFilterReadTime", parquet_profile, 1); +} + +void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_stats) const { + COUNTER_UPDATE(filtered_row_groups, + pruning_stats.total_row_groups - pruning_stats.selected_row_groups); + COUNTER_UPDATE(filtered_row_groups_by_min_max, pruning_stats.filtered_row_groups_by_statistics); + COUNTER_UPDATE(filtered_row_groups_by_dictionary, + pruning_stats.filtered_row_groups_by_dictionary); + COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, + pruning_stats.filtered_row_groups_by_bloom_filter); + COUNTER_UPDATE(filtered_row_groups_by_page_index, + pruning_stats.filtered_row_groups_by_page_index); + COUNTER_UPDATE(to_read_row_groups, pruning_stats.selected_row_groups); + COUNTER_UPDATE(total_row_groups, pruning_stats.total_row_groups); + COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); + COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); + COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); + COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); + COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); + COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); + COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); + COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); + COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); +} + +ParquetPageSkipProfile ParquetProfile::page_skip_profile() const { + return { + .skipped_pages = pages_skipped_by_data_page_filter, + .skipped_bytes = data_page_filter_skip_bytes, + }; +} + +ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { + return { + .reader_read_rows = reader_read_rows, + .reader_skip_rows = reader_skip_rows, + .reader_select_rows = reader_select_rows, + .arrow_read_records_time = arrow_read_records_time, + .materialization_time = materialization_time, + }; +} + +ParquetScanProfile ParquetProfile::scan_profile() const { + return { + .raw_rows_read = raw_rows_read, + .selected_rows = selected_rows, + .rows_filtered_by_conjunct = rows_filtered_by_conjunct, + .lazy_read_filtered_rows = lazy_read_filtered_rows, + .total_batches = total_batches, + .empty_selection_batches = empty_selection_batches, + .range_gap_skipped_rows = range_gap_skipped_rows, + .column_read_time = column_read_time, + .predicate_filter_time = predicate_filter_time, + .dict_filter_rewrite_time = dict_filter_rewrite_time, + .dict_filter_expr_rewrite_time = dict_filter_expr_rewrite_time, + .dict_filter_read_dict_time = dict_filter_read_dict_time, + .dict_filter_build_time = dict_filter_build_time, + .dict_filter_candidate_columns = dict_filter_candidate_columns, + .dict_filter_columns = dict_filter_columns, + .dict_filter_unsupported_columns = dict_filter_unsupported_columns, + .dict_filter_read_failures = dict_filter_read_failures, + .rows_filtered_by_dict_filter = rows_filtered_by_dict_filter, + .column_reader_profile = column_reader_profile(), + }; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h new file mode 100644 index 00000000000000..27f9818f4a0dcd --- /dev/null +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet { + +struct ParquetPruningStats; + +// ============================================================================ +// ============================================================================ +struct ParquetPageSkipProfile { + RuntimeProfile::Counter* skipped_pages = nullptr; // number of data pages skipped by page index + RuntimeProfile::Counter* skipped_bytes = nullptr; // compressed bytes skipped +}; + +// ============================================================================ +// ============================================================================ +struct ParquetColumnReaderProfile { + RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() + RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() + RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() + RuntimeProfile::Counter* arrow_read_records_time = nullptr; // Arrow RecordReader time (ns) + RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) +}; + +// ============================================================================ +// ============================================================================ +struct ParquetScanProfile { + RuntimeProfile::Counter* raw_rows_read = nullptr; // raw rows read from RecordReader + RuntimeProfile::Counter* selected_rows = nullptr; // rows selected after conjunct filtering + RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; // rows filtered by conjuncts + RuntimeProfile::Counter* lazy_read_filtered_rows = + nullptr; // rows avoided by late materialization + RuntimeProfile::Counter* total_batches = nullptr; // total batch count + RuntimeProfile::Counter* empty_selection_batches = + nullptr; // empty batches after full filtering + RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; // rows skipped by range gaps + RuntimeProfile::Counter* column_read_time = nullptr; // column read time (ns) + RuntimeProfile::Counter* predicate_filter_time = nullptr; // predicate filter time (ns) + RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; // dictionary rewrite time (ns) + RuntimeProfile::Counter* dict_filter_expr_rewrite_time = + nullptr; // expression/residual rewrite time (ns) + RuntimeProfile::Counter* dict_filter_read_dict_time = nullptr; // dictionary page read time (ns) + RuntimeProfile::Counter* dict_filter_build_time = + nullptr; // dictionary entry bitmap build time (ns) + RuntimeProfile::Counter* dict_filter_candidate_columns = nullptr; // candidate columns + RuntimeProfile::Counter* dict_filter_columns = nullptr; // optimized columns + RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; // unsupported columns + RuntimeProfile::Counter* dict_filter_read_failures = nullptr; // dictionary read failures + RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; // rows filtered by dict + ParquetColumnReaderProfile column_reader_profile; // nested column read statistics +}; + +// ============================================================================ +// ============================================================================ +// ============================================================================ +struct ParquetProfile { + void init(RuntimeProfile* profile); + void update_pruning_stats(const ParquetPruningStats& pruning_stats) const; + + ParquetPageSkipProfile page_skip_profile() const; + ParquetColumnReaderProfile column_reader_profile() const; + ParquetScanProfile scan_profile() const; + + RuntimeProfile::Counter* filtered_row_groups = nullptr; + RuntimeProfile::Counter* filtered_row_groups_by_min_max = nullptr; + RuntimeProfile::Counter* filtered_row_groups_by_dictionary = nullptr; + RuntimeProfile::Counter* filtered_row_groups_by_bloom_filter = nullptr; + RuntimeProfile::Counter* filtered_row_groups_by_page_index = nullptr; + RuntimeProfile::Counter* to_read_row_groups = nullptr; + RuntimeProfile::Counter* total_row_groups = nullptr; + RuntimeProfile::Counter* selected_row_ranges = nullptr; + RuntimeProfile::Counter* filtered_group_rows = nullptr; + RuntimeProfile::Counter* filtered_page_rows = nullptr; + + // ======== Page Skip ======== + RuntimeProfile::Counter* pages_skipped_by_data_page_filter = nullptr; + RuntimeProfile::Counter* data_page_filter_skip_bytes = nullptr; + + RuntimeProfile::Counter* selected_rows = nullptr; + RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; + RuntimeProfile::Counter* total_batches = nullptr; + RuntimeProfile::Counter* empty_selection_batches = nullptr; + RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; + + // ======== Column Reader ======== + RuntimeProfile::Counter* reader_read_rows = nullptr; + RuntimeProfile::Counter* reader_skip_rows = nullptr; + RuntimeProfile::Counter* reader_select_rows = nullptr; + RuntimeProfile::Counter* arrow_read_records_time = nullptr; + RuntimeProfile::Counter* materialization_time = nullptr; + + RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; + RuntimeProfile::Counter* filtered_bytes = nullptr; + RuntimeProfile::Counter* raw_rows_read = nullptr; + RuntimeProfile::Counter* column_read_time = nullptr; + + RuntimeProfile::Counter* parse_meta_time = nullptr; + RuntimeProfile::Counter* parse_footer_time = nullptr; + RuntimeProfile::Counter* file_reader_create_time = nullptr; + RuntimeProfile::Counter* open_file_num = nullptr; + RuntimeProfile::Counter* file_footer_read_calls = nullptr; + RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + + RuntimeProfile::Counter* row_group_filter_time = nullptr; + RuntimeProfile::Counter* page_index_read_calls = nullptr; + RuntimeProfile::Counter* page_index_filter_time = nullptr; + RuntimeProfile::Counter* read_page_index_time = nullptr; + RuntimeProfile::Counter* parse_page_index_time = nullptr; + RuntimeProfile::Counter* expr_zonemap_unusable = nullptr; + RuntimeProfile::Counter* in_zonemap_point_check = nullptr; + RuntimeProfile::Counter* in_zonemap_range_only = nullptr; + + RuntimeProfile::Counter* decompress_time = nullptr; + RuntimeProfile::Counter* decompress_cnt = nullptr; + RuntimeProfile::Counter* page_read_counter = nullptr; + RuntimeProfile::Counter* page_cache_write_counter = nullptr; + RuntimeProfile::Counter* page_cache_compressed_write_counter = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_write_counter = nullptr; + RuntimeProfile::Counter* page_cache_hit_counter = nullptr; + RuntimeProfile::Counter* page_cache_missing_counter = nullptr; + RuntimeProfile::Counter* page_cache_compressed_hit_counter = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_hit_counter = nullptr; + + RuntimeProfile::Counter* decode_header_time = nullptr; + RuntimeProfile::Counter* read_page_header_time = nullptr; + RuntimeProfile::Counter* decode_value_time = nullptr; + RuntimeProfile::Counter* decode_dict_time = nullptr; + RuntimeProfile::Counter* decode_level_time = nullptr; + RuntimeProfile::Counter* decode_null_map_time = nullptr; + RuntimeProfile::Counter* skip_page_header_num = nullptr; + RuntimeProfile::Counter* parse_page_header_num = nullptr; + + RuntimeProfile::Counter* predicate_filter_time = nullptr; + RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; + RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; + RuntimeProfile::Counter* dict_filter_read_dict_time = nullptr; + RuntimeProfile::Counter* dict_filter_build_time = nullptr; + RuntimeProfile::Counter* dict_filter_candidate_columns = nullptr; + RuntimeProfile::Counter* dict_filter_columns = nullptr; + RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; + RuntimeProfile::Counter* dict_filter_read_failures = nullptr; + RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; + RuntimeProfile::Counter* convert_time = nullptr; + RuntimeProfile::Counter* bloom_filter_read_time = nullptr; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp new file mode 100644 index 00000000000000..753b3628bfa19b --- /dev/null +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -0,0 +1,734 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_reader.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/column_mapper.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/parquet_scan.h" +#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "io/io_common.h" +#include "runtime/runtime_state.h" + +namespace doris::format::parquet { + +struct ParquetReaderScanState { + ParquetFileContext file_context; + std::vector> file_schema; + RowGroupScanPlan scan_plan; + ParquetScanScheduler scheduler; + const RuntimeState* runtime_state = nullptr; + const cctz::time_zone* timezone = nullptr; + bool enable_bloom_filter = false; + bool enable_page_cache = false; + bool enable_strict_mode = false; +}; + +int64_t column_chunk_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { + return column_metadata.has_dictionary_page() + ? cast_set(column_metadata.dictionary_page_offset()) + : cast_set(column_metadata.data_page_offset()); +} + +void collect_all_leaf_column_ids(const ParquetColumnSchema& column_schema, + std::unordered_set* leaf_column_ids) { + DORIS_CHECK(leaf_column_ids != nullptr); + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (column_schema.leaf_column_id >= 0) { + leaf_column_ids->insert(column_schema.leaf_column_id); + } + return; + } + for (const auto& child : column_schema.children) { + DORIS_CHECK(child != nullptr); + collect_all_leaf_column_ids(*child, leaf_column_ids); + } +} + +void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex& projection, + std::unordered_set* leaf_column_ids) { + DORIS_CHECK(leaf_column_ids != nullptr); + if (projection.project_all_children || projection.children.empty()) { + collect_all_leaf_column_ids(column_schema, leaf_column_ids); + return; + } + for (const auto& child_projection : projection.children) { + const auto child_it = + std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { + return child_schema->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != column_schema.children.end()); + collect_projected_leaf_column_ids(**child_it, child_projection, leaf_column_ids); + } +} + +void collect_request_leaf_column_ids( + const std::vector>& file_schema, + const format::FileScanRequest& request, std::unordered_set* leaf_column_ids) { + DORIS_CHECK(leaf_column_ids != nullptr); + auto collect_scan_column = [&](const format::LocalColumnIndex& projection) { + const auto local_id = projection.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID || + local_id == format::GLOBAL_ROWID_COLUMN_ID) { + return; + } + DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); + DORIS_CHECK(file_schema[local_id] != nullptr); + collect_projected_leaf_column_ids(*file_schema[local_id], projection, leaf_column_ids); + }; + for (const auto& column : request.predicate_columns) { + collect_scan_column(column); + } + for (const auto& column : request.non_predicate_columns) { + collect_scan_column(column); + } +} + +std::vector build_page_cache_ranges( + const ::parquet::FileMetaData& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const RowGroupScanPlan& row_group_plan) { + std::unordered_set leaf_column_ids; + collect_request_leaf_column_ids(file_schema, request, &leaf_column_ids); + std::vector ranges; + ranges.reserve(row_group_plan.row_groups.size() * leaf_column_ids.size()); + for (const auto& row_group_plan_item : row_group_plan.row_groups) { + auto row_group_metadata = metadata.RowGroup(row_group_plan_item.row_group_id); + DORIS_CHECK(row_group_metadata != nullptr); + for (const auto leaf_column_id : leaf_column_ids) { + DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); + auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); + DORIS_CHECK(column_metadata != nullptr); + const int64_t offset = column_chunk_start_offset(*column_metadata); + const int64_t size = column_metadata->total_compressed_size(); + DORIS_CHECK(offset >= 0); + DORIS_CHECK(size >= 0); + if (size > 0) { + ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); + } + } + } + return ranges; +} + +const ParquetColumnSchema& projected_root_schema( + const std::vector>& file_schema, + const format::LocalColumnIndex& projection) { + const auto local_id = projection.local_id(); + DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); + DORIS_CHECK(file_schema[local_id] != nullptr); + return *file_schema[local_id]; +} + +int64_t count_loaded_non_null_values(const ParquetColumnSchema& root_schema, + const ParquetColumnReader& shape_reader, + int64_t expected_rows) { + const auto& def_levels = shape_reader.nested_definition_levels(); + const auto& rep_levels = shape_reader.nested_repetition_levels(); + const int64_t levels_written = shape_reader.nested_levels_written(); + DORIS_CHECK(levels_written >= expected_rows); + if (root_schema.max_repetition_level == 0) { + DORIS_CHECK(levels_written == expected_rows); + const int16_t non_null_definition_level = root_schema.nullable_definition_level; + int64_t count = 0; + for (int64_t level_idx = 0; level_idx < levels_written; ++level_idx) { + count += def_levels[level_idx] >= non_null_definition_level ? 1 : 0; + } + return count; + } + + // For repeated encodings, repetition level zero starts a top-level row. Empty MAP/LIST rows + // have no entries but still carry a level slot; they are non-NULL and must be counted by + // count(col). The root nullable level distinguishes a NULL top-level value from a non-NULL + // value regardless of which repeated leaf represents its shape. + const int16_t non_null_definition_level = root_schema.nullable_definition_level; + int64_t counted_rows = 0; + int64_t non_null_rows = 0; + for (int64_t level_idx = 0; level_idx < levels_written && counted_rows < expected_rows; + ++level_idx) { + if (rep_levels[level_idx] != 0) { + continue; + } + ++counted_rows; + non_null_rows += def_levels[level_idx] >= non_null_definition_level ? 1 : 0; + } + DORIS_CHECK(counted_rows == expected_rows); + return non_null_rows; +} + +DataTypePtr nullable_like_original(const DataTypePtr& type, DataTypePtr nested_type) { + return type != nullptr && type->is_nullable() ? make_nullable(nested_type) : nested_type; +} + +int timestamp_tz_scale(const ParquetTypeDescriptor& type_descriptor) { + switch (type_descriptor.time_unit) { + case ParquetTimeUnit::MILLIS: + return 3; + case ParquetTimeUnit::MICROS: + case ParquetTimeUnit::UNKNOWN: + default: + return 6; + } +} + +bool should_map_to_timestamp_tz(const ParquetColumnSchema& column_schema) { + const auto& type_descriptor = column_schema.type_descriptor; + return type_descriptor.physical_type == ::parquet::Type::INT96 || + (type_descriptor.is_timestamp && type_descriptor.timestamp_is_adjusted_to_utc); +} + +DataTypePtr apply_timestamp_tz_mapping(ParquetColumnSchema* column_schema) { + DORIS_CHECK(column_schema != nullptr); + if (column_schema->kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (should_map_to_timestamp_tz(*column_schema)) { + const bool nullable = + column_schema->type != nullptr && column_schema->type->is_nullable(); + const auto scale = timestamp_tz_scale(column_schema->type_descriptor); + column_schema->type = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, + nullable, 0, scale); + column_schema->type_descriptor.doris_type = column_schema->type; + } + return column_schema->type; + } + + std::vector child_types; + child_types.reserve(column_schema->children.size()); + for (auto& child : column_schema->children) { + child_types.push_back(apply_timestamp_tz_mapping(child.get())); + } + + if (column_schema->kind == ParquetColumnSchemaKind::LIST) { + DORIS_CHECK(child_types.size() == 1); + column_schema->type = nullable_like_original( + column_schema->type, std::make_shared(child_types[0])); + } else if (column_schema->kind == ParquetColumnSchemaKind::MAP) { + DORIS_CHECK(child_types.size() == 2); + column_schema->type = nullable_like_original( + column_schema->type, std::make_shared(make_nullable(child_types[0]), + make_nullable(child_types[1]))); + } else if (column_schema->kind == ParquetColumnSchemaKind::STRUCT) { + Strings child_names; + child_names.reserve(column_schema->children.size()); + for (const auto& child : column_schema->children) { + child_names.push_back(child->name); + } + column_schema->type = nullable_like_original( + column_schema->type, std::make_shared(child_types, child_names)); + } + return column_schema->type; +} + +static Status find_projected_minmax_leaf(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex& projection, + const ParquetColumnSchema** leaf_schema) { + DORIS_CHECK(leaf_schema != nullptr); + if (projection.project_all_children || projection.children.empty()) { + if (column_schema.leaf_column_id < 0) { + return Status::NotSupported( + "Parquet aggregate pushdown only supports primitive column {}", + column_schema.name); + } + if (column_schema.max_repetition_level > 0) { + return Status::NotSupported( + "Parquet aggregate pushdown does not support repeated column {}", + column_schema.name); + } + *leaf_schema = &column_schema; + return Status::OK(); + } + if (projection.children.size() != 1) { + return Status::NotSupported( + "Parquet aggregate pushdown only supports a single nested leaf under column {}", + column_schema.name); + } + const auto& child_projection = projection.children[0]; + const auto child_schema_it = + std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { + return child_schema->local_id == child_projection.local_id(); + }); + if (child_schema_it != column_schema.children.end()) { + return find_projected_minmax_leaf(**child_schema_it, child_projection, leaf_schema); + } + return Status::InvalidArgument("Invalid parquet aggregate projection local id {} for column {}", + child_projection.local_id(), column_schema.name); +} + +static Status validate_minmax_aggregate_statistics(const ParquetColumnSchema& column_schema) { + DORIS_CHECK(column_schema.descriptor != nullptr); + switch (column_schema.descriptor->physical_type()) { + case ::parquet::Type::BYTE_ARRAY: + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + // Arrow 17 does not expose Parquet's min/max exactness flags. Binary statistics may be + // truncated bounds rather than values present in the file, so they are safe for pruning + // but cannot be returned as exact aggregate results. + return Status::NotSupported( + "Parquet MIN/MAX aggregate pushdown requires exact statistics for column {}", + column_schema.name); + default: + return Status::OK(); + } +} + +void ParquetReader::_fill_column_definition(const ParquetColumnSchema& column_schema, + format::ColumnDefinition* field) const { + if (column_schema.parquet_field_id >= 0) { + field->identifier = Field::create_field(column_schema.parquet_field_id); + } else { + field->identifier = Field::create_field(column_schema.name); + } + field->local_id = column_schema.local_id; + field->name = column_schema.name; + field->type = column_schema.type != nullptr && !column_schema.type->is_nullable() + ? make_nullable(column_schema.type) + : column_schema.type; + field->children.clear(); + field->children.reserve(column_schema.children.size()); + for (const auto& child : column_schema.children) { + format::ColumnDefinition child_field; + _fill_column_definition(*child, &child_field); + field->children.push_back(std::move(child_field)); + } +} + +ParquetReader::ParquetReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + std::optional global_rowid_context, + bool enable_mapping_timestamp_tz) + : FileReader(system_properties, file_description, io_ctx, profile), + _global_rowid_context(global_rowid_context), + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + +ParquetReader::~ParquetReader() = default; + +Status ParquetReader::init(RuntimeState* state) { + if (_io_ctx != nullptr && _io_ctx->should_stop) { + return Status::EndOfFile("stop"); + } + RETURN_IF_ERROR(format::FileReader::init(state)); + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_reader_create_time, + _reader_statistics.file_reader_create_time); + COUNTER_UPDATE(_parquet_profile.open_file_num, _reader_statistics.open_file_num); + } + _state = std::make_unique(); + _state->enable_bloom_filter = + state != nullptr && state->query_options().enable_parquet_filter_by_bloom_filter; + _state->enable_page_cache = + state != nullptr && state->query_options().enable_parquet_file_page_cache; + if (state != nullptr) { + _state->runtime_state = state; + _state->timezone = &state->timezone_obj(); + _state->enable_strict_mode = state->enable_strict_mode(); + _state->scheduler.set_timezone(&state->timezone_obj()); + _state->scheduler.set_enable_strict_mode(_state->enable_strict_mode); + } + int64_t merge_read_slice_size = -1; + if (state != nullptr && state->query_options().__isset.merge_read_slice_size) { + merge_read_slice_size = state->query_options().merge_read_slice_size; + } + _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); + _state->scheduler.set_batch_size(_batch_size); + // Open parquet file and parse metadata to get file schema. + RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), + _state->enable_page_cache, *_file_description)); + // Build file schema from parquet metadata. + // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier + RETURN_IF_ERROR( + build_parquet_column_schema(*_state->file_context.schema, &_state->file_schema)); + if (_enable_mapping_timestamp_tz) { + for (auto& column_schema : _state->file_schema) { + apply_timestamp_tz_mapping(column_schema.get()); + } + } + return Status::OK(); +} + +void ParquetReader::set_batch_size(size_t batch_size) { + _batch_size = std::max(1, batch_size); + if (_state != nullptr) { + _state->scheduler.set_batch_size(_batch_size); + } +} + +Status ParquetReader::get_schema(std::vector* file_schema) const { + if (file_schema == nullptr) { + return Status::InvalidArgument("file_schema is null"); + } + file_schema->clear(); + if (_state == nullptr || _state->file_context.schema == nullptr) { + return Status::Uninitialized("ParquetReader is not open"); + } + + file_schema->reserve(_state->file_schema.size()); + for (size_t column_idx = 0; column_idx < _state->file_schema.size(); ++column_idx) { + format::ColumnDefinition field; + _fill_column_definition(*_state->file_schema[column_idx], &field); + DORIS_CHECK(field.local_id == static_cast(column_idx)); + file_schema->push_back(std::move(field)); + } + if (_global_rowid_context.has_value()) { + file_schema->push_back(format::global_rowid_column_definition()); + } + return Status::OK(); +} + +std::unique_ptr ParquetReader::create_column_mapper( + format::TableColumnMapperOptions options) const { + return std::make_unique(std::move(options)); +} + +Status ParquetReader::open(std::shared_ptr request) { + if (_state == nullptr || _state->file_context.metadata == nullptr || + _state->file_context.schema == nullptr) { + return Status::Uninitialized("ParquetReader is not open"); + } + auto request_snapshot = request; + DORIS_CHECK(request_snapshot != nullptr); + RETURN_IF_ERROR(format::FileReader::open(std::move(request))); + + // `local_positions.empty()` means all columns are needed by table reader + // TODO(gabriel): It will happen only for TVF `select *` query. + if (request_snapshot->local_positions.empty()) { + for (const auto& col : request_snapshot->predicate_columns) { + request_snapshot->local_positions.emplace(col.column_id(), + format::LocalIndex(col.column_id().value())); + } + for (const auto& col : request_snapshot->non_predicate_columns) { + request_snapshot->local_positions.emplace(col.column_id(), + format::LocalIndex(col.column_id().value())); + } + } + + const auto num_fields = static_cast(_state->file_schema.size()); + for (const auto& col : request_snapshot->predicate_columns) { + DORIS_CHECK(request_snapshot->local_positions.count(col.column_id()) > 0); + const auto local_id = col.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID || + local_id == format::GLOBAL_ROWID_COLUMN_ID) { + continue; + } + DORIS_CHECK(local_id >= 0 && local_id < num_fields); + } + for (const auto& col : request_snapshot->non_predicate_columns) { + DORIS_CHECK(request_snapshot->local_positions.count(col.column_id()) > 0); + const auto local_id = col.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID || + local_id == format::GLOBAL_ROWID_COLUMN_ID) { + continue; + } + DORIS_CHECK(local_id >= 0 && local_id < num_fields); + } + + RowGroupScanPlan row_group_plan; + ParquetScanRange scan_range; + scan_range.start_offset = _file_description->range_start_offset; + scan_range.size = _file_description->range_size; + scan_range.file_size = _file_description->file_size; + // Get selected ranges in row groups according to metadata (Row-Group level index and Page Index including Zonemap, Dictionary, Bloom Filter). + RETURN_IF_ERROR(plan_parquet_row_groups( + *_state->file_context.metadata, _state->file_context.file_reader.get(), + _state->file_schema, *request_snapshot, scan_range, _state->enable_bloom_filter, + &row_group_plan, _state->timezone, _state->runtime_state)); + if (_profile != nullptr) { + _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); + } + if (_state->enable_page_cache) { + _state->file_context.register_page_cache_ranges( + build_page_cache_ranges(*_state->file_context.metadata, _state->file_schema, + *request_snapshot, row_group_plan)); + } + _state->scan_plan = row_group_plan; + _state->scheduler.set_page_skip_profile(_parquet_profile.page_skip_profile()); + _state->scheduler.set_global_rowid_context(_global_rowid_context); + _state->scheduler.set_scan_profile(_parquet_profile.scan_profile()); + _state->scheduler.set_plan(std::move(row_group_plan)); + _eof = _state->scheduler.empty(); + return Status::OK(); +} + +Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { + if (_state == nullptr || _state->file_context.file_reader == nullptr || + _state->file_context.schema == nullptr) { + return Status::Uninitialized("ParquetReader is not open"); + } + *rows = 0; + if (_io_ctx != nullptr && _io_ctx->should_stop) { + *eof = true; + return Status::OK(); + } + if (_eof) { + *eof = true; + return Status::OK(); + } + auto request_snapshot = _request; + if (request_snapshot == nullptr) { + return Status::Cancelled("ParquetReader is closed"); + } + + const auto predicate_filtered_rows_before = _state->scheduler.predicate_filtered_rows(); + const auto raw_rows_read_before = _state->scheduler.raw_rows_read(); + Status st = _state->scheduler.read_next_batch(_state->file_context, _state->file_schema, + *request_snapshot, file_block, rows, eof); + if (!st.ok()) { + if (_io_ctx != nullptr && _io_ctx->should_stop) { + *rows = 0; + *eof = true; + return Status::OK(); + } + return st; + } + _sync_page_cache_profile(); + if (_io_ctx != nullptr) { + _io_ctx->predicate_filtered_rows += + _state->scheduler.predicate_filtered_rows() - predicate_filtered_rows_before; + } + const auto raw_rows_read = _state->scheduler.raw_rows_read(); + DORIS_CHECK(raw_rows_read >= raw_rows_read_before); + _record_scan_rows(raw_rows_read - raw_rows_read_before); + _eof = *eof; + return Status::OK(); +} + +bool ParquetReader::_should_stop() const { + return _io_ctx != nullptr && _io_ctx->should_stop; +} + +Status ParquetReader::_stop_status_if_requested(const Status& status) const { + if (!status.ok() && _should_stop()) { + return Status::EndOfFile("stop"); + } + return status; +} + +void ParquetReader::_sync_page_cache_profile() { + if (_profile == nullptr || _state == nullptr) { + return; + } + const auto stats = _state->file_context.page_cache_stats(); + COUNTER_UPDATE(_parquet_profile.page_read_counter, + stats.read_count - _reported_page_cache_stats.read_count); + COUNTER_UPDATE(_parquet_profile.page_cache_write_counter, + stats.write_count - _reported_page_cache_stats.write_count); + COUNTER_UPDATE( + _parquet_profile.page_cache_compressed_write_counter, + stats.compressed_write_count - _reported_page_cache_stats.compressed_write_count); + COUNTER_UPDATE(_parquet_profile.page_cache_hit_counter, + stats.hit_count - _reported_page_cache_stats.hit_count); + COUNTER_UPDATE(_parquet_profile.page_cache_missing_counter, + stats.miss_count - _reported_page_cache_stats.miss_count); + COUNTER_UPDATE(_parquet_profile.page_cache_compressed_hit_counter, + stats.compressed_hit_count - _reported_page_cache_stats.compressed_hit_count); + _reported_page_cache_stats = stats; +} + +void ParquetReader::set_condition_cache_context(std::shared_ptr ctx) { + if (_state == nullptr) { + return; + } + _state->scheduler.set_condition_cache_context(std::move(ctx)); + if (_io_ctx != nullptr) { + // Condition-cache HIT filters row ranges before batch reading, so skipped rows never belong + // to a later get_block() batch. Report the plan-level skipped rows at the same point where + // the scan plan is rewritten. + _io_ctx->condition_cache_filtered_rows += _state->scheduler.condition_cache_filtered_rows(); + } +} + +int64_t ParquetReader::get_total_rows() const { + if (_state == nullptr) { + return 0; + } + int64_t rows = 0; + for (const auto& row_group_plan : _state->scan_plan.row_groups) { + rows += row_group_plan.row_group_rows; + } + return rows; +} + +Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& request, + format::FileAggregateResult* result) { + DORIS_CHECK(result != nullptr); + if (_state == nullptr || _state->file_context.metadata == nullptr || + _state->file_context.schema == nullptr) { + return Status::Uninitialized("ParquetReader is not open"); + } + if (_should_stop()) { + return Status::EndOfFile("stop"); + } + result->count = 0; + result->columns.clear(); + if (request.agg_type != TPushAggOp::type::COUNT && + request.agg_type != TPushAggOp::type::MINMAX) { + return Status::NotSupported("Unsupported parquet aggregate pushdown type {}", + request.agg_type); + } + + // Aggregate row count in all selected row groups. For MIN/MAX aggregate, this is used to determine whether there is no row group selected. + for (const auto& row_group_plan : _state->scan_plan.row_groups) { + auto row_group_metadata = + _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); + DORIS_CHECK(row_group_metadata != nullptr); + result->count += row_group_metadata->num_rows(); + } + if (request.agg_type == TPushAggOp::type::COUNT) { + if (request.columns.empty()) { + return Status::OK(); + } + if (request.columns.size() != 1) { + return Status::NotSupported("Parquet COUNT pushdown only supports one count column"); + } + const auto& count_projection = request.columns[0].projection; + const auto& root_schema = projected_root_schema(_state->file_schema, count_projection); + result->count = 0; + for (const auto& row_group_plan : _state->scan_plan.row_groups) { + std::shared_ptr<::parquet::RowGroupReader> row_group; + try { + row_group = _state->file_context.file_reader->RowGroup(row_group_plan.row_group_id); + } catch (const ::parquet::ParquetException& e) { + if (_should_stop()) { + return Status::EndOfFile("stop"); + } + return Status::Corruption("Failed to open parquet row group {}: {}", + row_group_plan.row_group_id, e.what()); + } catch (const std::exception& e) { + if (_should_stop()) { + return Status::EndOfFile("stop"); + } + return Status::InternalError("Failed to open parquet row group {}: {}", + row_group_plan.row_group_id, e.what()); + } + + ParquetColumnReaderFactory column_reader_factory( + row_group, _state->file_context.schema->num_columns(), + &row_group_plan.page_skip_plans, _parquet_profile.page_skip_profile(), + _state->timezone, _state->enable_strict_mode, + _parquet_profile.scan_profile().column_reader_profile); + std::unique_ptr shape_reader; + RETURN_IF_ERROR(column_reader_factory.create_count_shape_reader( + root_schema, &count_projection, &shape_reader)); + DORIS_CHECK(shape_reader != nullptr); + + int64_t row_group_cursor = 0; + for (const auto& selected_range : row_group_plan.selected_ranges) { + DORIS_CHECK(selected_range.start >= row_group_cursor); + RETURN_IF_ERROR(_stop_status_if_requested( + shape_reader->skip(selected_range.start - row_group_cursor))); + row_group_cursor = selected_range.start; + + int64_t range_rows_read = 0; + while (range_rows_read < selected_range.length) { + const int64_t batch_rows = + std::min(_batch_size, selected_range.length - range_rows_read); + // COUNT(col) only needs the top-level NULL state. The shape reader loads + // def/rep levels from one representative leaf and does not build value_indices + // or values_column. MAP chooses the key leaf; ARRAY/STRUCT may choose a string + // leaf, but the levels-only protocol still avoids Doris-side string + // materialization for that leaf. + RETURN_IF_ERROR(_stop_status_if_requested( + shape_reader->load_nested_levels_batch(batch_rows))); + _record_scan_rows(batch_rows); + result->count += + count_loaded_non_null_values(root_schema, *shape_reader, batch_rows); + range_rows_read += batch_rows; + row_group_cursor += batch_rows; + } + } + } + return Status::OK(); + } + + result->columns.resize(request.columns.size()); + for (size_t request_column_idx = 0; request_column_idx < request.columns.size(); + ++request_column_idx) { + const auto file_column_id = request.columns[request_column_idx].projection.local_id(); + if (file_column_id < 0 || + file_column_id >= static_cast(_state->file_schema.size())) { + return Status::InvalidArgument("Invalid parquet aggregate column id {}", + file_column_id); + } + const auto& column_schema = _state->file_schema[file_column_id]; + DORIS_CHECK(column_schema != nullptr); + const ParquetColumnSchema* leaf_schema = nullptr; + RETURN_IF_ERROR(find_projected_minmax_leaf( + *column_schema, request.columns[request_column_idx].projection, &leaf_schema)); + DORIS_CHECK(leaf_schema != nullptr); + RETURN_IF_ERROR(validate_minmax_aggregate_statistics(*leaf_schema)); + + auto& aggregate_column = result->columns[request_column_idx]; + aggregate_column.projection = request.columns[request_column_idx].projection; + for (const auto& row_group_plan : _state->scan_plan.row_groups) { + auto row_group_metadata = + _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); + DORIS_CHECK(row_group_metadata != nullptr); + auto column_chunk = row_group_metadata->ColumnChunk(leaf_schema->leaf_column_id); + DORIS_CHECK(column_chunk != nullptr); + const auto statistics = ParquetStatisticsUtils::TransformColumnStatistics( + *leaf_schema, column_chunk->statistics(), _state->timezone); + if (!statistics.has_min_max) { + return Status::NotSupported("Missing parquet min/max statistics for column {}", + leaf_schema->name); + } + if (!aggregate_column.has_min || statistics.min_value < aggregate_column.min_value) { + aggregate_column.min_value = statistics.min_value; + aggregate_column.has_min = true; + } + if (!aggregate_column.has_max || aggregate_column.max_value < statistics.max_value) { + aggregate_column.max_value = statistics.max_value; + aggregate_column.has_max = true; + } + } + if (!aggregate_column.has_min || !aggregate_column.has_max) { + return Status::NotSupported("No parquet row group selected for min/max pushdown"); + } + } + return Status::OK(); +} + +Status ParquetReader::close() { + if (_state != nullptr) { + _sync_page_cache_profile(); + RETURN_IF_ERROR(_state->file_context.close()); + } + return FileReader::close(); +} + +void ParquetReader::_init_profile() { + _parquet_profile.init(_profile); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h new file mode 100644 index 00000000000000..6c8e88cc27a9b6 --- /dev/null +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/parquet_scan.h" + +namespace doris { +namespace io { +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet { + +struct ParquetReaderScanState; + +// ============================================================================ +// ============================================================================ +// init() -> get_schema() -> open(request) -> get_block() [loop] -> close() +// ============================================================================ +class ParquetReader : public format::FileReader { +public: + ParquetReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + std::optional global_rowid_context = std::nullopt, + bool enable_mapping_timestamp_tz = false); + ~ParquetReader() override; + + Status init(RuntimeState* state) override; + + void set_batch_size(size_t batch_size) override; + + Status get_schema(std::vector* file_schema) const override; + + std::unique_ptr create_column_mapper( + format::TableColumnMapperOptions options) const override; + + Status open(std::shared_ptr request) override; + + Status get_block(Block* file_block, size_t* rows, bool* eof) override; + + Status get_aggregate_result(const format::FileAggregateRequest& request, + format::FileAggregateResult* result) override; + + void set_condition_cache_context(std::shared_ptr ctx) override; + + int64_t get_total_rows() const override; + + Status close() override; + +protected: + void _init_profile() override; + +private: + void _sync_page_cache_profile(); + bool _should_stop() const; + Status _stop_status_if_requested(const Status& status) const; + + void _fill_column_definition(const ParquetColumnSchema& column_schema, + format::ColumnDefinition* field) const; + + std::unique_ptr + _state; // complete scan state (file_context + schema + scheduler) + ParquetProfile _parquet_profile; // RuntimeProfile counter set + ParquetPageCacheStats _reported_page_cache_stats; + std::optional _global_rowid_context; // global RowId context + size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; + bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp new file mode 100644 index 00000000000000..a7602cd11b27d0 --- /dev/null +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -0,0 +1,1524 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_scan.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/exception.h" +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "exprs/vcompound_pred.h" +#include "exprs/vexpr_context.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/parquet_statistics.h" + +namespace doris::format::parquet { + +namespace { + +int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { + return column_metadata.has_dictionary_page() + ? cast_set(column_metadata.dictionary_page_offset()) + : cast_set(column_metadata.data_page_offset()); +} + +bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { + return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || + encoding == ::parquet::Encoding::RLE_DICTIONARY; +} + +bool is_level_encoding(::parquet::Encoding::type encoding) { + return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; +} + +bool is_data_page_type(::parquet::PageType::type page_type) { + return page_type == ::parquet::PageType::DATA_PAGE || + page_type == ::parquet::PageType::DATA_PAGE_V2; +} + +bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { + if (!column_metadata.has_dictionary_page()) { + return false; + } + + const auto& encoding_stats = column_metadata.encoding_stats(); + if (!encoding_stats.empty()) { + bool has_dictionary_data_page = false; + for (const auto& encoding_stat : encoding_stats) { + if (!is_data_page_type(encoding_stat.page_type) || encoding_stat.count <= 0) { + continue; + } + if (!is_dictionary_data_encoding(encoding_stat.encoding)) { + return false; + } + has_dictionary_data_page = true; + } + return has_dictionary_data_page; + } + + bool has_dictionary_encoding = false; + for (const auto encoding : column_metadata.encodings()) { + if (is_dictionary_data_encoding(encoding)) { + has_dictionary_encoding = true; + continue; + } + if (!is_level_encoding(encoding)) { + return false; + } + } + return has_dictionary_encoding; +} + +bool supports_row_level_dictionary_filter(const ParquetColumnSchema& column_schema, + const ::parquet::ColumnChunkMetaData& column_metadata) { + if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || + column_schema.descriptor == nullptr || column_schema.type == nullptr || + column_schema.max_repetition_level > 0) { + return false; + } + if (!column_schema.type_descriptor.is_string_like || + column_metadata.type() != ::parquet::Type::BYTE_ARRAY) { + return false; + } + // Row-level dictionary filtering consumes dictionary ids from DATA_PAGE payloads. It is exact + // only when every data page is dictionary encoded. Mixed dictionary/plain chunks are left on + // the normal decoded-value path, matching the safety rule used by StarRocks and Doris v1. + return is_fully_dictionary_encoded_chunk(column_metadata); +} + +void collect_all_leaf_column_ids(const ParquetColumnSchema& column_schema, + std::unordered_set* leaf_column_ids) { + DORIS_CHECK(leaf_column_ids != nullptr); + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (column_schema.leaf_column_id >= 0) { + leaf_column_ids->insert(column_schema.leaf_column_id); + } + return; + } + for (const auto& child : column_schema.children) { + DORIS_CHECK(child != nullptr); + collect_all_leaf_column_ids(*child, leaf_column_ids); + } +} + +void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex& projection, + std::unordered_set* leaf_column_ids) { + DORIS_CHECK(leaf_column_ids != nullptr); + if (projection.project_all_children || projection.children.empty()) { + collect_all_leaf_column_ids(column_schema, leaf_column_ids); + return; + } + for (const auto& child_projection : projection.children) { + const auto child_it = + std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { + return child_schema->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != column_schema.children.end()); + collect_projected_leaf_column_ids(**child_it, child_projection, leaf_column_ids); + } +} + +bool is_row_group_outside_range(const ::parquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, int row_group_idx) { + if (scan_range.size < 0) { + return false; + } + const int64_t range_start_offset = scan_range.start_offset; + const int64_t range_end_offset = range_start_offset + scan_range.size; + DORIS_CHECK(range_start_offset >= 0); + DORIS_CHECK(range_end_offset >= range_start_offset); + if (range_start_offset == 0 && + (scan_range.file_size < 0 || range_end_offset >= scan_range.file_size)) { + return false; + } + + auto row_group_metadata = metadata.RowGroup(row_group_idx); + DORIS_CHECK(row_group_metadata != nullptr); + DORIS_CHECK(row_group_metadata->num_columns() > 0); + const auto first_column = row_group_metadata->ColumnChunk(0); + const auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); + DORIS_CHECK(first_column != nullptr); + DORIS_CHECK(last_column != nullptr); + const int64_t row_group_start_offset = column_start_offset(*first_column); + const int64_t row_group_end_offset = + column_start_offset(*last_column) + last_column->total_compressed_size(); + const int64_t row_group_mid_offset = + row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; + return row_group_mid_offset < range_start_offset || row_group_mid_offset >= range_end_offset; +} + +std::vector request_scan_columns(const format::FileScanRequest& request) { + std::vector scan_columns; + scan_columns.reserve(request.predicate_columns.size() + request.non_predicate_columns.size()); + scan_columns.insert(scan_columns.end(), request.predicate_columns.begin(), + request.predicate_columns.end()); + scan_columns.insert(scan_columns.end(), request.non_predicate_columns.begin(), + request.non_predicate_columns.end()); + return scan_columns; +} + +std::vector build_row_group_prefetch_ranges( + const ::parquet::FileMetaData& metadata, + const std::vector>& file_schema, + const std::vector& scan_columns, int row_group_idx) { + std::unordered_set leaf_column_ids; + for (const auto& projection : scan_columns) { + const auto local_id = projection.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID || + local_id == format::GLOBAL_ROWID_COLUMN_ID) { + continue; + } + DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); + DORIS_CHECK(file_schema[local_id] != nullptr); + // Prefetch and merge-reader ranges must be physical leaf chunks, not Doris logical slots. + // Example: for a struct column s, projecting only s.a should include only + // the Parquet leaf chunk of a. Projecting the whole struct includes both a and b. + collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); + } + + auto row_group_metadata = metadata.RowGroup(row_group_idx); + DORIS_CHECK(row_group_metadata != nullptr); + std::vector ordered_leaf_column_ids(leaf_column_ids.begin(), leaf_column_ids.end()); + std::ranges::sort(ordered_leaf_column_ids); + + std::vector ranges; + ranges.reserve(ordered_leaf_column_ids.size()); + for (const auto leaf_column_id : ordered_leaf_column_ids) { + DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); + auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); + DORIS_CHECK(column_metadata != nullptr); + const int64_t offset = column_start_offset(*column_metadata); + const int64_t size = column_metadata->total_compressed_size(); + DORIS_CHECK(offset >= 0); + if (size > 0) { + ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); + } + } + return ranges; +} + +Status select_row_groups_by_scan_range(const ::parquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups) { + DORIS_CHECK(row_group_first_rows != nullptr); + DORIS_CHECK(selected_row_groups != nullptr); + row_group_first_rows->assign(metadata.num_row_groups(), 0); + selected_row_groups->clear(); + selected_row_groups->reserve(metadata.num_row_groups()); + int64_t next_row_group_first_row = 0; + for (int row_group_idx = 0; row_group_idx < metadata.num_row_groups(); ++row_group_idx) { + (*row_group_first_rows)[row_group_idx] = next_row_group_first_row; + auto row_group_metadata = metadata.RowGroup(row_group_idx); + DORIS_CHECK(row_group_metadata != nullptr); + const int64_t row_group_rows = row_group_metadata->num_rows(); + if (row_group_rows < 0) { + return Status::Corruption("Invalid negative row count in parquet row group {}", + row_group_idx); + } + next_row_group_first_row += row_group_rows; + if (!is_row_group_outside_range(metadata, scan_range, row_group_idx)) { + selected_row_groups->push_back(row_group_idx); + } + } + return Status::OK(); +} + +Status build_row_group_read_plans( + const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector& selected_row_groups, + const std::vector& row_group_first_rows, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state) { + DORIS_CHECK(plan != nullptr); + plan->row_groups.reserve(selected_row_groups.size()); + for (const auto row_group_idx : selected_row_groups) { + DORIS_CHECK(row_group_idx >= 0); + DORIS_CHECK(static_cast(row_group_idx) < row_group_first_rows.size()); + auto row_group_metadata = metadata.RowGroup(row_group_idx); + DORIS_CHECK(row_group_metadata != nullptr); + const int64_t row_group_rows = row_group_metadata->num_rows(); + if (row_group_rows == 0) { + continue; + } + + RowGroupReadPlan row_group_plan; + row_group_plan.row_group_id = row_group_idx; + row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; + row_group_plan.row_group_rows = row_group_rows; + RETURN_IF_ERROR(select_row_group_ranges_by_page_index( + file_reader, file_schema, request, row_group_idx, row_group_rows, + &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, + &plan->pruning_stats, timezone, runtime_state)); + if (row_group_plan.selected_ranges.empty()) { + continue; + } + plan->pruning_stats.selected_row_ranges += row_group_plan.selected_ranges.size(); + plan->row_groups.push_back(std::move(row_group_plan)); + } + return Status::OK(); +} + +} // namespace + +Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, + ::parquet::ParquetFileReader* file_reader, + const std::vector>& file_schema, + const format::FileScanRequest& request, + const ParquetScanRange& scan_range, bool enable_bloom_filter, + RowGroupScanPlan* plan, const cctz::time_zone* timezone, + const RuntimeState* runtime_state) { + DORIS_CHECK(plan != nullptr); + plan->row_groups.clear(); + plan->pruning_stats = ParquetPruningStats {}; + + // Row-group planning flow: + // + // parquet footer row groups + // | + // v + // split byte-range candidates + // | + // v + // row-group metadata pruning + // statistics/ZoneMap -> dictionary -> bloom filter + // | + // v + // page-index pruning per selected row group + // | + // v + // RowGroupReadPlan with selected row ranges + // + // Metadata pruning removes whole row groups before readers are opened. Page index pruning runs + // only for remaining row groups and produces selected row ranges; the scan scheduler later skips + // gaps between those ranges, while row-level VExpr conjuncts still run on loaded batches for + // correctness. + std::vector row_group_first_rows; + std::vector scan_range_selected_row_groups; + RETURN_IF_ERROR(select_row_groups_by_scan_range(metadata, scan_range, &row_group_first_rows, + &scan_range_selected_row_groups)); + + std::vector metadata_selected_row_groups; + RETURN_IF_ERROR(select_row_groups_by_metadata( + metadata, file_reader, file_schema, request, &scan_range_selected_row_groups, + &metadata_selected_row_groups, enable_bloom_filter, &plan->pruning_stats, timezone, + runtime_state)); + + RETURN_IF_ERROR(build_row_group_read_plans(metadata, file_reader, file_schema, request, + metadata_selected_row_groups, row_group_first_rows, + plan, timezone, runtime_state)); + plan->pruning_stats.selected_row_groups = plan->row_groups.size(); + return Status::OK(); +} + +namespace { + +using DictionaryResidualConjunct = std::pair; +using DictionaryResidualConjuncts = std::vector; + +void update_counter_if_not_null(RuntimeProfile::Counter* counter, int64_t value) { + if (counter != nullptr) { + COUNTER_UPDATE(counter, value); + } +} + +uint16_t apply_filter_to_selection(const IColumn::Filter& filter, SelectionVector* selection, + uint16_t selected_rows) { + uint16_t new_selected_rows = 0; + for (uint16_t selection_idx = 0; selection_idx < selected_rows; ++selection_idx) { + const auto row_idx = selection->get_index(selection_idx); + if (filter[row_idx] != 0) { + selection->set_index(new_selected_rows++, static_cast(row_idx)); + } + } + return new_selected_rows; +} + +Status execute_compact_filter_conjuncts(const VExprContextSPtrs& conjuncts, size_t rows, + Block* file_block, IColumn::Filter* compact_filter, + bool* can_filter_all) { + DORIS_CHECK(compact_filter != nullptr); + DORIS_CHECK(can_filter_all != nullptr); + compact_filter->resize_fill(rows, 1); + *can_filter_all = false; + for (const auto& conjunct : conjuncts) { + DORIS_CHECK(conjunct != nullptr); + IColumn::Filter filter(rows, 1); + bool conjunct_can_filter_all = false; + RETURN_IF_ERROR(conjunct->execute_filter(file_block, filter.data(), rows, false, + &conjunct_can_filter_all)); + if (conjunct_can_filter_all) { + std::ranges::fill(*compact_filter, 0); + *can_filter_all = true; + break; + } + for (size_t row = 0; row < rows; ++row) { + (*compact_filter)[row] &= filter[row]; + } + } + return Status::OK(); +} + +Status execute_compact_dictionary_residual_conjuncts(const DictionaryResidualConjuncts& conjuncts, + size_t rows, Block* file_block, + IColumn::Filter* compact_filter, + bool* can_filter_all) { + DORIS_CHECK(compact_filter != nullptr); + DORIS_CHECK(can_filter_all != nullptr); + compact_filter->resize_fill(rows, 1); + *can_filter_all = false; + for (const auto& [owner_context, residual_expr] : conjuncts) { + DORIS_CHECK(owner_context != nullptr); + DORIS_CHECK(residual_expr != nullptr); + IColumn::Filter filter(rows, 1); + bool conjunct_can_filter_all = false; + RETURN_IF_ERROR(residual_expr->execute_filter(owner_context.get(), file_block, + filter.data(), rows, false, + &conjunct_can_filter_all)); + if (conjunct_can_filter_all) { + std::ranges::fill(*compact_filter, 0); + *can_filter_all = true; + break; + } + for (size_t row = 0; row < rows; ++row) { + (*compact_filter)[row] &= filter[row]; + } + } + return Status::OK(); +} + +Status execute_compact_delete_conjuncts(const VExprContextSPtrs& delete_conjuncts, size_t rows, + Block* file_block, IColumn::Filter* compact_filter, + bool* can_filter_all) { + DORIS_CHECK(compact_filter != nullptr); + DORIS_CHECK(can_filter_all != nullptr); + compact_filter->resize_fill(rows, 1); + *can_filter_all = false; + for (const auto& delete_conjunct : delete_conjuncts) { + DORIS_CHECK(delete_conjunct != nullptr); + int result_column_id = -1; + RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, + &result_column_id)); + DORIS_CHECK(result_column_id >= 0 && + result_column_id < static_cast(file_block->columns())); + const auto& delete_filter = assert_cast( + *file_block->get_by_position(result_column_id).column) + .get_data(); + DORIS_CHECK(delete_filter.size() == rows); + bool has_kept_row = false; + for (size_t row = 0; row < rows; ++row) { + (*compact_filter)[row] &= !delete_filter[row]; + has_kept_row |= (*compact_filter)[row] != 0; + } + file_block->erase(result_column_id); + if (!has_kept_row) { + *can_filter_all = true; + break; + } + } + return Status::OK(); +} + +Status execute_filter_conjuncts(const format::FileScanRequest& request, int64_t batch_rows, + Block* file_block, SelectionVector* selection, + uint16_t* selected_rows) { + for (const auto& conjunct : request.conjuncts) { + if (*selected_rows == 0) { + break; + } + DORIS_CHECK(conjunct != nullptr); + IColumn::Filter filter(static_cast(batch_rows), 1); + bool can_filter_all = false; + RETURN_IF_ERROR(conjunct->execute_filter(file_block, filter.data(), + static_cast(batch_rows), false, + &can_filter_all)); + *selected_rows = + can_filter_all ? 0 : apply_filter_to_selection(filter, selection, *selected_rows); + } + return Status::OK(); +} + +Status execute_delete_conjuncts(const format::FileScanRequest& request, int64_t batch_rows, + Block* file_block, SelectionVector* selection, + uint16_t* selected_rows) { + for (const auto& delete_conjunct : request.delete_conjuncts) { + if (*selected_rows == 0) { + break; + } + DORIS_CHECK(delete_conjunct != nullptr); + int result_column_id = -1; + RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, + &result_column_id)); + DORIS_CHECK(result_column_id >= 0 && + result_column_id < static_cast(file_block->columns())); + const auto& delete_filter = assert_cast( + *file_block->get_by_position(result_column_id).column) + .get_data(); + DORIS_CHECK(delete_filter.size() == static_cast(batch_rows)); + IColumn::Filter keep_filter(static_cast(batch_rows), 1); + bool has_kept_row = false; + for (size_t row = 0; row < static_cast(batch_rows); ++row) { + keep_filter[row] = !delete_filter[row]; + has_kept_row |= keep_filter[row] != 0; + } + file_block->erase(result_column_id); + *selected_rows = + !has_kept_row ? 0 + : apply_filter_to_selection(keep_filter, selection, *selected_rows); + } + return Status::OK(); +} + +} // namespace + +uint16_t apply_compact_filter_to_selection(const IColumn::Filter& filter, + SelectionVector* selection, uint16_t selected_rows) { + DORIS_CHECK(selection != nullptr); + DORIS_CHECK(filter.size() == selected_rows); + uint16_t new_selected_rows = 0; + for (uint16_t selection_idx = 0; selection_idx < selected_rows; ++selection_idx) { + if (filter[selection_idx] != 0) { + selection->set_index(new_selected_rows++, static_cast( + selection->get_index(selection_idx))); + } + } + return new_selected_rows; +} + +IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows) { + IColumn::Filter filter(static_cast(batch_rows), 0); + for (uint16_t selection_idx = 0; selection_idx < selected_rows; ++selection_idx) { + filter[selection.get_index(selection_idx)] = 1; + } + return filter; +} + +Status execute_batch_filters(const format::FileScanRequest& request, int64_t batch_rows, + Block* file_block, SelectionVector* selection, uint16_t* selected_rows, + int64_t* conjunct_filtered_rows) { + if (request.conjuncts.empty() && request.delete_conjuncts.empty()) { + return Status::OK(); + } + const auto selected_rows_before_conjunct = *selected_rows; + RETURN_IF_ERROR( + execute_filter_conjuncts(request, batch_rows, file_block, selection, selected_rows)); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += static_cast(selected_rows_before_conjunct) - + static_cast(*selected_rows); + } + if (*selected_rows == 0) { + return Status::OK(); + } + return execute_delete_conjuncts(request, batch_rows, file_block, selection, selected_rows); +} + +namespace { +int64_t count_range_rows(const std::vector& ranges) { + int64_t rows = 0; + for (const auto& range : ranges) { + rows += range.length; + } + return rows; +} + +void append_intersection(const RowRange& left, const RowRange& right, + std::vector* result) { + const int64_t start = std::max(left.start, right.start); + const int64_t end = std::min(left.start + left.length, right.start + right.length); + if (start < end) { + result->push_back(RowRange {.start = start, .length = end - start}); + } +} + +std::vector filter_ranges_by_condition_cache(const std::vector& ranges, + const std::vector& cache, + int64_t row_group_first_row, + int64_t base_granule) { + std::vector result; + if (cache.empty()) { + return ranges; + } + + // Cache coordinates are file-global granules; RowRange coordinates are row-group-relative. + // Walk every selected range in order and split it by granule. Granules covered by the bitmap + // are kept only when the bit is true. Granules outside the bitmap are kept conservatively, so + // an undersized or old-format cache entry cannot skip valid rows. + for (const auto& range : ranges) { + const int64_t global_start = row_group_first_row + range.start; + const int64_t global_end = global_start + range.length; + for (int64_t granule = global_start / ConditionCacheContext::GRANULE_SIZE; + granule <= (global_end - 1) / ConditionCacheContext::GRANULE_SIZE; ++granule) { + const int64_t cache_idx = granule - base_granule; + const bool keep = cache_idx < 0 || static_cast(cache_idx) >= cache.size() || + cache[static_cast(cache_idx)]; + if (!keep) { + continue; + } + const int64_t granule_start = granule * ConditionCacheContext::GRANULE_SIZE; + const int64_t granule_end = granule_start + ConditionCacheContext::GRANULE_SIZE; + const RowRange file_granule_range {.start = granule_start - row_group_first_row, + .length = granule_end - granule_start}; + append_intersection(range, file_granule_range, &result); + } + } + return result; +} + +} // namespace + +void ParquetScanScheduler::set_plan(RowGroupScanPlan plan) { + _row_group_plans = std::move(plan.row_groups); + _condition_cache_filtered_rows = 0; + _predicate_filtered_rows = 0; + reset(); +} + +void ParquetScanScheduler::set_condition_cache_context(std::shared_ptr ctx) { + _condition_cache_ctx = std::move(ctx); + if (!_condition_cache_ctx || !_condition_cache_ctx->filter_result || _row_group_plans.empty()) { + return; + } + + if (!_condition_cache_ctx->is_hit) { + _condition_cache_ctx->base_granule = + _row_group_plans.front().first_file_row / ConditionCacheContext::GRANULE_SIZE; + const auto& last_plan = _row_group_plans.back(); + const int64_t end_granule = (last_plan.first_file_row + last_plan.row_group_rows + + ConditionCacheContext::GRANULE_SIZE - 1) / + ConditionCacheContext::GRANULE_SIZE; + DORIS_CHECK(end_granule > _condition_cache_ctx->base_granule); + _condition_cache_ctx->num_granules = + std::min(_condition_cache_ctx->filter_result->size(), + static_cast(end_granule - _condition_cache_ctx->base_granule)); + return; + } + + std::vector filtered_plans; + filtered_plans.reserve(_row_group_plans.size()); + for (auto& plan : _row_group_plans) { + const int64_t old_rows = count_range_rows(plan.selected_ranges); + plan.selected_ranges = filter_ranges_by_condition_cache( + plan.selected_ranges, *_condition_cache_ctx->filter_result, plan.first_file_row, + _condition_cache_ctx->base_granule); + const int64_t new_rows = count_range_rows(plan.selected_ranges); + _condition_cache_filtered_rows += old_rows - new_rows; + if (!plan.selected_ranges.empty()) { + filtered_plans.push_back(std::move(plan)); + } + } + _row_group_plans = std::move(filtered_plans); + reset(); +} + +void ParquetScanScheduler::reset() { + _next_row_group_plan_idx = 0; + _raw_rows_read = 0; + reset_current_row_group(); +} + +void ParquetScanScheduler::reset_current_row_group() { + _current_row_group.reset(); + _current_predicate_columns.clear(); + _current_non_predicate_columns.clear(); + _current_dictionary_filters.clear(); + _current_dictionary_residual_conjuncts.clear(); + _current_row_group_rows = 0; + _current_row_group_id = -1; + _current_row_group_rows_read = 0; + _current_row_group_first_row = 0; + _current_selected_ranges.clear(); + _current_range_idx = 0; + _current_range_rows_read = 0; + _current_predicate_prefetched = false; + _current_non_predicate_prefetched = false; + _current_merge_range_active = false; +} + +Status ParquetScanScheduler::open_next_row_group( + ParquetFileContext& file_context, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool* has_row_group) { + *has_row_group = false; + if (_next_row_group_plan_idx >= _row_group_plans.size()) { + return Status::OK(); + } + const RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; + const int row_group_idx = row_group_plan.row_group_id; + // Row-level dictionary filters read dictionary pages before Arrow RecordReaders are created. + // Keep that probe on the base reader: MergeRangeFileReader expects each registered range to be + // consumed as one forward pass, while the later RecordReader opens the same column chunk again + // for the data-page stream. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; + try { + _current_row_group = file_context.file_reader->RowGroup(row_group_idx); + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to open parquet row group {}: {}", row_group_idx, + e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to open parquet row group {}: {}", row_group_idx, + e.what()); + } + + auto row_group_metadata = file_context.metadata->RowGroup(row_group_idx); + DORIS_CHECK(row_group_metadata != nullptr); + _current_row_group_rows = row_group_metadata->num_rows(); + DORIS_CHECK(_current_row_group_rows == row_group_plan.row_group_rows); + DORIS_CHECK(_current_row_group_rows > 0); + _current_row_group_id = row_group_idx; + DORIS_CHECK(!row_group_plan.selected_ranges.empty()); + _current_row_group_first_row = row_group_plan.first_file_row; + _current_row_group_rows_read = 0; + _current_selected_ranges = row_group_plan.selected_ranges; + _current_range_idx = 0; + _current_range_rows_read = 0; + _current_predicate_columns.clear(); + _current_non_predicate_columns.clear(); + _current_dictionary_filters.clear(); + RETURN_IF_ERROR(prepare_current_dictionary_filters(file_context, file_schema, request, + row_group_idx, *row_group_metadata)); + _current_merge_range_active = + prepare_current_row_group_reader(file_context, file_schema, request, row_group_idx); + + ParquetColumnReaderFactory column_reader_factory( + _current_row_group, file_context.schema->num_columns(), &row_group_plan.page_skip_plans, + _page_skip_profile, _timezone, _enable_strict_mode, + _scan_profile.column_reader_profile); + for (const auto& col : request.predicate_columns) { + const auto local_id = col.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID) { + _current_predicate_columns[local_id] = + column_reader_factory.create_row_position_column_reader( + _current_row_group_first_row); + continue; + } + if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + DORIS_CHECK(_global_rowid_context.has_value()); + _current_predicate_columns[local_id] = + column_reader_factory.create_global_rowid_column_reader( + *_global_rowid_context, _current_row_group_first_row); + continue; + } + + DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(column_schema != nullptr); + std::unique_ptr column_reader; + RETURN_IF_ERROR( + column_reader_factory.create(*column_schema, &col, &column_reader, + _current_dictionary_filters.contains(local_id))); + _current_predicate_columns[local_id] = std::move(column_reader); + } + // Start warming filter-column chunks as soon as their row group is selected. Parquet v2 still + // reads through Arrow's random-access reader; this prefetch only warms Doris file cache blocks + // in the background and never changes the row/column materialization order. + if (!_current_merge_range_active) { + prefetch_current_row_group_columns(file_context, file_schema, request.predicate_columns, + &_current_predicate_prefetched); + } + for (const auto& col : request.non_predicate_columns) { + const auto local_id = col.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID) { + _current_non_predicate_columns[local_id] = + column_reader_factory.create_row_position_column_reader( + _current_row_group_first_row); + continue; + } + if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + DORIS_CHECK(_global_rowid_context.has_value()); + _current_non_predicate_columns[local_id] = + column_reader_factory.create_global_rowid_column_reader( + *_global_rowid_context, _current_row_group_first_row); + continue; + } + DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(column_schema != nullptr); + std::unique_ptr column_reader; + RETURN_IF_ERROR(column_reader_factory.create(*column_schema, &col, &column_reader)); + _current_non_predicate_columns[local_id] = std::move(column_reader); + } + if (!_current_merge_range_active && request.conjuncts.empty() && + request.delete_conjuncts.empty()) { + // With no row-level filters there is no lazy-read decision to wait for, so start warming + // output chunks immediately after their readers are created. Filtered scans still defer + // this until at least one row survives the predicate phase. + prefetch_current_row_group_columns(file_context, file_schema, request.non_predicate_columns, + &_current_non_predicate_prefetched); + } + *has_row_group = true; + return Status::OK(); +} + +Status ParquetScanScheduler::skip_current_row_group_rows(int64_t rows) { + DORIS_CHECK(rows >= 0); + if (rows == 0) { + return Status::OK(); + } + if (_scan_profile.range_gap_skipped_rows != nullptr) { + COUNTER_UPDATE(_scan_profile.range_gap_skipped_rows, rows); + } + for (const auto& column_reader : _current_predicate_columns | std::views::values) { + RETURN_IF_ERROR(column_reader->skip(rows)); + } + for (const auto& column_reader : _current_non_predicate_columns | std::views::values) { + RETURN_IF_ERROR(column_reader->skip(rows)); + } + _current_row_group_rows_read += rows; + return Status::OK(); +} + +namespace { + +struct PredicateConjunctSchedule { + std::map single_column_conjuncts; + VExprContextSPtrs remaining_conjuncts; +}; + +PredicateConjunctSchedule build_predicate_conjunct_schedule( + const format::FileScanRequest& request) { + std::unordered_set predicate_block_positions; + predicate_block_positions.reserve(request.predicate_columns.size()); + for (const auto& col : request.predicate_columns) { + const auto position_it = request.local_positions.find(col.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + predicate_block_positions.insert(position_it->second.value()); + } + + PredicateConjunctSchedule schedule; + for (const auto& conjunct : request.conjuncts) { + DORIS_CHECK(conjunct != nullptr); + DORIS_CHECK(conjunct->root() != nullptr); + if (!conjunct->root()->is_safe_to_execute_on_selected_rows()) { + // Round-by-round filtering can compact later predicate columns before evaluating + // remaining expressions. Stateful functions such as random(1) and error-preserving + // functions such as assert_true() must see the same full batch they saw before this + // optimization, so any unsafe conjunct disables the per-column schedule for the batch. + schedule.remaining_conjuncts = request.conjuncts; + schedule.single_column_conjuncts.clear(); + return schedule; + } + std::set referenced_positions; + conjunct->root()->collect_slot_column_ids(referenced_positions); + if (referenced_positions.size() != 1) { + schedule.remaining_conjuncts.push_back(conjunct); + continue; + } + const auto block_position = static_cast(*referenced_positions.begin()); + if (!predicate_block_positions.contains(block_position)) { + schedule.remaining_conjuncts.push_back(conjunct); + continue; + } + schedule.single_column_conjuncts[block_position].push_back(conjunct); + } + return schedule; +} + +bool can_evaluate_all_with_dictionary(const VExprContextSPtrs& conjuncts) { + if (conjuncts.empty()) { + return false; + } + return std::ranges::all_of(conjuncts, [](const auto& conjunct) { + return conjunct != nullptr && conjunct->root() != nullptr && + conjunct->root()->can_evaluate_dictionary_filter(); + }); +} + +bool can_evaluate_dictionary_exactly(const VExprSPtr& expr) { + DORIS_CHECK(expr != nullptr); + const auto* compound_pred = dynamic_cast(expr.get()); + if (compound_pred == nullptr) { + return expr->can_evaluate_dictionary_filter(); + } + if (compound_pred->op() != TExprOpcode::COMPOUND_AND && + compound_pred->op() != TExprOpcode::COMPOUND_OR) { + return false; + } + return !expr->children().empty() && + std::ranges::all_of(expr->children(), [](const auto& child) { + return can_evaluate_dictionary_exactly(child); + }); +} + +void collect_dictionary_residual_exprs(const VExprContextSPtr& owner_context, const VExprSPtr& expr, + DictionaryResidualConjuncts* residual_conjuncts) { + DORIS_CHECK(owner_context != nullptr); + DORIS_CHECK(expr != nullptr); + DORIS_CHECK(residual_conjuncts != nullptr); + + if (can_evaluate_dictionary_exactly(expr)) { + return; + } + + // VCompoundPred dictionary evaluation is a conservative prefilter for AND when only some + // children are dictionary-aware. Split AND so exact dictionary children are not executed again + // on materialized rows. Do not split a non-exact OR: its branches cannot be evaluated + // independently after a dictionary prefilter without changing the original boolean semantics. + const auto* compound_pred = dynamic_cast(expr.get()); + if (compound_pred != nullptr && compound_pred->op() == TExprOpcode::COMPOUND_AND) { + for (const auto& child : expr->children()) { + collect_dictionary_residual_exprs(owner_context, child, residual_conjuncts); + } + return; + } + + residual_conjuncts->emplace_back(owner_context, expr); +} + +DictionaryResidualConjuncts build_dictionary_residual_conjuncts( + const VExprContextSPtrs& conjuncts) { + DictionaryResidualConjuncts residual_conjuncts; + for (const auto& conjunct : conjuncts) { + DORIS_CHECK(conjunct != nullptr); + collect_dictionary_residual_exprs(conjunct, conjunct->root(), &residual_conjuncts); + } + return residual_conjuncts; +} + +uint16_t count_selected_rows(const IColumn::Filter& filter) { + uint16_t selected_rows = 0; + for (const auto value : filter) { + selected_rows += value != 0; + } + return selected_rows; +} + +Status filter_read_predicate_columns(Block* file_block, const std::vector& positions, + const IColumn::Filter& compact_filter) { + if (positions.empty()) { + return Status::OK(); + } + RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, positions, compact_filter)); + return Status::OK(); +} + +IColumn::Filter build_dictionary_entry_filter(size_t block_position, + const ParquetColumnSchema& column_schema, + const VExprContextSPtrs& conjuncts, + const ParquetDictionaryWords& dict_words) { + auto fields = dictionary_fields_from_words(dict_words); + IColumn::Filter dictionary_filter(fields.size(), 1); + DictionaryEvalContext ctx; + auto& slot = ctx.slots + .emplace(static_cast(block_position), + DictionaryEvalContext::SlotDictionary { + .data_type = column_schema.type, .values = {}}) + .first->second; + slot.values.reserve(1); + + for (size_t dict_idx = 0; dict_idx < fields.size(); ++dict_idx) { + slot.values.clear(); + slot.values.push_back(fields[dict_idx]); + dictionary_filter[dict_idx] = VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == + ZoneMapFilterResult::kNoMatch + ? 0 + : 1; + } + return dictionary_filter; +} + +} // namespace + +Status ParquetScanScheduler::prepare_current_dictionary_filters( + ParquetFileContext& file_context, + const std::vector>& file_schema, + const format::FileScanRequest& request, int row_group_idx, + const ::parquet::RowGroupMetaData& row_group_metadata) { + _current_dictionary_filters.clear(); + _current_dictionary_residual_conjuncts.clear(); + if (request.conjuncts.empty()) { + return Status::OK(); + } + PredicateConjunctSchedule schedule; + { + SCOPED_TIMER(_scan_profile.dict_filter_expr_rewrite_time); + schedule = build_predicate_conjunct_schedule(request); + } + if (schedule.single_column_conjuncts.empty()) { + return Status::OK(); + } + + SCOPED_TIMER(_scan_profile.dict_filter_rewrite_time); + for (const auto& col : request.predicate_columns) { + const auto local_id = col.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + continue; + } + const auto position_it = request.local_positions.find(col.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const auto block_position = static_cast(position_it->second.value()); + const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); + if (conjunct_it == schedule.single_column_conjuncts.end() || + !can_evaluate_all_with_dictionary(conjunct_it->second)) { + continue; + } + update_counter_if_not_null(_scan_profile.dict_filter_candidate_columns, 1); + + // This optimization is deliberately limited to single-column predicates with a dictionary + // evaluable part. Mixed AND predicates are split so dictionary-covered children run as a + // dict-id prefilter and residual children keep the normal row-level expression path. + const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(column_schema != nullptr); + if (column_schema->leaf_column_id < 0 || + column_schema->leaf_column_id >= row_group_metadata.num_columns()) { + update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); + continue; + } + auto column_chunk = row_group_metadata.ColumnChunk(column_schema->leaf_column_id); + if (column_chunk == nullptr || + !supports_row_level_dictionary_filter(*column_schema, *column_chunk)) { + update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); + continue; + } + + ParquetDictionaryWords dict_words; + { + SCOPED_TIMER(_scan_profile.dict_filter_read_dict_time); + if (!read_dictionary_words(file_context.file_reader.get(), row_group_idx, + column_schema->leaf_column_id, *column_schema, + &dict_words)) { + update_counter_if_not_null(_scan_profile.dict_filter_read_failures, 1); + continue; + } + } + + // Build a safe dictionary prefilter from the dictionary-filter interface instead of + // executing the row expression on a temporary dictionary block. For compound AND, + // VCompoundPred intentionally evaluates only dictionary-capable children, so residual + // predicates still run later on surviving rows. + IColumn::Filter dictionary_filter; + DictionaryResidualConjuncts residual_conjuncts; + { + SCOPED_TIMER(_scan_profile.dict_filter_build_time); + dictionary_filter = build_dictionary_entry_filter(block_position, *column_schema, + conjunct_it->second, dict_words); + residual_conjuncts = build_dictionary_residual_conjuncts(conjunct_it->second); + } + + // The bitmap is keyed by Parquet dictionary id. Later data-page reads evaluate the + // predicate with an integer lookup and only materialize STRING values for surviving rows. + _current_dictionary_filters.emplace(local_id, std::move(dictionary_filter)); + _current_dictionary_residual_conjuncts.emplace(local_id, std::move(residual_conjuncts)); + update_counter_if_not_null(_scan_profile.dict_filter_columns, 1); + } + return Status::OK(); +} + +Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, + const format::FileScanRequest& request, + Block* file_block, SelectionVector* selection, + uint16_t* selected_rows, + int64_t* conjunct_filtered_rows, + bool* predicate_columns_filtered) { + DORIS_CHECK(predicate_columns_filtered != nullptr); + *predicate_columns_filtered = false; + if (!request.conjuncts.empty() || !request.delete_conjuncts.empty()) { + selection->resize(static_cast(batch_rows)); + } + const auto schedule = build_predicate_conjunct_schedule(request); + const bool can_read_predicate_columns_round_by_round = + !schedule.single_column_conjuncts.empty(); + std::vector read_column_positions; + read_column_positions.reserve(request.predicate_columns.size()); + + auto read_predicate_column = [&](ParquetColumnReader* column_reader, size_t block_position, + ColumnId local_id, bool* used_dictionary_filter) -> Status { + DORIS_CHECK(used_dictionary_filter != nullptr); + *used_dictionary_filter = false; + DCHECK(remove_nullable(column_reader->type()) + ->equals(*remove_nullable(file_block->get_by_position(block_position).type))) + << column_reader->type()->get_name() << " " + << file_block->get_by_position(block_position).type->get_name() << " " + << column_reader->name() << " " << file_block->get_by_position(block_position).name; + auto column = file_block->get_by_position(block_position).column->assert_mutable(); + SCOPED_TIMER(_scan_profile.column_read_time); + const auto dictionary_filter_it = _current_dictionary_filters.find(local_id); + if (dictionary_filter_it != _current_dictionary_filters.end()) { + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool used_filter = false; + RETURN_IF_ERROR(column_reader->select_with_dictionary_filter( + *selection, *selected_rows, batch_rows, dictionary_filter_it->second, column, + &compact_filter, &used_filter)); + if (used_filter) { + DORIS_CHECK(compact_filter.size() == selected_rows_before); + const uint16_t new_selected_rows = count_selected_rows(compact_filter); + const auto filtered_rows = static_cast(selected_rows_before) - + static_cast(new_selected_rows); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += filtered_rows; + } + update_counter_if_not_null(_scan_profile.rows_filtered_by_dict_filter, + filtered_rows); + if (new_selected_rows != selected_rows_before) { + // The dictionary reader has already appended only surviving values for the + // current column. Apply the compact row filter only to columns read before this + // one, then update the shared selection for later predicate/lazy columns. + RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, + compact_filter)); + *selected_rows = apply_compact_filter_to_selection(compact_filter, selection, + selected_rows_before); + *predicate_columns_filtered = true; + } + file_block->replace_by_position(block_position, std::move(column)); + read_column_positions.push_back(cast_set(block_position)); + *used_dictionary_filter = true; + return Status::OK(); + } + } + + if (*selected_rows == batch_rows) { + int64_t column_rows = 0; + RETURN_IF_ERROR(column_reader->read(batch_rows, column, &column_rows)); + if (column_rows != batch_rows) { + return Status::Corruption( + "Parquet filter column {} returned {} rows, expected {} rows", + column_reader->name(), column_rows, batch_rows); + } + } else { + [[maybe_unused]] auto old_size = column->size(); + RETURN_IF_ERROR(column_reader->select(*selection, *selected_rows, batch_rows, column)); + if (column->size() != old_size + *selected_rows) { + return Status::Corruption( + "Parquet selected filter column {} returned {} rows, expected {} rows", + column_reader->name(), column->size(), old_size + *selected_rows); + } + *predicate_columns_filtered = true; + } + file_block->replace_by_position(block_position, std::move(column)); + read_column_positions.push_back(cast_set(block_position)); + return Status::OK(); + }; + + auto execute_scheduled_conjuncts = [&](const VExprContextSPtrs& conjuncts) -> Status { + if (conjuncts.empty() || *selected_rows == 0) { + return Status::OK(); + } + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool can_filter_all = false; + RETURN_IF_ERROR(execute_compact_filter_conjuncts( + conjuncts, selected_rows_before, file_block, &compact_filter, &can_filter_all)); + if (can_filter_all) { + compact_filter.resize_fill(selected_rows_before, 0); + } + const uint16_t new_selected_rows = can_filter_all ? 0 : count_selected_rows(compact_filter); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += static_cast(selected_rows_before) - + static_cast(new_selected_rows); + } + if (new_selected_rows != selected_rows_before) { + // All columns read so far are already compacted to the current selection. Apply the + // compact filter to those columns and the selection vector together, so later predicate + // columns can read only rows that survived previous predicate rounds. + RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, + compact_filter)); + *selected_rows = can_filter_all + ? 0 + : apply_compact_filter_to_selection(compact_filter, selection, + selected_rows_before); + *predicate_columns_filtered = true; + } + return Status::OK(); + }; + + auto execute_scheduled_dictionary_residual_conjuncts = + [&](const DictionaryResidualConjuncts& conjuncts) -> Status { + if (conjuncts.empty() || *selected_rows == 0) { + return Status::OK(); + } + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool can_filter_all = false; + RETURN_IF_ERROR(execute_compact_dictionary_residual_conjuncts( + conjuncts, selected_rows_before, file_block, &compact_filter, &can_filter_all)); + if (can_filter_all) { + compact_filter.resize_fill(selected_rows_before, 0); + } + const uint16_t new_selected_rows = can_filter_all ? 0 : count_selected_rows(compact_filter); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += static_cast(selected_rows_before) - + static_cast(new_selected_rows); + } + if (new_selected_rows != selected_rows_before) { + // Dictionary-covered children have already reduced the compact block. Apply only the + // residual child filters here, then keep the same compacted-column invariant as the + // normal conjunct path for later predicate rounds. + RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, + compact_filter)); + *selected_rows = can_filter_all + ? 0 + : apply_compact_filter_to_selection(compact_filter, selection, + selected_rows_before); + *predicate_columns_filtered = true; + } + return Status::OK(); + }; + + auto execute_scheduled_conjuncts_with_profile = + [&](const VExprContextSPtrs& conjuncts) -> Status { + if (_scan_profile.predicate_filter_time == nullptr) { + return execute_scheduled_conjuncts(conjuncts); + } + SCOPED_TIMER(_scan_profile.predicate_filter_time); + return execute_scheduled_conjuncts(conjuncts); + }; + + auto execute_scheduled_dictionary_residual_conjuncts_with_profile = + [&](const DictionaryResidualConjuncts& conjuncts) -> Status { + if (_scan_profile.predicate_filter_time == nullptr) { + return execute_scheduled_dictionary_residual_conjuncts(conjuncts); + } + SCOPED_TIMER(_scan_profile.predicate_filter_time); + return execute_scheduled_dictionary_residual_conjuncts(conjuncts); + }; + + auto execute_scheduled_delete_conjuncts = [&]() -> Status { + if (request.delete_conjuncts.empty() || *selected_rows == 0) { + return Status::OK(); + } + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool can_filter_all = false; + RETURN_IF_ERROR(execute_compact_delete_conjuncts(request.delete_conjuncts, + selected_rows_before, file_block, + &compact_filter, &can_filter_all)); + if (can_filter_all) { + compact_filter.resize_fill(selected_rows_before, 0); + } + if (can_filter_all || count_selected_rows(compact_filter) != selected_rows_before) { + RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, + compact_filter)); + *selected_rows = can_filter_all + ? 0 + : apply_compact_filter_to_selection(compact_filter, selection, + selected_rows_before); + *predicate_columns_filtered = true; + } + return Status::OK(); + }; + + auto read_all_predicate_columns = [&]() -> Status { + for (const auto& [fid, column_reader] : _current_predicate_columns) { + auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + DORIS_CHECK(position_it != request.local_positions.end()); + bool used_dictionary_filter = false; + RETURN_IF_ERROR(read_predicate_column(column_reader.get(), position_it->second.value(), + fid, &used_dictionary_filter)); + } + return Status::OK(); + }; + + if (!can_read_predicate_columns_round_by_round) { + RETURN_IF_ERROR(read_all_predicate_columns()); + if (_scan_profile.predicate_filter_time == nullptr) { + return execute_batch_filters(request, batch_rows, file_block, selection, selected_rows, + conjunct_filtered_rows); + } + SCOPED_TIMER(_scan_profile.predicate_filter_time); + return execute_batch_filters(request, batch_rows, file_block, selection, selected_rows, + conjunct_filtered_rows); + } + + auto read_round_by_round = [&]() -> Status { + // Single-column conjuncts can be evaluated immediately after their column is read. Once + // selection shrinks, later predicate columns use ParquetColumnReader::select() so the + // reader skips rows already rejected by earlier predicates instead of materializing them. + for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + const auto& col = request.predicate_columns[idx]; + const auto fid = col.local_id(); + auto reader_it = _current_predicate_columns.find(fid); + DORIS_CHECK(reader_it != _current_predicate_columns.end()); + auto position_it = request.local_positions.find(col.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const auto block_position = position_it->second.value(); + bool used_dictionary_filter = false; + RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, + &used_dictionary_filter)); + if (*selected_rows == 0) { + for (size_t remaining_idx = idx + 1; + remaining_idx < request.predicate_columns.size(); ++remaining_idx) { + const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); + auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); + DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); + RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); + } + return Status::OK(); + } + const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); + if (conjunct_it == schedule.single_column_conjuncts.end()) { + continue; + } + if (used_dictionary_filter) { + const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); + DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); + RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( + residual_it->second)); + } else { + RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); + } + if (*selected_rows != 0) { + continue; + } + for (size_t remaining_idx = idx + 1; remaining_idx < request.predicate_columns.size(); + ++remaining_idx) { + const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); + auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); + DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); + RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); + } + return Status::OK(); + } + return Status::OK(); + }; + + RETURN_IF_ERROR(read_round_by_round()); + RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(schedule.remaining_conjuncts)); + if (_scan_profile.predicate_filter_time == nullptr) { + return execute_scheduled_delete_conjuncts(); + } + SCOPED_TIMER(_scan_profile.predicate_filter_time); + return execute_scheduled_delete_conjuncts(); +} + +bool ParquetScanScheduler::prepare_current_row_group_reader( + ParquetFileContext& file_context, + const std::vector>& file_schema, + const format::FileScanRequest& request, int row_group_idx) { + if (file_context.metadata == nullptr) { + return false; + } + const auto ranges = build_row_group_prefetch_ranges( + *file_context.metadata, file_schema, request_scan_columns(request), row_group_idx); + const size_t avg_io_size = detail::average_prefetch_range_size(ranges); + return file_context.set_random_access_ranges(ranges, avg_io_size, _profile, + _merge_read_slice_size); +} + +void ParquetScanScheduler::prefetch_current_row_group_columns( + ParquetFileContext& file_context, + const std::vector>& file_schema, + const std::vector& scan_columns, bool* prefetched) { + DORIS_CHECK(prefetched != nullptr); + if (_current_merge_range_active || *prefetched || scan_columns.empty() || + _current_row_group_id < 0 || file_context.metadata == nullptr) { + return; + } + *prefetched = true; + // The scanner request separates predicate and non-predicate columns so Parquet can read + // predicate columns first and lazily materialize the rest. Keep the same contract for + // prefetch: callers decide which side to warm, and this helper only translates that selected + // projection into physical column-chunk byte ranges for the current row group. + file_context.prefetch_ranges( + build_row_group_prefetch_ranges(*file_context.metadata, file_schema, scan_columns, + _current_row_group_id), + nullptr); +} + +Status ParquetScanScheduler::read_current_row_group_batch( + ParquetFileContext& file_context, + const std::vector>& file_schema, int64_t batch_rows, + const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, + size_t* rows) { + if (_scan_profile.total_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.total_batches, 1); + } + if (_scan_profile.raw_rows_read != nullptr) { + COUNTER_UPDATE(_scan_profile.raw_rows_read, batch_rows); + } + _raw_rows_read += batch_rows; + if (_current_predicate_columns.empty() && _current_non_predicate_columns.empty()) { + *rows = static_cast(batch_rows); + if (_scan_profile.selected_rows != nullptr) { + COUNTER_UPDATE(_scan_profile.selected_rows, batch_rows); + } + return Status::OK(); + } + SelectionVector selection; + DORIS_CHECK(batch_rows <= std::numeric_limits::max()); + uint16_t selected_rows = static_cast(batch_rows); + int64_t conjunct_filtered_rows = 0; + bool predicate_columns_filtered = false; + RETURN_IF_ERROR(read_filter_columns(batch_rows, request, file_block, &selection, &selected_rows, + &conjunct_filtered_rows, &predicate_columns_filtered)); + _predicate_filtered_rows += conjunct_filtered_rows; + mark_condition_cache_granules(selection, selected_rows, batch_first_file_row); + + const bool need_filter_output = selected_rows != batch_rows; + if (_scan_profile.selected_rows != nullptr) { + COUNTER_UPDATE(_scan_profile.selected_rows, selected_rows); + } + if (_scan_profile.rows_filtered_by_conjunct != nullptr) { + COUNTER_UPDATE(_scan_profile.rows_filtered_by_conjunct, conjunct_filtered_rows); + } + if (!_current_non_predicate_columns.empty() && + _scan_profile.lazy_read_filtered_rows != nullptr) { + COUNTER_UPDATE(_scan_profile.lazy_read_filtered_rows, batch_rows - selected_rows); + } + if (selected_rows == 0 && _scan_profile.empty_selection_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.empty_selection_batches, 1); + } + if (need_filter_output && !predicate_columns_filtered) { + IColumn::Filter output_filter = selection_to_filter(selection, selected_rows, batch_rows); + for (const auto& col : request.predicate_columns) { + auto position_it = request.local_positions.find(col.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const auto block_position = position_it->second.value(); + RETURN_IF_CATCH_EXCEPTION(file_block->replace_by_position( + block_position, file_block->get_by_position(block_position) + .column->filter(output_filter, selected_rows))); + } + } + if (!_current_merge_range_active && selected_rows > 0 && + !_current_non_predicate_columns.empty()) { + // Do not prefetch lazy output columns until at least one row survives filtering. This is + // the same decision point where the v2 reader switches from predicate-only reads to + // materializing non-predicate columns, so fully filtered batches avoid unnecessary IO. + prefetch_current_row_group_columns(file_context, file_schema, request.non_predicate_columns, + &_current_non_predicate_prefetched); + } + + { + SCOPED_TIMER(_scan_profile.column_read_time); + for (const auto& [fid, column_reader] : _current_non_predicate_columns) { + auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + DORIS_CHECK(position_it != request.local_positions.end()); + const auto block_position = position_it->second.value(); + auto column = file_block->get_by_position(block_position).column->assert_mutable(); + DCHECK_EQ(file_block->get_by_position(block_position).type->get_primitive_type(), + column_reader->type()->get_primitive_type()) + << type_to_string(file_block->get_by_position(block_position) + .type->get_primitive_type()) + << " " << type_to_string(column_reader->type()->get_primitive_type()) << " " + << column_reader->name() << " " << fid << " " << block_position; + if (need_filter_output) { + [[maybe_unused]] auto old_size = column->size(); + RETURN_IF_ERROR( + column_reader->select(selection, selected_rows, batch_rows, column)); + if (column->size() != old_size + selected_rows) { + return Status::Corruption( + "Parquet selected output column {} returned {} rows, expected {} rows", + column_reader->name(), column->size(), old_size + selected_rows); + } + } else { + int64_t column_rows = 0; + RETURN_IF_ERROR(column_reader->read(batch_rows, column, &column_rows)); + if (column_rows != batch_rows) { + return Status::Corruption( + "Parquet output column {} returned {} rows, expected {} rows", + column_reader->name(), column_rows, batch_rows); + } + } + file_block->replace_by_position(block_position, std::move(column)); + } + } + *rows = static_cast(selected_rows); + return Status::OK(); +} + +void ParquetScanScheduler::mark_condition_cache_granules(const SelectionVector& selection, + uint16_t selected_rows, + int64_t batch_first_file_row) { + if (!_condition_cache_ctx || _condition_cache_ctx->is_hit || + !_condition_cache_ctx->filter_result) { + return; + } + auto& cache = *_condition_cache_ctx->filter_result; + for (uint16_t selection_idx = 0; selection_idx < selected_rows; ++selection_idx) { + const int64_t file_row = batch_first_file_row + selection.get_index(selection_idx); + const int64_t granule = file_row / ConditionCacheContext::GRANULE_SIZE; + const int64_t cache_idx = granule - _condition_cache_ctx->base_granule; + if (cache_idx >= 0 && static_cast(cache_idx) < cache.size()) { + cache[static_cast(cache_idx)] = true; + } + } +} + +Status ParquetScanScheduler::read_next_batch( + ParquetFileContext& file_context, + const std::vector>& file_schema, + const format::FileScanRequest& request, Block* file_block, size_t* rows, bool* eof) { + *rows = 0; + while (true) { + if (_current_row_group == nullptr) { + bool has_row_group = false; + RETURN_IF_ERROR( + open_next_row_group(file_context, file_schema, request, &has_row_group)); + if (!has_row_group) { + *eof = true; + return Status::OK(); + } + } + + if (_current_range_idx >= _current_selected_ranges.size()) { + // Current row group finished, try next row group. + reset_current_row_group(); + continue; + } + + const RowRange& current_range = _current_selected_ranges[_current_range_idx]; + DORIS_CHECK(current_range.start >= 0); + DORIS_CHECK(current_range.length > 0); + DORIS_CHECK(current_range.start + current_range.length <= _current_row_group_rows); + + if (_current_row_group_rows_read < current_range.start) { + // Skip filtered rows according to row group level pruning. + RETURN_IF_ERROR(skip_current_row_group_rows(current_range.start - + _current_row_group_rows_read)); + } + DORIS_CHECK(_current_row_group_rows_read == current_range.start + _current_range_rows_read); + const int64_t remaining_rows = current_range.length - _current_range_rows_read; + if (remaining_rows <= 0) { + // Current range finished, try next range in the same row group. + ++_current_range_idx; + _current_range_rows_read = 0; + continue; + } + + const int64_t batch_rows = std::min(_batch_size, remaining_rows); + const int64_t physical_rows_read = batch_rows; + const int64_t batch_first_file_row = + _current_row_group_first_row + _current_row_group_rows_read; + RETURN_IF_ERROR(read_current_row_group_batch(file_context, file_schema, batch_rows, request, + batch_first_file_row, file_block, rows)); + _current_row_group_rows_read += physical_rows_read; + _current_range_rows_read += physical_rows_read; + if (_current_range_rows_read >= current_range.length) { + ++_current_range_idx; + _current_range_rows_read = 0; + } + if (*rows == 0) { + continue; + } + *eof = false; + return Status::OK(); + } +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h new file mode 100644 index 00000000000000..3fa7586fb1b428 --- /dev/null +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/selection_vector.h" +#include "runtime/runtime_profile.h" +#include "storage/segment/condition_cache.h" + +namespace parquet { +class FileMetaData; +class ParquetFileReader; +class RowGroupMetaData; +class RowGroupReader; +} // namespace parquet + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { +class Block; +class RuntimeState; + +namespace format { +struct FileScanRequest; +} // namespace format +} // namespace doris + +namespace doris::format::parquet { + +struct ParquetFileContext; +struct ParquetColumnSchema; + +// ============================================================================ +// ============================================================================ + +struct ParquetScanRange { + int64_t start_offset = 0; + int64_t size = -1; // -1 means read the whole file + int64_t file_size = -1; // -1 means unknown +}; + +struct RowGroupReadPlan { + int row_group_id = -1; // row group id + int64_t first_file_row = 0; // first file row for this row group (0-based) + int64_t row_group_rows = 0; // row count of this row group + std::vector selected_ranges; // row ranges to read after page-index pruning + std::map + page_skip_plans; // leaf_column_id -> data pages that can be skipped completely +}; + +struct RowGroupScanPlan { + std::vector row_groups; // row groups selected after pruning + ParquetPruningStats pruning_stats; // pruning statistics +}; + +// ============================================================================ +// ============================================================================ + +Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, + ::parquet::ParquetFileReader* file_reader, + const std::vector>& file_schema, + const format::FileScanRequest& request, + const ParquetScanRange& scan_range, bool enable_bloom_filter, + RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, + const RuntimeState* runtime_state = nullptr); + +IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows); + +uint16_t apply_compact_filter_to_selection(const IColumn::Filter& filter, + SelectionVector* selection, uint16_t selected_rows); + +Status execute_batch_filters(const format::FileScanRequest& request, int64_t batch_rows, + Block* file_block, SelectionVector* selection, uint16_t* selected_rows, + int64_t* conjunct_filtered_rows = nullptr); + +// ============================================================================ +// ============================================================================ +// while true: +// 3. read_current_row_group_batch(batch_rows) +// ============================================================================ +class ParquetScanScheduler { +public: + static constexpr int64_t DEFAULT_READ_BATCH_SIZE = 4096; + + void set_plan(RowGroupScanPlan plan); + void set_page_skip_profile(ParquetPageSkipProfile page_skip_profile) { + _page_skip_profile = page_skip_profile; + } + void set_scan_profile(ParquetScanProfile scan_profile) { _scan_profile = scan_profile; } + void set_merge_read_options(RuntimeProfile* profile, int64_t merge_read_slice_size) { + _profile = profile; + _merge_read_slice_size = merge_read_slice_size; + } + void set_global_rowid_context(std::optional context) { + _global_rowid_context = context; + } + void set_condition_cache_context(std::shared_ptr ctx); + void set_timezone(const cctz::time_zone* timezone) { _timezone = timezone; } + void set_enable_strict_mode(bool enable_strict_mode) { + _enable_strict_mode = enable_strict_mode; + } + // Upper scanner owns adaptive memory feedback; scheduler only applies the current row cap when + // splitting selected row ranges into physical read batches. + void set_batch_size(size_t batch_size) { + _batch_size = batch_size == 0 ? 1 : static_cast(batch_size); + } + void reset(); + bool empty() const { return _row_group_plans.empty(); } + int64_t condition_cache_filtered_rows() const { return _condition_cache_filtered_rows; } + int64_t predicate_filtered_rows() const { return _predicate_filtered_rows; } + int64_t raw_rows_read() const { return _raw_rows_read; } + + Status read_next_batch(ParquetFileContext& file_context, + const std::vector>& file_schema, + const format::FileScanRequest& request, Block* file_block, size_t* rows, + bool* eof); + +private: + void reset_current_row_group(); + + Status open_next_row_group(ParquetFileContext& file_context, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool* has_row_group); + + Status skip_current_row_group_rows(int64_t rows); + + Status read_filter_columns(int64_t batch_rows, const format::FileScanRequest& request, + Block* file_block, SelectionVector* selection, + uint16_t* selected_rows, int64_t* conjunct_filtered_rows, + bool* predicate_columns_filtered); + + Status prepare_current_dictionary_filters( + ParquetFileContext& file_context, + const std::vector>& file_schema, + const format::FileScanRequest& request, int row_group_idx, + const ::parquet::RowGroupMetaData& row_group_metadata); + + void prefetch_current_row_group_columns( + ParquetFileContext& file_context, + const std::vector>& file_schema, + const std::vector& scan_columns, bool* prefetched); + + bool prepare_current_row_group_reader( + ParquetFileContext& file_context, + const std::vector>& file_schema, + const format::FileScanRequest& request, int row_group_idx); + + Status read_current_row_group_batch( + ParquetFileContext& file_context, + const std::vector>& file_schema, + int64_t batch_rows, const format::FileScanRequest& request, + int64_t batch_first_file_row, Block* file_block, size_t* rows); + + void mark_condition_cache_granules(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_first_file_row); + + std::vector _row_group_plans; // row group queue to scan + size_t _next_row_group_plan_idx = 0; // index of the next row group to process + + std::shared_ptr<::parquet::RowGroupReader> _current_row_group; // Arrow RowGroup reader + std::map> + _current_predicate_columns; // predicate ColumnReaders + std::map> + _current_non_predicate_columns; // non-predicate ColumnReaders + std::map + _current_dictionary_filters; // local id -> dict entry bitmap + std::map>> + _current_dictionary_residual_conjuncts; // local id -> row-level residual conjuncts + int64_t _current_row_group_rows = 0; // current row group row count + int _current_row_group_id = -1; // current row group id in parquet metadata + int64_t _current_row_group_rows_read = 0; // rows read in the current row group (cursor) + int64_t _current_row_group_first_row = 0; // first file row of the current row group + std::vector + _current_selected_ranges; // selected ranges for the current row group after page-index pruning + size_t _current_range_idx = 0; // current selected_range index + int64_t _current_range_rows_read = 0; // rows read in the current range + + bool _current_predicate_prefetched = false; + bool _current_non_predicate_prefetched = false; + bool _current_merge_range_active = false; + ParquetPageSkipProfile _page_skip_profile; + ParquetScanProfile _scan_profile; + RuntimeProfile* _profile = nullptr; + int64_t _merge_read_slice_size = -1; + std::optional _global_rowid_context; + const cctz::time_zone* _timezone = nullptr; + bool _enable_strict_mode = false; + int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; + std::shared_ptr _condition_cache_ctx; + int64_t _condition_cache_filtered_rows = 0; + int64_t _predicate_filtered_rows = 0; + int64_t _raw_rows_read = 0; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp new file mode 100644 index 00000000000000..b1eba0cf74fe93 --- /dev/null +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -0,0 +1,1549 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_statistics.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/field.h" +#include "exprs/expr_zonemap_filter.h" +#include "exprs/vexpr_context.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/timestamp_statistics.h" +#include "runtime/runtime_profile.h" +#include "storage/index/zone_map/zone_map_index.h" +#include "storage/index/zone_map/zonemap_eval_context.h" + +namespace doris::format::parquet { + +namespace { + +enum class ParquetRowGroupPruneReason { + NONE, // cannot prune; must read + STATISTICS, // excluded by ZoneMap statistics + DICTIONARY, // excluded by dictionary + BLOOM_FILTER, // excluded by bloom filter +}; + +bool bloom_logical_type_supported(const ParquetColumnSchema& column_schema) { + if (column_schema.type == nullptr) { + return false; + } + switch (remove_nullable(column_schema.type)->get_primitive_type()) { + case TYPE_BOOLEAN: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_FLOAT: + case TYPE_DOUBLE: + case TYPE_STRING: + return true; + default: + return false; + } +} + +DecodedTimeUnit decoded_time_unit(ParquetTimeUnit time_unit) { + switch (time_unit) { + case ParquetTimeUnit::MILLIS: + return DecodedTimeUnit::MILLIS; + case ParquetTimeUnit::MICROS: + return DecodedTimeUnit::MICROS; + case ParquetTimeUnit::NANOS: + return DecodedTimeUnit::NANOS; + default: + return DecodedTimeUnit::UNKNOWN; + } +} + +Status read_decoded_field(const ParquetColumnSchema& column_schema, DecodedColumnView view, + Field* field, const cctz::time_zone* timezone) { + DORIS_CHECK(column_schema.type != nullptr); + DORIS_CHECK(field != nullptr); + constexpr uint8_t not_null = 0; + view.row_count = 1; + view.null_map = ¬_null; + view.time_unit = decoded_time_unit(column_schema.type_descriptor.time_unit); + view.logical_integer_bit_width = column_schema.type_descriptor.integer_bit_width; + view.logical_integer_is_signed = !column_schema.type_descriptor.is_unsigned_integer; + view.decimal_precision = column_schema.type_descriptor.decimal_precision; + view.decimal_scale = column_schema.type_descriptor.decimal_scale; + view.fixed_length = column_schema.type_descriptor.fixed_length; + view.timestamp_is_adjusted_to_utc = column_schema.type_descriptor.timestamp_is_adjusted_to_utc; + view.timezone = timezone; + return column_schema.type->get_serde()->read_field_from_decoded_value(*column_schema.type, + field, view); +} + +template +bool set_decoded_field(const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, + const NativeType& value, Field* field, const cctz::time_zone* timezone) { + DecodedColumnView view; + view.value_kind = value_kind; + view.values = reinterpret_cast(&value); + return read_decoded_field(column_schema, view, field, timezone).ok(); +} + +int64_t floor_timestamp_seconds(int64_t value, ParquetTimeUnit time_unit) { + int64_t units_per_second = 1; + switch (time_unit) { + case ParquetTimeUnit::MILLIS: + units_per_second = 1000; + break; + case ParquetTimeUnit::MICROS: + units_per_second = 1000000; + break; + case ParquetTimeUnit::NANOS: + units_per_second = 1000000000; + break; + default: + DORIS_CHECK(false); + } + return format::floor_epoch_seconds(value, units_per_second); +} + +bool timestamp_min_max_is_safe(const ParquetColumnSchema& column_schema, int64_t min_value, + int64_t max_value, const cctz::time_zone* timezone) { + if (min_value > max_value) { + return false; + } + if (!column_schema.type_descriptor.is_timestamp || + !column_schema.type_descriptor.timestamp_is_adjusted_to_utc || timezone == nullptr || + remove_nullable(column_schema.type)->get_primitive_type() == TYPE_TIMESTAMPTZ) { + // TIMESTAMPTZ keeps the original UTC ordering, so local civil-time rollback does not make + // its converted min/max non-monotonic. + return true; + } + return format::utc_timestamp_range_is_monotonic( + floor_timestamp_seconds(min_value, column_schema.type_descriptor.time_unit), + floor_timestamp_seconds(max_value, column_schema.type_descriptor.time_unit), *timezone); +} + +template +bool valid_min_max(const NativeType& min_value, const NativeType& max_value) { + if constexpr (std::is_floating_point_v) { + // Parquet requires readers to ignore min/max statistics if either bound is NaN. + if (std::isnan(min_value) || std::isnan(max_value)) { + return false; + } + } + return true; +} + +bool decoded_min_max_is_ordered(const ParquetColumnStatistics& column_statistics) { + return !(column_statistics.max_value < column_statistics.min_value); +} + +template +bool set_decoded_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, + const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, + ParquetColumnStatistics* column_statistics, + const cctz::time_zone* timezone) { + auto typed_statistics = + std::static_pointer_cast<::parquet::TypedStatistics>(statistics); + const auto& min_value = typed_statistics->min(); + const auto& max_value = typed_statistics->max(); + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { + return false; + } + } + if (!valid_min_max(min_value, max_value) || + !set_decoded_field(column_schema, value_kind, min_value, &column_statistics->min_value, + timezone) || + !set_decoded_field(column_schema, value_kind, max_value, &column_statistics->max_value, + timezone)) { + return false; + } + return decoded_min_max_is_ordered(*column_statistics); +} + +bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, + const StringRef& value, Field* field, + const cctz::time_zone* timezone) { + std::vector binary_values {value}; + DecodedColumnView view; + view.value_kind = value_kind; + view.binary_values = &binary_values; + return read_decoded_field(column_schema, view, field, timezone).ok(); +} + +bool set_string_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, + const ParquetColumnSchema& column_schema, + ParquetColumnStatistics* column_statistics, + const cctz::time_zone* timezone) { + switch (statistics->physical_type()) { + case ::parquet::Type::BYTE_ARRAY: { + auto typed_statistics = + std::static_pointer_cast<::parquet::TypedStatistics<::parquet::ByteArrayType>>( + statistics); + const auto min = ::parquet::ByteArrayToString(typed_statistics->min()); + const auto max = ::parquet::ByteArrayToString(typed_statistics->max()); + if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, + StringRef(min.data(), min.size()), + &column_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, + StringRef(max.data(), max.size()), + &column_statistics->max_value, timezone)) { + return false; + } + return decoded_min_max_is_ordered(*column_statistics); + } + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { + if (column_schema.descriptor == nullptr || column_schema.descriptor->type_length() <= 0) { + return false; + } + auto typed_statistics = + std::static_pointer_cast<::parquet::TypedStatistics<::parquet::FLBAType>>( + statistics); + const int type_length = column_schema.descriptor->type_length(); + const std::string min(reinterpret_cast(typed_statistics->min().ptr), + type_length); + const std::string max(reinterpret_cast(typed_statistics->max().ptr), + type_length); + if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, + StringRef(min.data(), min.size()), + &column_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, + StringRef(max.data(), max.size()), + &column_statistics->max_value, timezone)) { + return false; + } + return decoded_min_max_is_ordered(*column_statistics); + } + default: + return false; + } +} + +template +T load_predicate_value(const char* data) { + T value; + memcpy(&value, data, sizeof(T)); + return value; +} + +std::optional load_predicate_integral_value(const char* buf, size_t size) { + switch (size) { + case sizeof(int8_t): + return static_cast(load_predicate_value(buf)); + case sizeof(int16_t): + return static_cast(load_predicate_value(buf)); + case sizeof(int32_t): + return static_cast(load_predicate_value(buf)); + case sizeof(int64_t): + return load_predicate_value(buf); + default: + return std::nullopt; + } +} + +bool logical_integer_fits_physical_int32(const ParquetTypeDescriptor& type_descriptor, + int64_t value) { + const int bit_width = + type_descriptor.integer_bit_width > 0 ? type_descriptor.integer_bit_width : 32; + if (type_descriptor.is_unsigned_integer) { + const uint64_t max_value = bit_width >= 32 ? std::numeric_limits::max() + : ((uint64_t {1} << bit_width) - 1); + return value >= 0 && static_cast(value) <= max_value; + } + const int64_t min_value = bit_width >= 32 ? std::numeric_limits::min() + : -(int64_t {1} << (bit_width - 1)); + const int64_t max_value = bit_width >= 32 ? std::numeric_limits::max() + : ((int64_t {1} << (bit_width - 1)) - 1); + return value >= min_value && value <= max_value; +} + +std::optional convert_logical_integer_to_physical_int32( + const ParquetTypeDescriptor& type_descriptor, int64_t value) { + if (!logical_integer_fits_physical_int32(type_descriptor, value)) { + return std::nullopt; + } + if (!type_descriptor.is_unsigned_integer) { + return static_cast(value); + } + const auto unsigned_value = static_cast(value); + int32_t physical_value; + memcpy(&physical_value, &unsigned_value, sizeof(physical_value)); + return physical_value; +} + +class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { +public: + ArrowParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, + const ::parquet::BloomFilter& bloom_filter) + : _column_schema(column_schema), _bloom_filter(bloom_filter) {} + + void add_bytes(const char* buf, size_t size) override { DORIS_CHECK(false); } + + bool test_bytes(const char* buf, size_t size) const override { + if (buf == nullptr) { + return true; + } + // Parquet bloom filters are populated from the physical column carrier, while VExpr + // literals are materialized as Doris logical values. Keep the logical type in + // BloomFilterEvalContext for expression compatibility, and normalize to the Parquet + // physical representation only at this adapter boundary. + switch (_column_schema.type_descriptor.physical_type) { + case ::parquet::Type::BOOLEAN: + return test_boolean(buf, size); + case ::parquet::Type::INT32: + return test_physical_int32(buf, size); + case ::parquet::Type::INT64: + return test_int64(buf, size); + case ::parquet::Type::FLOAT: + return test_float(buf, size); + case ::parquet::Type::DOUBLE: + return test_double(buf, size); + case ::parquet::Type::BYTE_ARRAY: + return test_byte_array(buf, size); + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + return test_fixed_len_byte_array(buf, size); + default: + return true; + } + } + + void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } + bool has_null() const override { return false; } + void add_hash(uint64_t hash) override { DORIS_CHECK(false); } + bool test_hash(uint64_t hash) const override { return _bloom_filter.FindHash(hash); } + +private: + bool test_boolean(const char* buf, size_t size) const { + if (size == sizeof(bool)) { + const int32_t value = load_predicate_value(buf) ? 1 : 0; + return _bloom_filter.FindHash(_bloom_filter.Hash(value)); + } + if (size == sizeof(int32_t)) { + const int32_t value = load_predicate_value(buf); + return _bloom_filter.FindHash(_bloom_filter.Hash(value != 0 ? 1 : 0)); + } + return true; + } + + bool test_physical_int32(const char* buf, size_t size) const { + const auto logical_value = load_predicate_integral_value(buf, size); + if (!logical_value.has_value()) { + return true; + } + const auto physical_value = convert_logical_integer_to_physical_int32( + _column_schema.type_descriptor, *logical_value); + if (!physical_value.has_value()) { + return false; + } + return find_int32(*physical_value); + } + + bool test_int64(const char* buf, size_t size) const { + if (size != sizeof(int64_t)) { + return true; + } + const int64_t value = load_predicate_value(buf); + return _bloom_filter.FindHash(_bloom_filter.Hash(value)); + } + + bool test_float(const char* buf, size_t size) const { + if (size != sizeof(float)) { + return true; + } + const float value = load_predicate_value(buf); + return _bloom_filter.FindHash(_bloom_filter.Hash(value)); + } + + bool test_double(const char* buf, size_t size) const { + if (size != sizeof(double)) { + return true; + } + const double value = load_predicate_value(buf); + return _bloom_filter.FindHash(_bloom_filter.Hash(value)); + } + + bool test_byte_array(const char* buf, size_t size) const { + ::parquet::ByteArray value(static_cast(size), + reinterpret_cast(buf)); + return _bloom_filter.FindHash(_bloom_filter.Hash(&value)); + } + + bool test_fixed_len_byte_array(const char* buf, size_t size) const { + if (_column_schema.type_descriptor.fixed_length <= 0) { + return true; + } + if (size != static_cast(_column_schema.type_descriptor.fixed_length)) { + return false; + } + ::parquet::FLBA value(reinterpret_cast(buf)); + return _bloom_filter.FindHash( + _bloom_filter.Hash(&value, _column_schema.type_descriptor.fixed_length)); + } + + bool find_int32(int32_t value) const { + return _bloom_filter.FindHash(_bloom_filter.Hash(value)); + } + + const ParquetColumnSchema& _column_schema; + const ::parquet::BloomFilter& _bloom_filter; +}; + +bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { + if (!bloom_logical_type_supported(column_schema)) { + return false; + } + switch (column_schema.type_descriptor.physical_type) { + case ::parquet::Type::BOOLEAN: + case ::parquet::Type::INT32: + case ::parquet::Type::INT64: + case ::parquet::Type::FLOAT: + case ::parquet::Type::DOUBLE: + case ::parquet::Type::BYTE_ARRAY: + return true; + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + return column_schema.type_descriptor.is_string_like && + column_schema.type_descriptor.fixed_length > 0; + default: + return false; + } +} + +bool bloom_filter_excludes(const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, + const ::parquet::BloomFilter& bloom_filter) { + if (!bloom_filter_supported(column_schema)) { + return false; + } + ArrowParquetBloomFilterAdapter adapter(column_schema, bloom_filter); + BloomFilterEvalContext ctx; + ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { + .data_type = column_schema.type, + .bloom_filter = &adapter, + }); + return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; +} + +struct RowGroupBloomFilterCache { + using CacheKey = std::pair; + + ::parquet::BloomFilterReader* bloom_filter_reader = nullptr; + std::map> column_bloom_filters; + std::set loaded_columns; + + ::parquet::BloomFilter* get(int row_group_idx, int leaf_column_id, + ParquetPruningStats* pruning_stats) { + if (bloom_filter_reader == nullptr || leaf_column_id < 0) { + return nullptr; + } + const CacheKey cache_key {row_group_idx, leaf_column_id}; + if (loaded_columns.find(cache_key) == loaded_columns.end()) { + loaded_columns.insert(cache_key); + try { + std::shared_ptr<::parquet::RowGroupBloomFilterReader> row_group_reader; + if (pruning_stats != nullptr) { + SCOPED_RAW_TIMER(&pruning_stats->bloom_filter_read_time); + row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); + if (row_group_reader != nullptr) { + column_bloom_filters[cache_key] = + row_group_reader->GetColumnBloomFilter(leaf_column_id); + } + } else { + row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); + if (row_group_reader != nullptr) { + column_bloom_filters[cache_key] = + row_group_reader->GetColumnBloomFilter(leaf_column_id); + } + } + } catch (const ::parquet::ParquetException&) { + return nullptr; + } catch (const std::exception&) { + return nullptr; + } + } + auto it = column_bloom_filters.find(cache_key); + return it == column_bloom_filters.end() ? nullptr : it->second.get(); + } +}; + +bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { + return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || + encoding == ::parquet::Encoding::RLE_DICTIONARY; +} + +bool is_level_encoding(::parquet::Encoding::type encoding) { + return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; +} + +bool is_data_page_type(::parquet::PageType::type page_type) { + return page_type == ::parquet::PageType::DATA_PAGE || + page_type == ::parquet::PageType::DATA_PAGE_V2; +} + +bool is_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { + if (!column_metadata.has_dictionary_page()) { + return false; + } + + const auto& encoding_stats = column_metadata.encoding_stats(); + if (!encoding_stats.empty()) { + bool has_dictionary_data_page = false; + for (const auto& encoding_stat : encoding_stats) { + if (!is_data_page_type(encoding_stat.page_type) || encoding_stat.count <= 0) { + continue; + } + if (!is_dictionary_data_encoding(encoding_stat.encoding)) { + return false; + } + has_dictionary_data_page = true; + } + return has_dictionary_data_page; + } + + bool has_dictionary_encoding = false; + for (const auto encoding : column_metadata.encodings()) { + if (is_dictionary_data_encoding(encoding)) { + has_dictionary_encoding = true; + continue; + } + if (!is_level_encoding(encoding)) { + return false; + } + } + return has_dictionary_encoding; +} + +bool supports_dictionary_pruning(const ParquetColumnSchema& column_schema, + const ::parquet::ColumnChunkMetaData& column_metadata) { + if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || + column_schema.descriptor == nullptr || column_schema.type == nullptr) { + return false; + } + if (!column_schema.type_descriptor.is_string_like) { + return false; + } + if (column_metadata.type() != ::parquet::Type::BYTE_ARRAY && + column_metadata.type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { + return false; + } + return true; +} + +} // namespace + +bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, + int leaf_column_id, const ParquetColumnSchema& column_schema, + ParquetDictionaryWords* dict_words) { + DORIS_CHECK(dict_words != nullptr); + dict_words->clear(); + if (file_reader == nullptr || leaf_column_id < 0) { + return false; + } + + auto row_group_reader = file_reader->RowGroup(row_group_idx); + if (row_group_reader == nullptr) { + return false; + } + auto page_reader = row_group_reader->GetColumnPageReader(leaf_column_id); + if (page_reader == nullptr) { + return false; + } + + std::shared_ptr<::parquet::Page> page; + try { + page = page_reader->NextPage(); + } catch (const ::parquet::ParquetException&) { + return false; + } catch (const std::exception&) { + return false; + } + if (page == nullptr || page->type() != ::parquet::PageType::DICTIONARY_PAGE) { + return false; + } + const auto* dictionary_page = static_cast(page.get()); + if (dictionary_page->encoding() != ::parquet::Encoding::PLAIN && + dictionary_page->encoding() != ::parquet::Encoding::PLAIN_DICTIONARY) { + return false; + } + const int32_t dictionary_length = dictionary_page->num_values(); + if (dictionary_length <= 0) { + return false; + } + const auto* dictionary_data = dictionary_page->data(); + const int dictionary_size = dictionary_page->size(); + + dict_words->values.reserve(static_cast(dictionary_length)); + if (column_schema.descriptor->physical_type() == ::parquet::Type::BYTE_ARRAY) { + auto decoder = ::parquet::MakeTypedDecoder<::parquet::ByteArrayType>( + ::parquet::Encoding::PLAIN, column_schema.descriptor); + decoder->SetData(dictionary_length, dictionary_data, dictionary_size); + std::vector<::parquet::ByteArray> byte_array_values(static_cast(dictionary_length)); + if (decoder->Decode(byte_array_values.data(), dictionary_length) != dictionary_length) { + return false; + } + for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { + dict_words->values.emplace_back( + reinterpret_cast(byte_array_values[dict_idx].ptr), + byte_array_values[dict_idx].len); + } + dict_words->build_refs(); + return true; + } + if (column_schema.descriptor->physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { + const int type_length = column_schema.descriptor->type_length(); + if (type_length <= 0) { + return false; + } + auto decoder = ::parquet::MakeTypedDecoder<::parquet::FLBAType>(::parquet::Encoding::PLAIN, + column_schema.descriptor); + decoder->SetData(dictionary_length, dictionary_data, dictionary_size); + std::vector<::parquet::FixedLenByteArray> flba_values( + static_cast(dictionary_length)); + if (decoder->Decode(flba_values.data(), dictionary_length) != dictionary_length) { + return false; + } + for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { + dict_words->values.emplace_back( + reinterpret_cast(flba_values[dict_idx].ptr), type_length); + } + dict_words->build_refs(); + return true; + } + return false; +} + +std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words) { + std::vector fields; + fields.reserve(dict_words.refs.size()); + for (const auto& ref : dict_words.refs) { + fields.push_back(Field::create_field(String(ref.data, ref.size))); + } + return fields; +} + +namespace { + +const ParquetColumnSchema* resolve_local_leaf_schema( + const std::vector>& schema, + const format::LocalColumnId file_column_id) { + if (!file_column_id.is_valid() || file_column_id.value() >= static_cast(schema.size())) { + return nullptr; + } + const ParquetColumnSchema* column_schema = schema[file_column_id.value()].get(); + if (column_schema == nullptr || column_schema->kind != ParquetColumnSchemaKind::PRIMITIVE || + column_schema->leaf_column_id < 0 || column_schema->max_repetition_level > 0) { + return nullptr; + } + return column_schema; +} + +std::optional file_column_id_by_block_position( + const format::FileScanRequest& request, int block_position) { + for (const auto& [file_column_id, local_index] : request.local_positions) { + if (local_index.value() == block_position) { + return file_column_id; + } + } + return std::nullopt; +} + +bool has_expr_zonemap_filter(const format::FileScanRequest& request, + const RuntimeState* runtime_state) { + if (!expr_zonemap::is_expr_zonemap_filter_enabled(runtime_state)) { + return false; + } + for (const auto& conjunct : request.conjuncts) { + if (conjunct != nullptr && conjunct->root() != nullptr && + conjunct->root()->can_evaluate_zonemap_filter()) { + return true; + } + } + return false; +} + +std::set collect_expr_zonemap_slot_indexes(const VExprContextSPtrs& conjuncts) { + std::set slot_indexes; + for (const auto& conjunct : conjuncts) { + if (conjunct != nullptr && conjunct->root() != nullptr && + conjunct->root()->can_evaluate_zonemap_filter()) { + conjunct->root()->collect_slot_column_ids(slot_indexes); + } + } + return slot_indexes; +} + +template +std::map collect_conjuncts_by_single_slot( + const VExprContextSPtrs& conjuncts, SlotIndexSelector slot_index_selector) { + std::map conjuncts_by_slot; + for (const auto& conjunct : conjuncts) { + const auto slot_index = slot_index_selector(conjunct); + if (slot_index >= 0) { + conjuncts_by_slot[slot_index].push_back(conjunct); + } + } + return conjuncts_by_slot; +} + +std::shared_ptr make_zonemap_from_statistics( + const ParquetColumnStatistics& statistics) { + if (!statistics.has_null_count && !statistics.has_min_max) { + return nullptr; + } + segment_v2::ZoneMap zone_map; + zone_map.has_null = statistics.has_null; + zone_map.has_not_null = statistics.has_not_null; + if (!statistics.has_not_null) { + return std::make_shared(std::move(zone_map)); + } + if (!statistics.has_min_max) { + // Null counts remain trustworthy when min/max decoding fails (for example, because a + // floating-point bound is NaN). pass_all prevents range pruning without discarding the + // has_null/has_not_null flags needed by IS NULL and IS NOT NULL predicates. + zone_map.pass_all = true; + return std::make_shared(std::move(zone_map)); + } + zone_map.min_value = statistics.min_value; + zone_map.max_value = statistics.max_value; + return std::make_shared(std::move(zone_map)); +} + +void add_slot_zonemap(ZoneMapEvalContext* ctx, int slot_index, const DataTypePtr& data_type, + std::shared_ptr zone_map) { + DORIS_CHECK(ctx != nullptr); + ZoneMapEvalContext::SlotZoneMap slot_zone_map; + slot_zone_map.data_type = data_type; + slot_zone_map.zone_map = std::move(zone_map); + ctx->slots.emplace(slot_index, std::move(slot_zone_map)); +} + +void accumulate_zonemap_stats(const ZoneMapEvalContext& ctx, ParquetPruningStats* pruning_stats) { + if (pruning_stats == nullptr) { + return; + } + pruning_stats->expr_zonemap_unusable_evals += ctx.stats.unusable_zonemap_eval_count; + pruning_stats->in_zonemap_point_check_count += ctx.stats.in_zonemap_point_check_count; + pruning_stats->in_zonemap_range_only_count += ctx.stats.in_zonemap_range_only_count; +} + +} // namespace + +std::shared_ptr ParquetStatisticsUtils::MakeZoneMap( + const ParquetColumnStatistics& statistics) { + return make_zonemap_from_statistics(statistics); +} + +ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( + const ParquetColumnSchema& column_schema, + const std::shared_ptr<::parquet::Statistics>& statistics, const cctz::time_zone* timezone) { + ParquetColumnStatistics result; + if (statistics == nullptr) { + return result; + } + + result.has_null = !statistics->HasNullCount() || statistics->null_count() > 0; + result.has_not_null = statistics->num_values() > 0 || statistics->HasMinMax(); + result.has_null_count = statistics->HasNullCount(); + if (!result.has_not_null || !statistics->HasMinMax()) { + return result; + } + + DORIS_CHECK(column_schema.type != nullptr); + switch (statistics->physical_type()) { + case ::parquet::Type::BOOLEAN: + result.has_min_max = set_decoded_min_max<::parquet::BooleanType>( + statistics, column_schema, DecodedValueKind::BOOL, &result, timezone); + return result; + case ::parquet::Type::INT32: + result.has_min_max = set_decoded_min_max<::parquet::Int32Type>( + statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), + &result, timezone); + return result; + case ::parquet::Type::INT64: + result.has_min_max = set_decoded_min_max<::parquet::Int64Type>( + statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), + &result, timezone); + return result; + case ::parquet::Type::FLOAT: + result.has_min_max = set_decoded_min_max<::parquet::FloatType>( + statistics, column_schema, DecodedValueKind::FLOAT, &result, timezone); + return result; + case ::parquet::Type::DOUBLE: + result.has_min_max = set_decoded_min_max<::parquet::DoubleType>( + statistics, column_schema, DecodedValueKind::DOUBLE, &result, timezone); + return result; + case ::parquet::Type::BYTE_ARRAY: + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + result.has_min_max = set_string_min_max(statistics, column_schema, &result, timezone); + return result; + default: + return result; + } +} + +bool ParquetStatisticsUtils::BloomFilterExcludes(const ParquetColumnSchema& column_schema, + int slot_index, const VExprContextSPtrs& conjuncts, + const ::parquet::BloomFilter& bloom_filter) { + return bloom_filter_excludes(column_schema, slot_index, conjuncts, bloom_filter); +} + +namespace { + +ParquetRowGroupPruneReason dictionary_prune_reason( + const ::parquet::RowGroupMetaData& row_group, ::parquet::ParquetFileReader* file_reader, + int row_group_idx, const std::vector>& file_schema, + const format::FileScanRequest& request) { + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_dictionary_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr) { + continue; + } + DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); + auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); + if (column_chunk == nullptr || + !supports_dictionary_pruning(*column_schema, *column_chunk) || + !is_dictionary_encoded_chunk(*column_chunk)) { + continue; + } + + ParquetDictionaryWords dict_words; + if (!read_dictionary_words(file_reader, row_group_idx, column_schema->leaf_column_id, + *column_schema, &dict_words)) { + continue; + } + DictionaryEvalContext ctx; + ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { + .data_type = column_schema->type, + .values = dictionary_fields_from_words(dict_words), + }); + if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == + ZoneMapFilterResult::kNoMatch) { + return ParquetRowGroupPruneReason::DICTIONARY; + } + } + return ParquetRowGroupPruneReason::NONE; +} + +ParquetRowGroupPruneReason bloom_filter_prune_reason( + int row_group_idx, const std::vector>& file_schema, + const format::FileScanRequest& request, RowGroupBloomFilterCache* bloom_filter_cache, + ParquetPruningStats* pruning_stats) { + if (bloom_filter_cache == nullptr) { + return ParquetRowGroupPruneReason::NONE; + } + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !bloom_filter_supported(*column_schema)) { + continue; + } + auto* bloom_filter = bloom_filter_cache->get(row_group_idx, column_schema->leaf_column_id, + pruning_stats); + if (bloom_filter == nullptr) { + continue; + } + if (ParquetStatisticsUtils::BloomFilterExcludes(*column_schema, slot_index, conjuncts, + *bloom_filter)) { + return ParquetRowGroupPruneReason::BLOOM_FILTER; + } + } + return ParquetRowGroupPruneReason::NONE; +} + +void init_bloom_filter_cache(::parquet::ParquetFileReader* file_reader, bool enable_bloom_filter, + RowGroupBloomFilterCache* bloom_filter_cache) { + DORIS_CHECK(bloom_filter_cache != nullptr); + if (!enable_bloom_filter || file_reader == nullptr) { + return; + } + try { + bloom_filter_cache->bloom_filter_reader = &file_reader->GetBloomFilterReader(); + } catch (const ::parquet::ParquetException&) { + bloom_filter_cache->bloom_filter_reader = nullptr; + } catch (const std::exception&) { + bloom_filter_cache->bloom_filter_reader = nullptr; + } +} + +bool check_statistics(const ::parquet::RowGroupMetaData& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, ParquetPruningStats* pruning_stats, + const cctz::time_zone* timezone) { + const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); + if (slot_indexes.empty()) { + return false; + } + + ZoneMapEvalContext ctx; + for (const int slot_index : slot_indexes) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr) { + continue; + } + + std::shared_ptr zone_map; + DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); + auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); + if (column_chunk != nullptr) { + zone_map = ParquetStatisticsUtils::MakeZoneMap( + ParquetStatisticsUtils::TransformColumnStatistics( + *column_schema, column_chunk->statistics(), timezone)); + } + add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map)); + } + + const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx); + accumulate_zonemap_stats(ctx, pruning_stats); + return result == ZoneMapFilterResult::kNoMatch; +} + +Status select_row_groups_by_metadata_impl( + const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state) { + int64_t row_group_filter_time_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &row_group_filter_time_sink + : &pruning_stats->row_group_filter_time); + if (selected_row_groups == nullptr) { + return Status::InvalidArgument("selected_row_groups is null"); + } + selected_row_groups->clear(); + + const int num_row_groups = metadata.num_row_groups(); + if (pruning_stats != nullptr) { + pruning_stats->total_row_groups = num_row_groups; + } + const auto candidate_size = candidate_row_groups == nullptr + ? static_cast(num_row_groups) + : candidate_row_groups->size(); + selected_row_groups->reserve(candidate_size); + RowGroupBloomFilterCache bloom_filter_cache; + init_bloom_filter_cache(file_reader, enable_bloom_filter, &bloom_filter_cache); + for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { + const int row_group_idx = candidate_row_groups == nullptr + ? static_cast(candidate_idx) + : (*candidate_row_groups)[candidate_idx]; + DORIS_CHECK(row_group_idx >= 0); + DORIS_CHECK(row_group_idx < num_row_groups); + auto row_group = metadata.RowGroup(row_group_idx); + if (row_group == nullptr) { + selected_row_groups->push_back(row_group_idx); + continue; + } + ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; + if (has_expr_zonemap_filter(request, runtime_state) && + check_statistics(*row_group, file_schema, request, pruning_stats, timezone)) { + prune_reason = ParquetRowGroupPruneReason::STATISTICS; + } + + if (prune_reason == ParquetRowGroupPruneReason::NONE) { + prune_reason = dictionary_prune_reason(*row_group, file_reader, row_group_idx, + file_schema, request); + if (prune_reason == ParquetRowGroupPruneReason::NONE) { + prune_reason = bloom_filter_prune_reason(row_group_idx, file_schema, request, + &bloom_filter_cache, pruning_stats); + } + } + + if (prune_reason != ParquetRowGroupPruneReason::NONE) { + if (pruning_stats != nullptr) { + pruning_stats->filtered_group_rows += row_group->num_rows(); + if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { + ++pruning_stats->filtered_row_groups_by_statistics; + } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { + ++pruning_stats->filtered_row_groups_by_dictionary; + } else if (prune_reason == ParquetRowGroupPruneReason::BLOOM_FILTER) { + ++pruning_stats->filtered_row_groups_by_bloom_filter; + } + } + continue; + } + selected_row_groups->push_back(row_group_idx); + } + return Status::OK(); +} + +} // namespace + +Status select_row_groups_by_metadata( + const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state) { + return select_row_groups_by_metadata_impl( + metadata, file_reader, file_schema, request, candidate_row_groups, selected_row_groups, + enable_bloom_filter, pruning_stats, timezone, runtime_state); +} + +namespace { + +template +bool set_page_decoded_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + DecodedValueKind value_kind, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + const auto typed_index = + std::static_pointer_cast<::parquet::TypedColumnIndex>(column_index); + if (page_idx >= typed_index->min_values().size() || + page_idx >= typed_index->max_values().size()) { + return false; + } + const auto& min_value = typed_index->min_values()[page_idx]; + const auto& max_value = typed_index->max_values()[page_idx]; + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { + return false; + } + } + if (!valid_min_max(min_value, max_value)) { + // A NaN invalidates only this page's bounds, not the ColumnIndex itself. Keep the page + // conservatively by returning usable null-count statistics with has_min_max=false, while + // allowing later pages with finite bounds to remain eligible for pruning. + return true; + } + if (!set_decoded_field(column_schema, value_kind, min_value, &page_statistics->min_value, + timezone) || + !set_decoded_field(column_schema, value_kind, max_value, &page_statistics->max_value, + timezone)) { + return false; + } + if (!decoded_min_max_is_ordered(*page_statistics)) { + return true; + } + page_statistics->has_min_max = true; + return true; +} + +bool set_page_string_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + switch (column_schema.descriptor->physical_type()) { + case ::parquet::Type::BYTE_ARRAY: { + const auto typed_index = + std::static_pointer_cast<::parquet::ByteArrayColumnIndex>(column_index); + if (page_idx >= typed_index->min_values().size() || + page_idx >= typed_index->max_values().size()) { + return false; + } + const auto min = ::parquet::ByteArrayToString(typed_index->min_values()[page_idx]); + const auto max = ::parquet::ByteArrayToString(typed_index->max_values()[page_idx]); + if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, + StringRef(min.data(), min.size()), + &page_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, + StringRef(max.data(), max.size()), + &page_statistics->max_value, timezone)) { + return false; + } + if (!decoded_min_max_is_ordered(*page_statistics)) { + return true; + } + page_statistics->has_min_max = true; + return true; + } + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { + const int type_length = column_schema.descriptor->type_length(); + if (type_length <= 0) { + return false; + } + const auto typed_index = std::static_pointer_cast<::parquet::FLBAColumnIndex>(column_index); + if (page_idx >= typed_index->min_values().size() || + page_idx >= typed_index->max_values().size()) { + return false; + } + const std::string min( + reinterpret_cast(typed_index->min_values()[page_idx].ptr), + type_length); + const std::string max( + reinterpret_cast(typed_index->max_values()[page_idx].ptr), + type_length); + if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, + StringRef(min.data(), min.size()), + &page_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, + StringRef(max.data(), max.size()), + &page_statistics->max_value, timezone)) { + return false; + } + if (!decoded_min_max_is_ordered(*page_statistics)) { + return true; + } + page_statistics->has_min_max = true; + return true; + } + default: + return false; + } +} + +bool set_page_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { + DORIS_CHECK(column_schema.type != nullptr); + switch (column_schema.descriptor->physical_type()) { + case ::parquet::Type::BOOLEAN: + return set_page_decoded_min_max<::parquet::BooleanType>(column_index, column_schema, + page_idx, DecodedValueKind::BOOL, + page_statistics, timezone); + case ::parquet::Type::INT32: + return set_page_decoded_min_max<::parquet::Int32Type>( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case ::parquet::Type::INT64: + return set_page_decoded_min_max<::parquet::Int64Type>( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case ::parquet::Type::FLOAT: + return set_page_decoded_min_max<::parquet::FloatType>(column_index, column_schema, page_idx, + DecodedValueKind::FLOAT, + page_statistics, timezone); + case ::parquet::Type::DOUBLE: + return set_page_decoded_min_max<::parquet::DoubleType>(column_index, column_schema, + page_idx, DecodedValueKind::DOUBLE, + page_statistics, timezone); + case ::parquet::Type::BYTE_ARRAY: + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + return set_page_string_min_max(column_index, column_schema, page_idx, page_statistics, + timezone); + default: + return false; + } +} + +bool build_page_statistics(const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + DORIS_CHECK(page_statistics != nullptr); + *page_statistics = ParquetColumnStatistics {}; + + const auto& null_pages = column_index->null_pages(); + if (!column_index->has_null_counts() || page_idx >= null_pages.size() || + page_idx >= column_index->null_counts().size()) { + return false; + } + + page_statistics->has_null_count = true; + page_statistics->has_null = column_index->null_counts()[page_idx] > 0; + page_statistics->has_not_null = !null_pages[page_idx]; + if (!page_statistics->has_not_null) { + return true; + } + return set_page_min_max(column_index, column_schema, page_idx, page_statistics, timezone); +} + +std::vector intersect_ranges(const std::vector& left, + const std::vector& right) { + std::vector result; + size_t left_idx = 0; + size_t right_idx = 0; + while (left_idx < left.size() && right_idx < right.size()) { + const int64_t left_start = left[left_idx].start; + const int64_t left_end = left_start + left[left_idx].length; + const int64_t right_start = right[right_idx].start; + const int64_t right_end = right_start + right[right_idx].length; + const int64_t start = std::max(left_start, right_start); + const int64_t end = std::min(left_end, right_end); + if (start < end) { + result.push_back(RowRange {start, end - start}); + } + if (left_end < right_end) { + ++left_idx; + } else { + ++right_idx; + } + } + return result; +} + +int64_t count_range_rows(const std::vector& ranges) { + int64_t rows = 0; + for (const auto& range : ranges) { + rows += range.length; + } + return rows; +} + +RowRange page_row_range(const ::parquet::OffsetIndex& offset_index, size_t page_idx, + int64_t row_group_rows) { + const auto& page_locations = offset_index.page_locations(); + const int64_t start = page_locations[page_idx].first_row_index; + const int64_t end = page_idx + 1 == page_locations.size() + ? row_group_rows + : page_locations[page_idx + 1].first_row_index; + DORIS_CHECK(start >= 0); + DORIS_CHECK(end >= start); + DORIS_CHECK(end <= row_group_rows); + return RowRange {start, end - start}; +} + +void append_row_range(const RowRange& range, std::vector* ranges) { + if (range.length == 0) { + return; + } + if (!ranges->empty()) { + auto& previous = ranges->back(); + if (previous.start + previous.length == range.start) { + previous.length += range.length; + return; + } + } + ranges->push_back(range); +} + +std::optional< + std::pair, std::shared_ptr<::parquet::OffsetIndex>>> +load_page_indexes_for_slot(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, int slot_index, + const ParquetColumnSchema** column_schema) { + DORIS_CHECK(column_schema != nullptr); + *column_schema = nullptr; + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + return std::nullopt; + } + *column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (*column_schema == nullptr || (*column_schema)->descriptor == nullptr) { + return std::nullopt; + } + + try { + auto column_index = row_group->GetColumnIndex((*column_schema)->leaf_column_id); + auto offset_index = row_group->GetOffsetIndex((*column_schema)->leaf_column_id); + if (column_index == nullptr || offset_index == nullptr || + column_index->null_pages().size() != offset_index->page_locations().size()) { + return std::nullopt; + } + return std::make_pair(std::move(column_index), std::move(offset_index)); + } catch (const ::parquet::ParquetException&) { + return std::nullopt; + } catch (const std::exception&) { + return std::nullopt; + } +} + +bool select_ranges_for_expr_zonemap( + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, int slot_index, const VExprContextSPtrs& conjuncts, + int64_t row_group_rows, std::vector* ranges, ParquetPruningStats* pruning_stats, + const cctz::time_zone* timezone) { + DORIS_CHECK(ranges != nullptr); + if (conjuncts.empty()) { + return false; + } + const ParquetColumnSchema* column_schema = nullptr; + const auto page_indexes = + load_page_indexes_for_slot(row_group, file_schema, request, slot_index, &column_schema); + if (!page_indexes.has_value()) { + return false; + } + const auto& [column_index, offset_index] = *page_indexes; + + ranges->clear(); + ZoneMapEvalStats page_stats; + const auto page_count = offset_index->page_locations().size(); + for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { + ParquetColumnStatistics page_statistics; + if (!ParquetStatisticsUtils::TransformColumnIndexStatistics( + column_index, *column_schema, page_idx, &page_statistics, timezone)) { + ranges->clear(); + return false; + } + + ZoneMapEvalContext ctx; + add_slot_zonemap(&ctx, slot_index, column_schema->type, + ParquetStatisticsUtils::MakeZoneMap(page_statistics)); + const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx); + page_stats.merge_page_eval_stats(ctx.stats); + if (result == ZoneMapFilterResult::kNoMatch) { + continue; + } + append_row_range(page_row_range(*offset_index, page_idx, row_group_rows), ranges); + } + if (pruning_stats != nullptr) { + pruning_stats->expr_zonemap_unusable_evals += page_stats.unusable_zonemap_eval_count; + pruning_stats->in_zonemap_point_check_count += page_stats.in_zonemap_point_check_count; + pruning_stats->in_zonemap_range_only_count += page_stats.in_zonemap_range_only_count; + } + return true; +} + +bool ranges_intersect(const std::vector& ranges, const RowRange& range) { + const int64_t range_end = range.start + range.length; + for (const auto& selected_range : ranges) { + const int64_t selected_end = selected_range.start + selected_range.length; + if (selected_end <= range.start) { + continue; + } + if (selected_range.start >= range_end) { + return false; + } + return true; + } + return false; +} + +void collect_leaf_schemas(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::vector* leaf_schemas) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + leaf_schemas->push_back(&column_schema); + return; + } + for (const auto& child_schema : column_schema.children) { + if (!format::is_child_projected(projection, child_schema->local_id)) { + continue; + } + const auto* child_projection = + format::find_child_projection(projection, child_schema->local_id); + collect_leaf_schemas(*child_schema, child_projection, leaf_schemas); + } +} + +void collect_request_leaf_schemas( + const std::vector>& file_schema, + const format::FileScanRequest& request, + std::vector* leaf_schemas) { + std::set seen_leaf_ids; + auto collect_projection = [&](const format::LocalColumnIndex& projection) { + const int32_t local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + return; + } + std::vector projection_leaf_schemas; + collect_leaf_schemas(*file_schema[local_id], &projection, &projection_leaf_schemas); + for (const auto* leaf_schema : projection_leaf_schemas) { + DORIS_CHECK(leaf_schema != nullptr); + if (seen_leaf_ids.insert(leaf_schema->leaf_column_id).second) { + leaf_schemas->push_back(leaf_schema); + } + } + }; + for (const auto& projection : request.predicate_columns) { + collect_projection(projection); + } + for (const auto& projection : request.non_predicate_columns) { + collect_projection(projection); + } +} + +bool build_page_skip_plan_for_leaf( + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, + const ParquetColumnSchema& column_schema, const std::vector& selected_ranges, + int64_t row_group_rows, ParquetPageSkipPlan* page_skip_plan) { + DORIS_CHECK(page_skip_plan != nullptr); + *page_skip_plan = ParquetPageSkipPlan {}; + if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || + column_schema.descriptor == nullptr || column_schema.leaf_column_id < 0 || + column_schema.descriptor->max_repetition_level() != 0) { + return false; + } + + std::shared_ptr<::parquet::OffsetIndex> offset_index; + try { + offset_index = row_group->GetOffsetIndex(column_schema.leaf_column_id); + } catch (const ::parquet::ParquetException&) { + return false; + } catch (const std::exception&) { + return false; + } + if (offset_index == nullptr) { + return false; + } + + const auto page_count = offset_index->page_locations().size(); + page_skip_plan->leaf_column_id = column_schema.leaf_column_id; + page_skip_plan->skipped_pages.resize(page_count); + page_skip_plan->skipped_page_compressed_sizes.resize(page_count); + const auto& page_locations = offset_index->page_locations(); + for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { + const RowRange row_range = page_row_range(*offset_index, page_idx, row_group_rows); + if (row_range.length == 0 || ranges_intersect(selected_ranges, row_range)) { + continue; + } + page_skip_plan->skipped_pages[page_idx] = 1; + page_skip_plan->skipped_page_compressed_sizes[page_idx] = + page_locations[page_idx].compressed_page_size; + append_row_range(row_range, &page_skip_plan->skipped_ranges); + } + if (page_skip_plan->empty()) { + *page_skip_plan = ParquetPageSkipPlan {}; + return false; + } + return true; +} + +void build_page_skip_plans(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, + const std::vector& selected_ranges, int64_t row_group_rows, + std::map* page_skip_plans) { + DORIS_CHECK(page_skip_plans != nullptr); + page_skip_plans->clear(); + std::vector leaf_schemas; + collect_request_leaf_schemas(file_schema, request, &leaf_schemas); + for (const auto* leaf_schema : leaf_schemas) { + DORIS_CHECK(leaf_schema != nullptr); + ParquetPageSkipPlan page_skip_plan; + if (build_page_skip_plan_for_leaf(row_group, *leaf_schema, selected_ranges, row_group_rows, + &page_skip_plan)) { + page_skip_plans->emplace(page_skip_plan.leaf_column_id, std::move(page_skip_plan)); + } + } +} + +} // namespace + +bool ParquetStatisticsUtils::TransformColumnIndexStatistics( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { + return build_page_statistics(column_index, column_schema, page_idx, page_statistics, timezone); +} + +Status select_row_group_ranges_by_page_index( + ::parquet::ParquetFileReader* file_reader, + const std::vector>& file_schema, + const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + std::vector* selected_ranges, std::map* page_skip_plans, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state) { + int64_t page_index_filter_time_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &page_index_filter_time_sink + : &pruning_stats->page_index_filter_time); + DORIS_CHECK(selected_ranges != nullptr); + selected_ranges->clear(); + if (page_skip_plans != nullptr) { + page_skip_plans->clear(); + } + if (row_group_rows <= 0) { + return Status::OK(); + } + selected_ranges->push_back(RowRange {0, row_group_rows}); + if (!config::enable_parquet_page_index || !has_expr_zonemap_filter(request, runtime_state) || + file_reader == nullptr) { + return Status::OK(); + } + + std::shared_ptr<::parquet::PageIndexReader> page_index_reader; + std::shared_ptr<::parquet::RowGroupPageIndexReader> row_group_index_reader; + try { + if (pruning_stats != nullptr) { + ++pruning_stats->page_index_read_calls; + } + { + int64_t read_page_index_time_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &read_page_index_time_sink + : &pruning_stats->read_page_index_time); + page_index_reader = file_reader->GetPageIndexReader(); + if (page_index_reader == nullptr) { + return Status::OK(); + } + row_group_index_reader = page_index_reader->RowGroup(row_group_idx); + } + } catch (const ::parquet::ParquetException&) { + return Status::OK(); + } catch (const std::exception&) { + return Status::OK(); + } + if (row_group_index_reader == nullptr) { + return Status::OK(); + } + + std::map conjuncts_by_slot; + for (const auto& conjunct : request.conjuncts) { + const auto slot_index = expr_zonemap::single_slot_zonemap_index(conjunct); + if (slot_index >= 0) { + conjuncts_by_slot[slot_index].push_back(conjunct); + } + } + + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + std::vector filter_ranges; + if (!select_ranges_for_expr_zonemap(row_group_index_reader, file_schema, request, + slot_index, conjuncts, row_group_rows, &filter_ranges, + pruning_stats, timezone)) { + continue; + } + *selected_ranges = intersect_ranges(*selected_ranges, filter_ranges); + if (selected_ranges->empty()) { + if (page_skip_plans != nullptr) { + page_skip_plans->clear(); + } + if (pruning_stats != nullptr) { + pruning_stats->filtered_page_rows += row_group_rows; + ++pruning_stats->filtered_row_groups_by_page_index; + } + return Status::OK(); + } + } + if (page_skip_plans != nullptr) { + build_page_skip_plans(row_group_index_reader, file_schema, request, *selected_ranges, + row_group_rows, page_skip_plans); + } + if (pruning_stats != nullptr) { + const int64_t selected_rows = count_range_rows(*selected_ranges); + DORIS_CHECK(selected_rows <= row_group_rows); + pruning_stats->filtered_page_rows += row_group_rows - selected_rows; + } + return Status::OK(); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h new file mode 100644 index 00000000000000..9e94562bf2d37b --- /dev/null +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/field.h" +#include "core/string_ref.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/selection_vector.h" + +namespace parquet { +class BloomFilter; +class ColumnIndex; +class FileMetaData; +class ParquetFileReader; +class Statistics; +} // namespace parquet + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { +class RuntimeState; +namespace segment_v2 { +struct ZoneMap; +} // namespace segment_v2 +} // namespace doris + +namespace doris::format::parquet { + +struct ParquetColumnSchema; + +// ============================================================================ +// ============================================================================ + +struct ParquetDictionaryWords { + std::vector values; + std::vector refs; + + void clear() { + values.clear(); + refs.clear(); + } + + void build_refs() { + refs.clear(); + refs.reserve(values.size()); + for (const auto& value : values) { + refs.emplace_back(value.data(), value.size()); + } + } +}; + +// Reads the PLAIN dictionary page for BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY columns and owns copied +// dictionary bytes in `values`. Both row-group pruning and row-level dictionary predicates use this +// helper so they agree on dictionary id -> Doris string value mapping. +bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, + int leaf_column_id, const ParquetColumnSchema& column_schema, + ParquetDictionaryWords* dict_words); + +std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words); + +// ============================================================================ +// ============================================================================ + +struct ParquetPruningStats { + int64_t total_row_groups = 0; // total row groups in the file + int64_t selected_row_groups = 0; // row groups selected after pruning + int64_t filtered_row_groups_by_statistics = 0; // row groups pruned by ZoneMap statistics + int64_t filtered_row_groups_by_dictionary = 0; // row groups pruned by dictionary + int64_t filtered_row_groups_by_bloom_filter = 0; // row groups pruned by bloom filter + int64_t filtered_row_groups_by_page_index = 0; // row groups fully pruned by page index + int64_t filtered_group_rows = 0; // rows in pruned row groups + int64_t filtered_page_rows = 0; // rows pruned by page index + int64_t selected_row_ranges = 0; // selected row range count + int64_t page_index_read_calls = 0; // Page Index read count + int64_t bloom_filter_read_time = 0; // Bloom filter read time (ns) + int64_t row_group_filter_time = 0; // row-group pruning time (ns) + int64_t page_index_filter_time = 0; // page-index pruning time (ns) + int64_t read_page_index_time = 0; // page-index read time (ns) + int64_t expr_zonemap_unusable_evals = 0; // VExpr ZoneMap unusable evaluations + int64_t in_zonemap_point_check_count = 0; // VExpr IN ZoneMap point checks + int64_t in_zonemap_range_only_count = 0; // VExpr IN ZoneMap range-only checks +}; + +struct ParquetColumnStatistics { + Field min_value; // column minimum value converted to Doris type + Field max_value; // column maximum value + bool has_null = false; // whether NULL exists + bool has_not_null = false; // whether non-NULL values exist + bool has_null_count = false; // whether null_count is valid + bool has_min_max = false; // whether min/max is valid after conversion + + bool has_any_statistics() const { return has_null_count || has_min_max; } +}; + +// ============================================================================ +// ============================================================================ +// VExpr ZoneMap(TransformColumnStatistics + evaluate_zonemap_filter) +// -> page-index ZoneMap(evaluate_zonemap_filter) +// dictionary(read_dictionary_words + evaluate_dictionary_filter) +// -> bloom filter(evaluate_bloom_filter) +// ============================================================================ +struct ParquetStatisticsUtils { + static std::shared_ptr MakeZoneMap( + const ParquetColumnStatistics& statistics); + + static ParquetColumnStatistics TransformColumnStatistics( + const ParquetColumnSchema& column_schema, + const std::shared_ptr<::parquet::Statistics>& statistics, + const cctz::time_zone* timezone = nullptr); + + static bool TransformColumnIndexStatistics( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone = nullptr); + + static bool BloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, + const ::parquet::BloomFilter& bloom_filter); +}; + +Status select_row_groups_by_metadata( + const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, + const RuntimeState* runtime_state = nullptr); + +Status select_row_group_ranges_by_page_index( + ::parquet::ParquetFileReader* file_reader, + const std::vector>& file_schema, + const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + std::vector* selected_ranges, std::map* page_skip_plans, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, + const RuntimeState* runtime_state = nullptr); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_type.cpp b/be/src/format_v2/parquet/parquet_type.cpp new file mode 100644 index 00000000000000..d35181d0397178 --- /dev/null +++ b/be/src/format_v2/parquet/parquet_type.cpp @@ -0,0 +1,358 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_type.h" + +#include + +#include +#include + +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/primitive_type.h" + +namespace doris::format::parquet { +namespace { + +DataTypePtr create_type(PrimitiveType type, bool nullable, int precision = 0, int scale = 0) { + return DataTypeFactory::instance().create_data_type(type, nullable, precision, scale); +} + +PrimitiveType decimal_primitive_type(int precision) { + return precision > 38 ? TYPE_DECIMAL256 : TYPE_DECIMAL128I; +} + +void mark_decimal(const ::parquet::ColumnDescriptor* column, int precision, int scale, + ParquetTypeDescriptor* result) { + result->is_decimal = true; + result->decimal_precision = precision; + result->decimal_scale = scale; + switch (column->physical_type()) { + case ::parquet::Type::INT32: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; + break; + case ::parquet::Type::INT64: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; + break; + case ::parquet::Type::BYTE_ARRAY: + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; + break; + default: + result->extra_type_info = ParquetExtraTypeInfo::NONE; + break; + } +} + +void mark_integer(int bit_width, bool is_signed, ParquetTypeDescriptor* result) { + result->integer_bit_width = bit_width; + result->is_unsigned_integer = !is_signed; +} + +DataTypePtr converted_type_to_doris_type(const ::parquet::ColumnDescriptor* column, + ParquetTypeDescriptor* result) { + const bool nullable = column->max_definition_level() > 0; + switch (column->converted_type()) { + case ::parquet::ConvertedType::UTF8: + case ::parquet::ConvertedType::ENUM: + case ::parquet::ConvertedType::JSON: + case ::parquet::ConvertedType::BSON: + return create_type(TYPE_STRING, nullable); + case ::parquet::ConvertedType::DECIMAL: + mark_decimal(column, column->type_precision(), column->type_scale(), result); + return create_type(decimal_primitive_type(column->type_precision()), nullable, + column->type_precision(), column->type_scale()); + case ::parquet::ConvertedType::DATE: + return create_type(TYPE_DATEV2, nullable); + case ::parquet::ConvertedType::TIME_MILLIS: + result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; + return nullptr; + case ::parquet::ConvertedType::TIME_MICROS: + result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; + return nullptr; + case ::parquet::ConvertedType::TIMESTAMP_MILLIS: + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = true; + result->time_unit = ParquetTimeUnit::MILLIS; + result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; + return create_type(TYPE_DATETIMEV2, nullable, 0, 3); + case ::parquet::ConvertedType::TIMESTAMP_MICROS: + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = true; + result->time_unit = ParquetTimeUnit::MICROS; + result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; + return create_type(TYPE_DATETIMEV2, nullable, 0, 6); + // Parquet stores signed and unsigned integer logical annotations on signed physical carriers: + // INT_8/UINT_8/INT_16/UINT_16/INT_32/UINT_32 use physical INT32, and + // INT_64/UINT_64 use physical INT64. Doris maps unsigned integers to the next wider + // signed type so all values in the unsigned range can be represented. + case ::parquet::ConvertedType::INT_8: + mark_integer(8, true, result); + return create_type(TYPE_TINYINT, nullable); + case ::parquet::ConvertedType::UINT_8: + mark_integer(8, false, result); + return create_type(TYPE_SMALLINT, nullable); + case ::parquet::ConvertedType::INT_16: + mark_integer(16, true, result); + return create_type(TYPE_SMALLINT, nullable); + case ::parquet::ConvertedType::UINT_16: + mark_integer(16, false, result); + return create_type(TYPE_INT, nullable); + case ::parquet::ConvertedType::INT_32: + mark_integer(32, true, result); + return create_type(TYPE_INT, nullable); + case ::parquet::ConvertedType::UINT_32: + mark_integer(32, false, result); + return create_type(TYPE_BIGINT, nullable); + case ::parquet::ConvertedType::INT_64: + mark_integer(64, true, result); + return create_type(TYPE_BIGINT, nullable); + case ::parquet::ConvertedType::UINT_64: + mark_integer(64, false, result); + return create_type(TYPE_LARGEINT, nullable); + case ::parquet::ConvertedType::NONE: + default: + return nullptr; + } +} + +DataTypePtr logical_type_to_doris_type(const ::parquet::ColumnDescriptor* column, + ParquetTypeDescriptor* result) { + const auto& logical_type = column->logical_type(); + if (logical_type == nullptr || !logical_type->is_valid() || logical_type->is_none()) { + return nullptr; + } + const bool nullable = column->max_definition_level() > 0; + if (logical_type->is_string() || logical_type->is_enum() || logical_type->is_JSON() || + logical_type->is_BSON() || logical_type->is_UUID()) { + return create_type(TYPE_STRING, nullable); + } + if (logical_type->is_decimal()) { + const auto& decimal_type = static_cast(*logical_type); + mark_decimal(column, decimal_type.precision(), decimal_type.scale(), result); + return create_type(decimal_primitive_type(decimal_type.precision()), nullable, + decimal_type.precision(), decimal_type.scale()); + } + if (logical_type->is_date()) { + return create_type(TYPE_DATEV2, nullable); + } + if (logical_type->is_time()) { + const auto& time_type = static_cast(*logical_type); + if (time_type.is_adjusted_to_utc()) { + result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; + return nullptr; + } + int scale = 0; + if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { + scale = 3; + result->time_unit = ParquetTimeUnit::MILLIS; + result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; + } else if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { + scale = 6; + result->time_unit = ParquetTimeUnit::MICROS; + result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; + } else { + return nullptr; + } + return create_type(TYPE_TIMEV2, nullable, 0, scale); + } + if (logical_type->is_timestamp()) { + const auto& timestamp_type = + static_cast(*logical_type); + int scale = 0; + if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { + scale = 3; + result->time_unit = ParquetTimeUnit::MILLIS; + result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; + } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { + scale = 6; + result->time_unit = ParquetTimeUnit::MICROS; + result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; + } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::NANOS) { + scale = 6; + result->time_unit = ParquetTimeUnit::NANOS; + result->extra_type_info = ParquetExtraTypeInfo::UNIT_NS; + } else { + return nullptr; + } + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = timestamp_type.is_adjusted_to_utc(); + return create_type(TYPE_DATETIMEV2, nullable, 0, scale); + } + if (logical_type->is_int()) { + const auto& int_type = static_cast(*logical_type); + mark_integer(int_type.bit_width(), int_type.is_signed(), result); + switch (int_type.bit_width()) { + case 8: + return create_type(int_type.is_signed() ? TYPE_TINYINT : TYPE_SMALLINT, nullable); + case 16: + return create_type(int_type.is_signed() ? TYPE_SMALLINT : TYPE_INT, nullable); + case 32: + return create_type(int_type.is_signed() ? TYPE_INT : TYPE_BIGINT, nullable); + case 64: + return create_type(int_type.is_signed() ? TYPE_BIGINT : TYPE_LARGEINT, nullable); + default: + return nullptr; + } + } + if (logical_type->is_float16()) { + if (column->physical_type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY || + column->type_length() != 2) { + return nullptr; + } + result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; + return create_type(TYPE_FLOAT, nullable); + } + return nullptr; +} + +DataTypePtr physical_type_to_doris_type(const ::parquet::ColumnDescriptor* column) { + const bool nullable = column->max_definition_level() > 0; + DataTypePtr type; + switch (column->physical_type()) { + case ::parquet::Type::BOOLEAN: + type = std::make_shared(); + break; + case ::parquet::Type::INT32: + type = std::make_shared(); + break; + case ::parquet::Type::INT64: + type = std::make_shared(); + break; + case ::parquet::Type::FLOAT: + type = std::make_shared(); + break; + case ::parquet::Type::DOUBLE: + type = std::make_shared(); + break; + case ::parquet::Type::BYTE_ARRAY: + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + type = std::make_shared(); + break; + case ::parquet::Type::INT96: + type = create_type(TYPE_DATETIMEV2, nullable, 0, 6); + break; + default: + return nullptr; + } + return nullable ? make_nullable(type) : type; +} + +bool record_reader_physical_type_supported(::parquet::Type::type physical_type) { + switch (physical_type) { + case ::parquet::Type::BOOLEAN: + case ::parquet::Type::INT32: + case ::parquet::Type::INT64: + case ::parquet::Type::INT96: + case ::parquet::Type::FLOAT: + case ::parquet::Type::DOUBLE: + case ::parquet::Type::BYTE_ARRAY: + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + return true; + default: + return false; + } +} + +} // namespace + +std::string parquet_column_name(const ::parquet::ColumnDescriptor* column) { + if (column == nullptr) { + return {}; + } + auto path = column->path(); + if (path) { + return path->ToDotString(); + } + return column->name(); +} + +ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column) { + ParquetTypeDescriptor result; + if (column == nullptr) { + return result; + } + + result.physical_type = column->physical_type(); + result.converted_type = column->converted_type(); + result.fixed_length = column->type_length(); + + if (auto logical_type = logical_type_to_doris_type(column, &result); logical_type != nullptr) { + result.doris_type = logical_type; + } else if (!result.unsupported_reason.empty()) { + result.doris_type = nullptr; + result.supports_record_reader = false; + } else if (auto converted_type = converted_type_to_doris_type(column, &result); + converted_type != nullptr) { + result.doris_type = converted_type; + } else if (!result.unsupported_reason.empty()) { + result.doris_type = nullptr; + result.supports_record_reader = false; + } else { + result.doris_type = physical_type_to_doris_type(column); + if (result.physical_type == ::parquet::Type::INT96) { + result.extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; + } + } + + result.is_string_like = !result.is_decimal && + result.extra_type_info != ParquetExtraTypeInfo::FLOAT16 && + (result.physical_type == ::parquet::Type::BYTE_ARRAY || + result.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); + + if (!record_reader_physical_type_supported(result.physical_type)) { + result.supports_record_reader = false; + } + return result; +} + +bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor) { + return type_descriptor.supports_record_reader; +} + +DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor) { + switch (type_descriptor.physical_type) { + case ::parquet::Type::BOOLEAN: + return DecodedValueKind::BOOL; + case ::parquet::Type::INT32: + if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 32) { + return DecodedValueKind::UINT32; + } + return DecodedValueKind::INT32; + case ::parquet::Type::INT64: + if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 64) { + return DecodedValueKind::UINT64; + } + return DecodedValueKind::INT64; + case ::parquet::Type::INT96: + return DecodedValueKind::INT96; + case ::parquet::Type::FLOAT: + return DecodedValueKind::FLOAT; + case ::parquet::Type::DOUBLE: + return DecodedValueKind::DOUBLE; + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + return DecodedValueKind::FIXED_BINARY; + case ::parquet::Type::BYTE_ARRAY: + default: + return DecodedValueKind::BINARY; + } +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_type.h b/be/src/format_v2/parquet/parquet_type.h new file mode 100644 index 00000000000000..5d21aae6bae092 --- /dev/null +++ b/be/src/format_v2/parquet/parquet_type.h @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "core/data_type/data_type.h" +#include "core/data_type_serde/decoded_column_view.h" + +namespace parquet { +class ColumnDescriptor; +} // namespace parquet + +namespace doris::format::parquet { + +// ============================================================================ +// ============================================================================ + +enum class ParquetExtraTypeInfo { + NONE, // no special encoding; read by physical type + DECIMAL_INT32, // decimal stored as a 4-byte big-endian int + DECIMAL_INT64, // decimal stored as an 8-byte big-endian int + DECIMAL_BYTE_ARRAY, // decimal stored as a variable/fixed-length big-endian byte array + UNIT_MS, // time unit is milliseconds + UNIT_MICROS, // time unit is microseconds + UNIT_NS, // time unit is nanoseconds + IMPALA_TIMESTAMP, // Impala-compatible timestamp encoded as INT96 + FLOAT16, // half-precision float (FIXED_LEN_BYTE_ARRAY(2) -> Float32) +}; + +enum class ParquetTimeUnit { + UNKNOWN, + MILLIS, + MICROS, + NANOS, +}; + +// ============================================================================ +// ============================================================================ +struct ParquetTypeDescriptor { + DataTypePtr doris_type; + ParquetExtraTypeInfo extra_type_info = ParquetExtraTypeInfo::NONE; + ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; + ::parquet::Type::type physical_type = ::parquet::Type::UNDEFINED; + ::parquet::ConvertedType::type converted_type = ::parquet::ConvertedType::UNDEFINED; + int integer_bit_width = -1; // bit width for INT_8/16/32/64 + int decimal_precision = -1; // precision for DECIMAL(p,s) + int decimal_scale = -1; // scale for DECIMAL(p,s) + int fixed_length = -1; // fixed length for FIXED_LEN_BYTE_ARRAY + bool is_unsigned_integer = false; // whether the integer is unsigned (UINT_8/16/32/64) + bool is_decimal = false; // whether this is a decimal type + bool is_timestamp = false; // whether this is a timestamp type + bool timestamp_is_adjusted_to_utc = false; // whether the timestamp is UTC-normalized + bool is_string_like = false; // binary type that is neither decimal nor FLOAT16 + bool supports_record_reader = true; // whether Arrow RecordReader can read this type + std::string unsupported_reason; // non-empty when this Parquet logical type is unsupported +}; + +std::string parquet_column_name(const ::parquet::ColumnDescriptor* column); + +ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column); + +bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor); + +DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.cpp b/be/src/format_v2/parquet/reader/column_reader.cpp new file mode 100644 index 00000000000000..352fbbd7c3d215 --- /dev/null +++ b/be/src/format_v2/parquet/reader/column_reader.cpp @@ -0,0 +1,658 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/column_reader.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/reader/global_rowid_column_reader.h" +#include "format_v2/parquet/reader/list_column_reader.h" +#include "format_v2/parquet/reader/map_column_reader.h" +#include "format_v2/parquet/reader/row_position_column_reader.h" +#include "format_v2/parquet/reader/scalar_column_reader.h" +#include "format_v2/parquet/reader/struct_column_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet { +namespace { + +class DataPageSkipFilter { +public: + DataPageSkipFilter(const ParquetPageSkipPlan* page_skip_plan, + ParquetPageSkipProfile page_skip_profile) + : _page_skip_plan(page_skip_plan), _page_skip_profile(page_skip_profile) { + DORIS_CHECK(_page_skip_plan != nullptr); + } + + bool operator()(const ::parquet::DataPageStats&) { + // Arrow invokes this callback once for each DATA_PAGE/DATA_PAGE_V2 and never for + // dictionary pages, so this ordinal matches Parquet OffsetIndex page locations. + const size_t page_idx = _next_data_page_idx++; + const bool skip = _page_skip_plan->should_skip_page(page_idx); + if (!skip) { + return false; + } + update_skip_profile(page_idx); + return true; + } + +private: + void update_skip_profile(size_t page_idx) const { + if (_page_skip_profile.skipped_pages != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); + } + if (_page_skip_profile.skipped_bytes != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_bytes, + _page_skip_plan->skipped_page_compressed_size(page_idx)); + } + } + + const ParquetPageSkipPlan* _page_skip_plan = nullptr; + ParquetPageSkipProfile _page_skip_profile; + size_t _next_data_page_idx = 0; +}; + +const ParquetPageSkipPlan* find_page_skip_plan( + const std::map* page_skip_plans, int leaf_column_id) { + if (page_skip_plans == nullptr) { + return nullptr; + } + const auto plan_it = page_skip_plans->find(leaf_column_id); + return plan_it == page_skip_plans->end() ? nullptr : &plan_it->second; +} + +void install_data_page_filter(std::unique_ptr<::parquet::PageReader>& page_reader, + const std::map* page_skip_plans, + int leaf_column_id, ParquetPageSkipProfile page_skip_profile) { + DORIS_CHECK(page_reader != nullptr); + const ParquetPageSkipPlan* page_skip_plan = + find_page_skip_plan(page_skip_plans, leaf_column_id); + if (page_skip_plan == nullptr) { + return; + } + page_reader->set_data_page_filter(DataPageSkipFilter(page_skip_plan, page_skip_profile)); +} + +bool supports_nested_scalar_record_reader(const ParquetColumnSchema& column_schema) { + if (column_schema.type_descriptor.supports_record_reader) { + return true; + } + const auto& type_descriptor = column_schema.type_descriptor; + if ((type_descriptor.extra_type_info != ParquetExtraTypeInfo::NONE && + type_descriptor.extra_type_info != ParquetExtraTypeInfo::FLOAT16) || + type_descriptor.is_decimal || type_descriptor.is_timestamp || + type_descriptor.is_string_like) { + return false; + } + if (type_descriptor.converted_type != ::parquet::ConvertedType::NONE && + type_descriptor.converted_type != ::parquet::ConvertedType::UNDEFINED) { + return false; + } + switch (type_descriptor.physical_type) { + case ::parquet::Type::BOOLEAN: + case ::parquet::Type::INT32: + case ::parquet::Type::INT64: + case ::parquet::Type::FLOAT: + case ::parquet::Type::DOUBLE: + return true; + default: + return false; + } + return true; +} + +} // namespace + +Status ParquetColumnReader::skip(int64_t rows) { + return Status::NotSupported("Parquet column skip is not implemented, rows={}", rows); +} + +void ParquetColumnReader::advance_nested_build_level_cursor_past_parent( + int16_t parent_repetition_level) { + int64_t child_cursor = nested_build_level_cursor(); + const auto& child_rep_levels = nested_repetition_levels(); + const int64_t child_levels_written = nested_levels_written(); + while (child_cursor < child_levels_written) { + const int16_t child_rep_level = child_rep_levels[child_cursor]; + ++child_cursor; + if (!is_or_has_repeated_child() || child_rep_level <= parent_repetition_level) { + break; + } + } + set_nested_build_level_cursor(child_cursor); +} + +void ParquetColumnReader::update_reader_read_rows(int64_t rows) const { + if (_profile.reader_read_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_read_rows, rows); + } +} + +void ParquetColumnReader::update_reader_skip_rows(int64_t rows) const { + if (_profile.reader_skip_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_skip_rows, rows); + } +} + +Status ParquetColumnReader::select(const SelectionVector& sel, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column) { + if (column.get() == nullptr) { + return Status::InvalidArgument("Parquet selected read result is null for column {}", + name()); + } + RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); + + const auto ranges = selection_to_ranges(sel, selected_rows); + int64_t cursor = 0; + for (const auto& range : ranges) { + if (range.start < cursor || range.start + range.length > batch_rows) { + return Status::InvalidArgument("Invalid parquet selection range [{}, {}) for column {}", + range.start, range.start + range.length, name()); + } + RETURN_IF_ERROR(skip(range.start - cursor)); + + int64_t range_rows_read = 0; + RETURN_IF_ERROR(read(range.length, column, &range_rows_read)); + if (range_rows_read != range.length) { + return Status::Corruption( + "Parquet selected read returned {} rows, expected {} rows for column {}", + range_rows_read, range.length, name()); + } + cursor = range.start + range.length; + } + RETURN_IF_ERROR(skip(batch_rows - cursor)); + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + return Status::OK(); +} + +Status ParquetColumnReader::select_with_dictionary_filter(const SelectionVector&, uint16_t, int64_t, + const IColumn::Filter&, MutableColumnPtr&, + IColumn::Filter*, bool*) { + return Status::NotSupported("Parquet dictionary filter is not implemented for column {}", + name()); +} + +ParquetColumnReaderFactory::ParquetColumnReaderFactory( + std::shared_ptr<::parquet::RowGroupReader> row_group, int num_leaf_columns, + const std::map* page_skip_plans, + ParquetPageSkipProfile page_skip_profile, const cctz::time_zone* timezone, + bool enable_strict_mode, ParquetColumnReaderProfile column_reader_profile) + : _row_group(std::move(row_group)), + _record_readers(static_cast(num_leaf_columns)), + _dictionary_record_readers(static_cast(num_leaf_columns)), + _page_skip_plans(page_skip_plans), + _page_skip_profile(page_skip_profile), + _timezone(timezone), + _enable_strict_mode(enable_strict_mode), + _column_reader_profile(column_reader_profile) {} + +std::unique_ptr ParquetColumnReaderFactory::create_row_position_column_reader( + int64_t row_group_first_row) const { + return std::make_unique(row_group_first_row, _column_reader_profile); +} + +std::unique_ptr ParquetColumnReaderFactory::create_global_rowid_column_reader( + const format::GlobalRowIdContext& context, int64_t row_group_first_row) const { + return std::make_unique(context, row_group_first_row, + _column_reader_profile); +} + +Status ParquetColumnReaderFactory::make_scalar_column_reader( + const ParquetColumnSchema& column_schema, + std::shared_ptr<::parquet::internal::RecordReader> record_reader, bool use_page_skip_plan, + std::unique_ptr* reader) const { + if (reader == nullptr) { + return Status::InvalidArgument("reader is null"); + } + const auto* page_skip_plan = + use_page_skip_plan ? find_page_skip_plan(_page_skip_plans, column_schema.leaf_column_id) + : nullptr; + *reader = std::make_unique(column_schema, std::move(record_reader), + page_skip_plan, _timezone, _enable_strict_mode, + _column_reader_profile); + return Status::OK(); +} + +Status ParquetColumnReaderFactory::create_scalar_column_reader( + const ParquetColumnSchema& column_schema, bool is_nested, bool read_dictionary, + std::unique_ptr* reader) const { + if (reader == nullptr) { + return Status::InvalidArgument("reader is null"); + } + if (!column_schema.type_descriptor.unsupported_reason.empty()) { + return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, + column_schema.type_descriptor.unsupported_reason); + } + if (is_nested && column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { + return Status::InvalidArgument("Parquet nested scalar reader requires primitive column {}", + column_schema.name); + } + if (column_schema.leaf_column_id < 0 || + column_schema.leaf_column_id >= static_cast(_record_readers.size())) { + return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", + column_schema.leaf_column_id, column_schema.name); + } + if (column_schema.descriptor == nullptr) { + return Status::InvalidArgument("Parquet column descriptor is null for column {}", + column_schema.name); + } + if (!is_nested && (column_schema.descriptor->max_repetition_level() != 0 || + column_schema.descriptor->max_definition_level() > 1)) { + return Status::NotSupported( + "Current parquet scalar reader only supports flat primitive columns; column {} is " + "not supported", + column_schema.name); + } + if (is_nested && !supports_nested_scalar_record_reader(column_schema)) { + return Status::NotSupported( + "Current parquet nested scalar reader does not support column {}", + column_schema.name); + } + if (!is_nested && !column_schema.type_descriptor.supports_record_reader) { + return Status::NotSupported("Current parquet scalar reader does not support column {}", + column_schema.name); + } + std::shared_ptr<::parquet::internal::RecordReader> record_reader; + // Nested readers implement skip() by materializing rows into a scratch column. If Arrow + // page filtering is also installed, those scratch reads can consume the next selected row + // after a page-index range gap. Keep page filtering on flat scalar readers only. + RETURN_IF_ERROR(get_record_reader(column_schema.leaf_column_id, column_schema.descriptor, + column_schema.name, !is_nested, read_dictionary, + &record_reader)); + return make_scalar_column_reader(column_schema, std::move(record_reader), !is_nested, reader); +} + +// 1. RowGroupReader::GetColumnPageReader(leaf_column_id) -> Arrow PageReader +Status ParquetColumnReaderFactory::get_record_reader( + int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, const std::string& name, + bool install_page_filter, bool read_dictionary, + std::shared_ptr<::parquet::internal::RecordReader>* reader) const { + if (reader == nullptr) { + return Status::InvalidArgument("reader is null"); + } + if (_row_group == nullptr) { + return Status::InternalError("Parquet row group reader is not initialized for column {}", + name); + } + if (leaf_column_id < 0 || leaf_column_id >= static_cast(_record_readers.size())) { + return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", + leaf_column_id, name); + } + if (descriptor == nullptr) { + return Status::InvalidArgument("Parquet column descriptor is null for column {}", name); + } + auto& record_readers = read_dictionary ? _dictionary_record_readers : _record_readers; + if (record_readers[leaf_column_id] == nullptr) { + try { + auto page_reader = _row_group->GetColumnPageReader(leaf_column_id); + if (install_page_filter) { + install_data_page_filter(page_reader, _page_skip_plans, leaf_column_id, + _page_skip_profile); + } + const auto level_info = ::parquet::internal::LevelInfo::ComputeLevelInfo(descriptor); + record_readers[leaf_column_id] = ::parquet::internal::RecordReader::Make( + descriptor, level_info, ::arrow::default_memory_pool(), + /*read_dictionary=*/read_dictionary, + /*read_dense_for_nullable=*/false); + record_readers[leaf_column_id]->SetPageReader(std::move(page_reader)); + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to create parquet record reader for column {}: {}", + name, e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to create parquet record reader for column {}: {}", + name, e.what()); + } + } + if (record_readers[leaf_column_id] == nullptr) { + return Status::Corruption("Failed to create parquet record reader for column {}", name); + } + *reader = record_readers[leaf_column_id]; + return Status::OK(); +} + +Status ParquetColumnReaderFactory::create_struct_column_reader( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + std::unique_ptr* reader) const { + if (reader == nullptr) { + return Status::InvalidArgument("reader is null"); + } + std::vector> child_readers; + child_readers.reserve(column_schema.children.size()); + std::vector child_output_indices; + child_output_indices.reserve(column_schema.children.size()); + DataTypes projected_child_types; + Strings projected_child_names; + for (size_t child_idx = 0; child_idx < column_schema.children.size(); ++child_idx) { + const auto& child_schema = column_schema.children[child_idx]; + const auto* child_projection = + format::find_child_projection(projection, child_schema->local_id); + if (!format::is_child_projected(projection, child_schema->local_id)) { + continue; + } + std::unique_ptr child_reader; + RETURN_IF_ERROR( + create_column_reader(*child_schema, child_projection, true, false, &child_reader)); + child_output_indices.push_back(static_cast(projected_child_types.size())); + projected_child_types.push_back(make_nullable(child_reader->type())); + projected_child_names.push_back(child_reader->name()); + child_readers.push_back(std::move(child_reader)); + } + if (format::is_partial_projection(projection) && + projected_child_types.size() != projection->children.size()) { + return Status::InvalidArgument( + "Parquet STRUCT projection for column {} contains invalid child", + column_schema.name); + } + if (projected_child_types.empty() && !column_schema.children.empty()) { + return Status::NotSupported("Parquet STRUCT projection for column {} contains no children", + column_schema.name); + } + DataTypePtr type = column_schema.type; + if (format::is_partial_projection(projection)) { + type = std::make_shared(projected_child_types, projected_child_names); + if (column_schema.type != nullptr && column_schema.type->is_nullable()) { + type = make_nullable(type); + } + } + *reader = std::make_unique( + column_schema, std::move(type), std::move(child_readers), + std::move(child_output_indices), _column_reader_profile); + return Status::OK(); +} + +Status ParquetColumnReaderFactory::create_list_column_reader( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + std::unique_ptr* reader) const { + if (reader == nullptr) { + return Status::InvalidArgument("reader is null"); + } + if (column_schema.children.size() != 1) { + return Status::NotSupported("Unsupported parquet LIST layout for column {}", + column_schema.name); + } + std::unique_ptr element_reader; + const auto& element_schema = *column_schema.children[0]; + const auto* element_projection = + format::find_child_projection(projection, element_schema.local_id); + if (format::is_partial_projection(projection) && element_projection == nullptr) { + return Status::NotSupported("Parquet LIST projection for column {} contains no element", + column_schema.name); + } + RETURN_IF_ERROR( + create_column_reader(element_schema, element_projection, true, false, &element_reader)); + DataTypePtr type = column_schema.type; + if (format::is_partial_projection(element_projection)) { + type = std::make_shared(element_reader->type()); + if (column_schema.type != nullptr && column_schema.type->is_nullable()) { + type = make_nullable(type); + } + } + *reader = std::make_unique(column_schema, std::move(type), + std::move(element_reader), _column_reader_profile); + return Status::OK(); +} + +Status ParquetColumnReaderFactory::create_map_column_reader( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + std::unique_ptr* reader) const { + if (reader == nullptr) { + return Status::InvalidArgument("reader is null"); + } + if (column_schema.children.size() != 2) { + return Status::NotSupported("Unsupported parquet MAP layout for column {}", + column_schema.name); + } + const auto& key_schema = *column_schema.children[0]; + const auto& value_schema = *column_schema.children[1]; + const auto* value_projection = format::find_child_projection(projection, value_schema.local_id); + if (format::is_partial_projection(projection)) { + if (value_projection == nullptr) { + return Status::NotSupported("Parquet MAP projection for column {} contains no value", + column_schema.name); + } + for (const auto& child_projection : projection->children) { + if (child_projection.local_id() == key_schema.local_id) { + continue; + } + if (child_projection.local_id() != value_schema.local_id) { + return Status::InvalidArgument( + "Parquet MAP projection for column {} contains invalid child", + column_schema.name); + } + } + } + std::unique_ptr key_reader; + // MAP materialization always needs the full key stream. It owns entry existence, offsets and + // key equality semantics, so MAP projection is defined only as value-subtree pruning. + RETURN_IF_ERROR(create_column_reader(key_schema, nullptr, true, false, &key_reader)); + std::unique_ptr value_reader; + RETURN_IF_ERROR( + create_column_reader(value_schema, value_projection, true, false, &value_reader)); + DataTypePtr type = column_schema.type; + if (format::is_partial_projection(value_projection)) { + type = std::make_shared(make_nullable(key_reader->type()), + make_nullable(value_reader->type())); + if (column_schema.type != nullptr && column_schema.type->is_nullable()) { + type = make_nullable(type); + } + } + *reader = + std::make_unique(column_schema, std::move(type), std::move(key_reader), + std::move(value_reader), _column_reader_profile); + return Status::OK(); +} + +Status ParquetColumnReaderFactory::create(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::unique_ptr* reader, + bool read_dictionary) const { + return create_column_reader(column_schema, projection, false, read_dictionary, reader); +} + +Status ParquetColumnReaderFactory::create_count_shape_reader( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + std::unique_ptr* reader) const { + return create_count_shape_reader_impl(column_schema, projection, false, reader); +} + +Status ParquetColumnReaderFactory::create_count_shape_reader_impl( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + bool is_nested, std::unique_ptr* reader) const { + if (reader == nullptr) { + return Status::InvalidArgument("reader is null"); + } + switch (column_schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + if (format::is_partial_projection(projection)) { + return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", + column_schema.name); + } + return create_scalar_column_reader(column_schema, is_nested, false, reader); + case ParquetColumnSchemaKind::STRUCT: { + if (column_schema.children.empty()) { + return Status::NotSupported("Parquet COUNT shape reader found empty STRUCT column {}", + column_schema.name); + } + const ParquetColumnSchema* child_schema = nullptr; + const format::LocalColumnIndex* child_projection = nullptr; + if (format::is_partial_projection(projection)) { + const auto child_id = projection->children[0].local_id(); + const auto child_it = std::ranges::find_if( + column_schema.children, + [&](const auto& child) { return child->local_id == child_id; }); + if (child_it == column_schema.children.end()) { + return Status::InvalidArgument( + "Parquet COUNT projection for column {} contains invalid child", + column_schema.name); + } + child_schema = child_it->get(); + child_projection = &projection->children[0]; + } else { + child_schema = column_schema.children[0].get(); + } + DORIS_CHECK(child_schema != nullptr); + return create_count_shape_reader_impl(*child_schema, child_projection, true, reader); + } + case ParquetColumnSchemaKind::LIST: { + if (column_schema.children.size() != 1) { + return Status::NotSupported("Unsupported parquet LIST layout for COUNT column {}", + column_schema.name); + } + const auto& element_schema = *column_schema.children[0]; + const auto* element_projection = + format::find_child_projection(projection, element_schema.local_id); + return create_count_shape_reader_impl(element_schema, element_projection, true, reader); + } + case ParquetColumnSchemaKind::MAP: { + if (column_schema.children.empty()) { + return Status::NotSupported("Unsupported parquet MAP layout for COUNT column {}", + column_schema.name); + } + // The key stream defines MAP entry existence and offsets. Counting top-level MAP NULL-ness + // from it avoids creating a value reader, which is the expensive path for files with huge + // MAP value strings. + return create_count_shape_reader_impl(*column_schema.children[0], nullptr, true, reader); + } + } + return Status::NotSupported("Unsupported parquet column schema kind for COUNT column {}", + column_schema.name); +} + +Status ParquetColumnReaderFactory::create_column_reader( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + bool is_nested, bool read_dictionary, std::unique_ptr* reader) const { + if (reader == nullptr) { + return Status::InvalidArgument("reader is null"); + } + switch (column_schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + if (is_nested) { + if (format::is_partial_projection(projection)) { + return Status::InvalidArgument("Parquet scalar projection is invalid for column {}", + column_schema.name); + } + return create_scalar_column_reader(column_schema, true, false, reader); + } + return create_scalar_column_reader(column_schema, false, read_dictionary, reader); + case ParquetColumnSchemaKind::STRUCT: + return create_struct_column_reader(column_schema, projection, reader); + case ParquetColumnSchemaKind::LIST: + return create_list_column_reader(column_schema, projection, reader); + case ParquetColumnSchemaKind::MAP: + return create_map_column_reader(column_schema, projection, reader); + } + return Status::NotSupported("Unsupported parquet column schema kind for column {}", + column_schema.name); +} + +ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, + ParquetColumnReaderProfile profile) + : _profile(profile), + _field_id(schema.local_id), + _leaf_column_id(schema.leaf_column_id), + _nullable_definition_level(schema.nullable_definition_level), + _repeated_repetition_level(schema.repeated_repetition_level), + _definition_level(schema.definition_level), + _repetition_level(schema.repetition_level), + _repeated_ancestor_definition_level(schema.repeated_ancestor_definition_level), + _type(std::move(type)), + _name(schema.name) {} + +Status ParquetColumnReader::load_nested_batch(int64_t) { + return Status::NotSupported("Parquet nested batch load is not supported for column {}", _name); +} + +Status ParquetColumnReader::load_nested_levels_batch(int64_t) { + return Status::NotSupported("Parquet nested levels batch load is not supported for column {}", + _name); +} + +Status ParquetColumnReader::build_nested_column(int64_t, MutableColumnPtr&, int64_t*) { + return Status::NotSupported("Parquet nested column build is not supported for column {}", + _name); +} + +Status ParquetColumnReader::consume_nested_column(int64_t, int64_t*) { + return Status::NotSupported("Parquet nested column consume is not supported for column {}", + _name); +} + +Status ParquetColumnReader::skip_nested_rows(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + + // A nested parent row may expand to many child values. Capping the number of parent rows per + // loaded batch bounds that amplification for large holes. The consume interface advances the + // loaded definition/repetition levels recursively without constructing a discarded Column. + constexpr int64_t MAX_NESTED_SKIP_BATCH_SIZE = 4096; + int64_t remaining_rows = rows; + while (remaining_rows > 0) { + const int64_t batch_rows = std::min(remaining_rows, MAX_NESTED_SKIP_BATCH_SIZE); + RETURN_IF_ERROR(load_nested_levels_batch(batch_rows)); + int64_t rows_consumed = 0; + RETURN_IF_ERROR(consume_nested_column(batch_rows, &rows_consumed)); + if (rows_consumed != batch_rows) { + return Status::Corruption( + "Failed to skip nested parquet column {}: skipped {} of {} rows in batch", + _name, rows_consumed, batch_rows); + } + remaining_rows -= batch_rows; + } + update_reader_skip_rows(rows); + return Status::OK(); +} + +const std::vector& ParquetColumnReader::nested_definition_levels() const { + static const std::vector empty; + return empty; +} + +const std::vector& ParquetColumnReader::nested_repetition_levels() const { + static const std::vector empty; + return empty; +} + +int64_t ParquetColumnReader::nested_levels_written() const { + return 0; +} + +bool ParquetColumnReader::is_or_has_repeated_child() const { + return _repetition_level > 0; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h new file mode 100644 index 00000000000000..51dbd44c11c226 --- /dev/null +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -0,0 +1,231 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/parquet_type.h" +#include "format_v2/parquet/selection_vector.h" +#include "runtime/runtime_profile.h" + +namespace parquet { +class ColumnDescriptor; +class RowGroupReader; + +namespace internal { +class RecordReader; +} // namespace internal +} // namespace parquet + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { +class IColumn; +} // namespace doris + +namespace doris::format::parquet { +struct ParquetColumnSchema; + +class ParquetColumnReader { +public: + virtual ~ParquetColumnReader() = default; + + virtual int file_column_id() const { return _field_id; } + + virtual int parquet_leaf_column_id() const { return _leaf_column_id; } + + int16_t nullable_definition_level() const { return _nullable_definition_level; } + int16_t repeated_repetition_level() const { return _repeated_repetition_level; } + + virtual const DataTypePtr& type() const { return _type; } + virtual const std::string& name() const { return _name; } + const ParquetColumnReaderProfile& profile() const { return _profile; } + + virtual Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) = 0; + + virtual Status skip(int64_t rows); + + virtual Status select(const SelectionVector& sel, uint16_t selected_rows, int64_t batch_rows, + MutableColumnPtr& column); + + virtual Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, + int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, bool* used_filter); + + virtual Status load_nested_batch(int64_t rows); + + // Shape-only load interface for COUNT(col) and skip. Implementations guarantee only that + // nested_definition_levels(), nested_repetition_levels(), and nested_levels_written() are + // available; value indices and payload columns may be absent. Callers may inspect the levels or + // call consume_nested_column(), but must not call build_nested_column() afterwards. For example, + // skipping ARRAY uses this method to find ARRAY boundaries without constructing a + // ColumnString. The underlying Arrow reader may still decode a page because it has no public + // levels-only API. Normal scans that need output values use load_nested_batch() instead. + virtual Status load_nested_levels_batch(int64_t rows); + + virtual Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read); + + // Consume logical values from a batch previously loaded by load_nested_batch() or + // load_nested_levels_batch() without appending them to an output Column. Implementations must + // advance exactly the same nested level cursors and perform the same shape/null/alignment + // validation as build_nested_column(). The levels-only form is preferred for skip paths because + // it avoids transferring leaf payloads into Doris Columns when they will be discarded. + // + // `length_upper_bound` is expressed at this reader's logical level, not in physical leaf + // values. For example, consuming two rows from ARRAY [[1, 2], []] consumes two parent ARRAY + // rows but only two element values. A MAP implementation must also consume key/value streams + // in lockstep, while a nullable STRUCT consumes no child value for a null parent. + // + // Callers must not use the ordinary skip() after either load call: the leaf stream has already + // advanced into an in-memory nested batch, and doing so would advance it twice. + // `values_consumed` may be smaller than the requested bound only when the loaded batch ends. + virtual Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed); + + virtual const std::vector& nested_definition_levels() const; + virtual const std::vector& nested_repetition_levels() const; + virtual int64_t nested_levels_written() const; + virtual bool is_or_has_repeated_child() const; + virtual void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level); + + int64_t nested_build_level_cursor() const { return _nested_build_level_cursor; } + void set_nested_build_level_cursor(int64_t cursor) { + DORIS_CHECK(cursor >= 0); + _nested_build_level_cursor = cursor; + } + void reset_nested_build_level_cursor() { _nested_build_level_cursor = 0; } + +protected: + ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, + ParquetColumnReaderProfile profile = {}); + ParquetColumnReader() = default; + // Load shape levels and consume skipped parent rows in bounded batches. The bound limits level + // memory when a parent expands to many children; the levels-only load plus + // consume_nested_column() avoids payload materialization and output Columns. + Status skip_nested_rows(int64_t rows); + void update_reader_read_rows(int64_t rows) const; + void update_reader_skip_rows(int64_t rows) const; + + ParquetColumnReaderProfile _profile; + const int _field_id = -1; // child ordinal in the parent node + const int _leaf_column_id = -1; // Parquet physical leaf column id (-1 = non-leaf) + const int16_t _nullable_definition_level = + 0; // definition-level threshold where this node becomes nullable + const int16_t _repeated_repetition_level = + 0; // repetition level of the nearest repeated ancestor + const int16_t _definition_level = 0; // definition level accumulated to this node + const int16_t _repetition_level = 0; // repetition level accumulated to this node + const int16_t _repeated_ancestor_definition_level = + 0; // definition level of the nearest repeated ancestor + const DataTypePtr _type; // Doris target type + const std::string _name; // column name for error messages + int64_t _nested_build_level_cursor = 0; // nested build cursor (current level position) +}; + +class ParquetColumnReaderFactory { +public: + ParquetColumnReaderFactory(std::shared_ptr<::parquet::RowGroupReader> row_group, + int num_leaf_columns, + const std::map* page_skip_plans = nullptr, + ParquetPageSkipProfile page_skip_profile = {}, + const cctz::time_zone* timezone = nullptr, + bool enable_strict_mode = false, + ParquetColumnReaderProfile column_reader_profile = {}); + + Status create(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::unique_ptr* reader, bool read_dictionary = false) const; + + // Create a scalar reader for one representative leaf that carries the top-level column shape. + // This is used by COUNT(col): the caller needs definition/repetition levels to decide whether + // the top-level value is NULL, but must not materialize heavy payload leaves. MAP deliberately + // uses the key leaf because the key stream owns entry existence and avoids reading value pages. + Status create_count_shape_reader(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::unique_ptr* reader) const; + + Status create(const ParquetColumnSchema& column_schema, + std::unique_ptr* reader) const { + return create(column_schema, nullptr, reader); + } + + std::unique_ptr create_row_position_column_reader( + int64_t row_group_first_row) const; + std::unique_ptr create_global_rowid_column_reader( + const format::GlobalRowIdContext& context, int64_t row_group_first_row) const; + +private: + Status create_scalar_column_reader(const ParquetColumnSchema& column_schema, bool is_nested, + bool read_dictionary, + std::unique_ptr* reader) const; + + Status create_struct_column_reader(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::unique_ptr* reader) const; + + Status create_list_column_reader(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::unique_ptr* reader) const; + + Status create_map_column_reader(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::unique_ptr* reader) const; + + Status create_column_reader(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, bool is_nested, + bool read_dictionary, + std::unique_ptr* reader) const; + Status create_count_shape_reader_impl(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + bool is_nested, + std::unique_ptr* reader) const; + + Status get_record_reader(int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, + const std::string& name, bool install_page_filter, + bool read_dictionary, + std::shared_ptr<::parquet::internal::RecordReader>* reader) const; + + Status make_scalar_column_reader( + const ParquetColumnSchema& column_schema, + std::shared_ptr<::parquet::internal::RecordReader> record_reader, + bool use_page_skip_plan, std::unique_ptr* reader) const; + + std::shared_ptr<::parquet::RowGroupReader> _row_group; // Arrow RowGroup reader + mutable std::vector> + _record_readers; // RecordReader cache by leaf_column_id + mutable std::vector> + _dictionary_record_readers; // dictionary-exposing RecordReader cache by leaf_column_id + const std::map* _page_skip_plans = + nullptr; // page-index pruning result + ParquetPageSkipProfile _page_skip_profile; // page skip profile + const cctz::time_zone* _timezone = nullptr; // timezone + bool _enable_strict_mode = false; // strict mode + ParquetColumnReaderProfile _column_reader_profile; // column reader profile +}; +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/global_rowid_column_reader.cpp b/be/src/format_v2/parquet/reader/global_rowid_column_reader.cpp new file mode 100644 index 00000000000000..82b2838ba2cbfe --- /dev/null +++ b/be/src/format_v2/parquet/reader/global_rowid_column_reader.cpp @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/global_rowid_column_reader.h" + +#include + +#include "common/cast_set.h" +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_string.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "storage/utils.h" + +namespace doris::format::parquet { + +GlobalRowIdColumnReader::GlobalRowIdColumnReader(format::GlobalRowIdContext context, + int64_t row_group_first_row, + ParquetColumnReaderProfile profile) + : ParquetColumnReader(ParquetColumnSchema {.name = BeConsts::GLOBAL_ROWID_COL}, + std::make_shared(), profile), + _context(context), + _row_group_first_row(row_group_first_row) {} + +int GlobalRowIdColumnReader::file_column_id() const { + return format::GLOBAL_ROWID_COLUMN_ID; +} + +int GlobalRowIdColumnReader::parquet_leaf_column_id() const { + return -1; +} + +const DataTypePtr& GlobalRowIdColumnReader::type() const { + return _type; +} + +const std::string& GlobalRowIdColumnReader::name() const { + return _name; +} + +Status GlobalRowIdColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + if (column.get() == nullptr || rows_read == nullptr) { + return Status::InvalidArgument("Invalid parquet global rowid read result pointer"); + } + if (rows < 0) { + return Status::InvalidArgument("Invalid parquet global rowid read rows {}", rows); + } + for (int64_t row = 0; row < rows; ++row) { + append_row_id(cast_set(_row_group_first_row + _next_row_position + row), column); + } + _next_row_position += rows; + *rows_read = rows; + return Status::OK(); +} + +Status GlobalRowIdColumnReader::skip(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + _next_row_position += rows; + return Status::OK(); +} + +void GlobalRowIdColumnReader::append_row_id(uint32_t row_id, MutableColumnPtr& column) const { + auto* string_column = assert_cast(column.get()); + GlobalRowLoacationV2 location(_context.version, _context.backend_id, _context.file_id, row_id); + string_column->insert_data(reinterpret_cast(&location), + sizeof(GlobalRowLoacationV2)); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/global_rowid_column_reader.h b/be/src/format_v2/parquet/reader/global_rowid_column_reader.h new file mode 100644 index 00000000000000..b3f71645923010 --- /dev/null +++ b/be/src/format_v2/parquet/reader/global_rowid_column_reader.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "format_v2/column_data.h" +#include "format_v2/parquet/reader/column_reader.h" + +namespace doris::format::parquet { + +class GlobalRowIdColumnReader final : public ParquetColumnReader { +public: + GlobalRowIdColumnReader(format::GlobalRowIdContext context, int64_t row_group_first_row, + ParquetColumnReaderProfile profile = {}); + + int file_column_id() const override; + int parquet_leaf_column_id() const override; + const DataTypePtr& type() const override; + const std::string& name() const override; + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + +private: + void append_row_id(uint32_t row_id, MutableColumnPtr& column) const; + + format::GlobalRowIdContext _context; // RowId prefix (version + backend_id + file_id) + int64_t _row_group_first_row = 0; // first file row of the current row group + int64_t _next_row_position = 0; // next row position to emit +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.cpp b/be/src/format_v2/parquet/reader/list_column_reader.cpp new file mode 100644 index 00000000000000..c042fc99b512aa --- /dev/null +++ b/be/src/format_v2/parquet/reader/list_column_reader.cpp @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/list_column_reader.h" + +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" +#include "format_v2/parquet/reader/nested_column_materializer.h" + +namespace doris::format::parquet { +namespace { + +void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, + MutableColumnPtr* column) { + DORIS_CHECK(column != nullptr); + if (output_type->is_nullable()) { + return; + } + if (auto* nullable_column = check_and_get_column(**column)) { + *column = nullable_column->get_nested_column_ptr(); + } +} + +} // namespace + +Status ListColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + RETURN_IF_ERROR(load_nested_batch(rows)); + return build_nested_column(rows, column, rows_read); +} + +Status ListColumnReader::skip(int64_t rows) { + return skip_nested_rows(rows); +} + +Status ListColumnReader::load_nested_batch(int64_t rows) { + DORIS_CHECK(_element_reader != nullptr); + reset_nested_build_level_cursor(); + return _element_reader->load_nested_batch(rows); +} + +Status ListColumnReader::load_nested_levels_batch(int64_t rows) { + DORIS_CHECK(_element_reader != nullptr); + reset_nested_build_level_cursor(); + return _element_reader->load_nested_levels_batch(rows); +} + +Status ListColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) { + if (column.get() == nullptr) { + return Status::InvalidArgument("Invalid parquet list build result pointer for column {}", + _name); + } + return _consume_or_build_nested_column(length_upper_bound, &column, values_read); +} + +Status ListColumnReader::consume_nested_column(int64_t length_upper_bound, + int64_t* values_consumed) { + return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); +} + +Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, + MutableColumnPtr* column, + int64_t* values_processed) { + if (values_processed == nullptr) { + return Status::InvalidArgument("Invalid parquet list process result pointer for column {}", + _name); + } + DORIS_CHECK(_element_reader != nullptr); + ColumnArray* array_column = nullptr; + NullMap* parent_null_map = nullptr; + MutableColumnPtr nested_column; + if (column != nullptr) { + array_column = array_column_from_output(*column); + DORIS_CHECK(array_column != nullptr); + parent_null_map = null_map_from_nullable_output(*column); + nested_column = array_column->get_data_ptr()->assert_mutable(); + const auto& element_output_type = + assert_cast(*remove_nullable(_type)).get_nested_type(); + remove_nullable_wrapper_if_not_expected(element_output_type, &nested_column); + } + + const auto& def_levels = _element_reader->nested_definition_levels(); + const auto& rep_levels = _element_reader->nested_repetition_levels(); + const int64_t levels_written = _element_reader->nested_levels_written(); + std::vector entry_counts; + NullMap parent_nulls; + *values_processed = 0; + int64_t level_idx = nested_build_level_cursor(); + const int16_t min_parent_definition_level = + static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); + while (level_idx < levels_written) { + const int16_t def_level = def_levels[level_idx]; + const int16_t rep_level = rep_levels[level_idx]; + const bool starts_parent = rep_level < _repetition_level; + if (starts_parent && *values_processed >= length_upper_bound) { + break; + } + ++level_idx; + if (rep_level > _repetition_level || def_level < min_parent_definition_level || + (!starts_parent && def_level < _repeated_ancestor_definition_level)) { + continue; + } + if (rep_level == _repetition_level) { + if (entry_counts.empty()) { + return Status::Corruption("Invalid repeated level for parquet LIST column {}", + _name); + } + if (def_level >= _definition_level) { + ++entry_counts.back(); + } + continue; + } + + const bool parent_is_null = def_level < _definition_level - 1; + if (parent_is_null && !_type->is_nullable()) { + return Status::Corruption("Parquet LIST column {} contains null for non-nullable LIST", + _name); + } + parent_nulls.push_back(parent_is_null); + entry_counts.push_back(def_level >= _definition_level ? 1 : 0); + ++*values_processed; + } + set_nested_build_level_cursor(level_idx); + + uint64_t total_entries = 0; + int64_t child_value_count = 0; + if (!_element_reader->is_or_has_repeated_child()) { + for (const auto entry_count : entry_counts) { + total_entries += entry_count; + } + if (column != nullptr) { + RETURN_IF_ERROR(_element_reader->build_nested_column( + static_cast(total_entries), nested_column, &child_value_count)); + } else { + RETURN_IF_ERROR(_element_reader->consume_nested_column( + static_cast(total_entries), &child_value_count)); + } + } else { + uint64_t pending_entries = 0; + auto flush_pending_entries = [&]() -> Status { + if (pending_entries == 0) { + return Status::OK(); + } + int64_t span_child_value_count = 0; + if (column != nullptr) { + RETURN_IF_ERROR(_element_reader->build_nested_column( + static_cast(pending_entries), nested_column, + &span_child_value_count)); + } else { + RETURN_IF_ERROR(_element_reader->consume_nested_column( + static_cast(pending_entries), &span_child_value_count)); + } + if (span_child_value_count != static_cast(pending_entries)) { + return Status::Corruption( + "Parquet LIST column {} built {} child values, expected {}", _name, + span_child_value_count, pending_entries); + } + child_value_count += span_child_value_count; + pending_entries = 0; + return Status::OK(); + }; + + for (const auto entry_count : entry_counts) { + total_entries += entry_count; + if (entry_count > 0) { + pending_entries += entry_count; + continue; + } + RETURN_IF_ERROR(flush_pending_entries()); + _element_reader->advance_nested_build_level_cursor_past_parent(_repetition_level); + } + RETURN_IF_ERROR(flush_pending_entries()); + } + if (child_value_count != static_cast(total_entries)) { + return Status::Corruption("Parquet LIST column {} built {} child values, expected {}", + _name, child_value_count, total_entries); + } + if (column != nullptr) { + array_column->get_data_ptr() = std::move(nested_column); + append_offsets(array_column->get_offsets(), entry_counts); + append_parent_nulls(parent_null_map, parent_nulls); + } + return Status::OK(); +} + +const std::vector& ListColumnReader::nested_definition_levels() const { + DORIS_CHECK(_element_reader != nullptr); + return _element_reader->nested_definition_levels(); +} + +const std::vector& ListColumnReader::nested_repetition_levels() const { + DORIS_CHECK(_element_reader != nullptr); + return _element_reader->nested_repetition_levels(); +} + +int64_t ListColumnReader::nested_levels_written() const { + DORIS_CHECK(_element_reader != nullptr); + return _element_reader->nested_levels_written(); +} + +bool ListColumnReader::is_or_has_repeated_child() const { + return true; +} + +void ListColumnReader::advance_nested_build_level_cursor_past_parent( + int16_t parent_repetition_level) { + DORIS_CHECK(_element_reader != nullptr); + ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); + _element_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.h b/be/src/format_v2/parquet/reader/list_column_reader.h new file mode 100644 index 00000000000000..d64be546e394d7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/list_column_reader.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/reader/column_reader.h" + +namespace doris::format::parquet { + +class ListColumnReader final : public ParquetColumnReader { +public: + ListColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, + std::unique_ptr element_reader, + ParquetColumnReaderProfile profile = {}) + : ParquetColumnReader(schema, type, profile), + _element_reader(std::move(element_reader)) {} + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status load_nested_batch(int64_t rows) override; + Status load_nested_levels_batch(int64_t rows) override; + Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) override; + Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; + const std::vector& nested_definition_levels() const override; + const std::vector& nested_repetition_levels() const override; + int64_t nested_levels_written() const override; + bool is_or_has_repeated_child() const override; + void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; + +private: + Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, + int64_t* values_processed); + + std::unique_ptr + _element_reader; // element reader (recursive; may be Scalar/Struct/List/Map) +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.cpp b/be/src/format_v2/parquet/reader/map_column_reader.cpp new file mode 100644 index 00000000000000..8217d0c013abc0 --- /dev/null +++ b/be/src/format_v2/parquet/reader/map_column_reader.cpp @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/map_column_reader.h" + +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "format_v2/parquet/reader/nested_column_materializer.h" +#include "format_v2/parquet/reader/scalar_column_reader.h" + +namespace doris::format::parquet { +namespace { + +void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, + MutableColumnPtr* column) { + DORIS_CHECK(column != nullptr); + if (output_type->is_nullable()) { + return; + } + if (auto* nullable_column = check_and_get_column(**column)) { + *column = nullable_column->get_nested_column_ptr(); + } +} + +} // namespace + +Status MapColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + RETURN_IF_ERROR(load_nested_batch(rows)); + return build_nested_column(rows, column, rows_read); +} + +Status MapColumnReader::skip(int64_t rows) { + return skip_nested_rows(rows); +} + +Status MapColumnReader::load_nested_batch(int64_t rows) { + DORIS_CHECK(_key_reader != nullptr); + DORIS_CHECK(_value_reader != nullptr); + reset_nested_build_level_cursor(); + RETURN_IF_ERROR(_key_reader->load_nested_batch(rows)); + return _value_reader->load_nested_batch(rows); +} + +Status MapColumnReader::load_nested_levels_batch(int64_t rows) { + DORIS_CHECK(_key_reader != nullptr); + DORIS_CHECK(_value_reader != nullptr); + reset_nested_build_level_cursor(); + RETURN_IF_ERROR(_key_reader->load_nested_levels_batch(rows)); + return _value_reader->load_nested_levels_batch(rows); +} + +Status MapColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) { + if (column.get() == nullptr) { + return Status::InvalidArgument("Invalid parquet map build result pointer for column {}", + _name); + } + return _consume_or_build_nested_column(length_upper_bound, &column, values_read); +} + +Status MapColumnReader::consume_nested_column(int64_t length_upper_bound, + int64_t* values_consumed) { + return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); +} + +Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, + MutableColumnPtr* column, + int64_t* values_processed) { + if (values_processed == nullptr) { + return Status::InvalidArgument("Invalid parquet map process result pointer for column {}", + _name); + } + DORIS_CHECK(_key_reader != nullptr); + DORIS_CHECK(_value_reader != nullptr); + ColumnMap* map_column = nullptr; + NullMap* parent_null_map = nullptr; + MutableColumnPtr key_column; + MutableColumnPtr value_column; + if (column != nullptr) { + map_column = map_column_from_output(*column); + DORIS_CHECK(map_column != nullptr); + parent_null_map = null_map_from_nullable_output(*column); + key_column = map_column->get_keys_ptr()->assert_mutable(); + value_column = map_column->get_values_ptr()->assert_mutable(); + const auto& map_output_type = assert_cast(*remove_nullable(_type)); + remove_nullable_wrapper_if_not_expected(map_output_type.get_key_type(), &key_column); + remove_nullable_wrapper_if_not_expected(map_output_type.get_value_type(), &value_column); + } + + const auto& def_levels = _key_reader->nested_definition_levels(); + const auto& rep_levels = _key_reader->nested_repetition_levels(); + const int64_t levels_written = _key_reader->nested_levels_written(); + + std::vector entry_counts; + std::vector map_level_indices; + NullMap parent_nulls; + *values_processed = 0; + int64_t level_idx = nested_build_level_cursor(); + const int16_t min_parent_definition_level = + static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); + while (level_idx < levels_written) { + const int16_t def_level = def_levels[level_idx]; + const int16_t rep_level = rep_levels[level_idx]; + const bool starts_parent = rep_level < _repetition_level; + if (starts_parent && *values_processed >= length_upper_bound) { + break; + } + const int64_t current_level_idx = level_idx; + ++level_idx; + if (rep_level > _repetition_level || def_level < min_parent_definition_level || + (!starts_parent && def_level < _repeated_ancestor_definition_level)) { + continue; + } + map_level_indices.push_back(current_level_idx); + if (rep_level == _repetition_level) { + if (entry_counts.empty()) { + return Status::Corruption("Invalid repeated level for parquet MAP column {}", + _name); + } + if (def_level >= _definition_level) { + ++entry_counts.back(); + } + continue; + } + + const bool parent_is_null = def_level < _definition_level - 1; + if (parent_is_null && !_type->is_nullable()) { + return Status::Corruption("Parquet MAP column {} contains null for non-nullable MAP", + _name); + } + parent_nulls.push_back(parent_is_null); + entry_counts.push_back(def_level >= _definition_level ? 1 : 0); + ++*values_processed; + } + set_nested_build_level_cursor(level_idx); + + uint64_t total_entries = 0; + for (const auto entry_count : entry_counts) { + total_entries += entry_count; + } + int64_t key_value_count = 0; + size_t key_start = 0; + if (column != nullptr) { + key_start = key_column->size(); + RETURN_IF_ERROR(_key_reader->build_nested_column(static_cast(total_entries), + key_column, &key_value_count)); + } else if (auto* scalar_key_reader = dynamic_cast(_key_reader.get())) { + // MAP keys are required even if a projected Doris key type is nullable. Validate each + // actual entry directly from the key level stream while advancing past empty/null maps. + for (const int64_t key_level_idx : map_level_indices) { + if (def_levels[key_level_idx] >= _definition_level) { + RETURN_IF_ERROR(scalar_key_reader->validate_nested_value(key_level_idx, true)); + ++key_value_count; + } + } + scalar_key_reader->set_nested_build_level_cursor(level_idx); + } else { + RETURN_IF_ERROR(_key_reader->consume_nested_column(static_cast(total_entries), + &key_value_count)); + } + if (key_value_count != static_cast(total_entries)) { + return Status::Corruption("Parquet MAP column {} built {} keys, expected {}", _name, + key_value_count, total_entries); + } + if (column != nullptr) { + if (const auto* nullable_key_column = check_and_get_column(*key_column); + nullable_key_column != nullptr && + nullable_key_column->has_null(key_start, nullable_key_column->size())) { + return Status::Corruption("Parquet MAP column {} contains null key", _name); + } + } + int64_t value_count = 0; + if (auto* scalar_value_reader = dynamic_cast(_value_reader.get())) { + const auto& value_def_levels = scalar_value_reader->nested_definition_levels(); + const auto& value_rep_levels = scalar_value_reader->nested_repetition_levels(); + const int64_t value_levels_written = scalar_value_reader->nested_levels_written(); + int64_t value_level_idx = scalar_value_reader->nested_build_level_cursor(); + for (const int64_t key_level_idx : map_level_indices) { + while (value_level_idx < value_levels_written && + (value_rep_levels[value_level_idx] > _repetition_level || + value_def_levels[value_level_idx] < min_parent_definition_level || + (value_rep_levels[value_level_idx] >= _repetition_level && + value_def_levels[value_level_idx] < _repeated_ancestor_definition_level))) { + ++value_level_idx; + } + if (value_level_idx >= value_levels_written) { + return Status::Corruption( + "Parquet MAP column {} value stream ended before key stream", _name); + } + // MAP is encoded as a repeated key/value struct. The key stream owns entry existence, + // but the value stream still has one shape slot for every consumed MAP slot. Consume + // value slots in lockstep with key slots so shape-only slots from empty/null maps do + // not become scalar values. + if (value_rep_levels[value_level_idx] != rep_levels[key_level_idx]) { + return Status::Corruption( + "Parquet MAP column {} value repetition level is not aligned with key " + "stream", + _name); + } + if (def_levels[key_level_idx] >= _definition_level) { + if (column != nullptr) { + RETURN_IF_ERROR(scalar_value_reader->append_nested_value(value_level_idx, + value_column)); + } else { + RETURN_IF_ERROR( + scalar_value_reader->validate_nested_value(value_level_idx, false)); + } + ++value_count; + } + ++value_level_idx; + } + scalar_value_reader->set_nested_build_level_cursor(value_level_idx); + } else { + // Complex MAP values own their nested shape below the entry slot, so they recursively + // process exactly one child value for each MAP entry. + if (column != nullptr) { + RETURN_IF_ERROR(_value_reader->build_nested_column(static_cast(total_entries), + value_column, &value_count)); + } else { + RETURN_IF_ERROR(_value_reader->consume_nested_column( + static_cast(total_entries), &value_count)); + } + } + if (value_count != static_cast(total_entries)) { + return Status::Corruption("Parquet MAP column {} built {} values, expected {}", _name, + value_count, total_entries); + } + + if (column != nullptr) { + map_column->get_keys_ptr() = std::move(key_column); + map_column->get_values_ptr() = std::move(value_column); + append_offsets(map_column->get_offsets(), entry_counts); + append_parent_nulls(parent_null_map, parent_nulls); + } + return Status::OK(); +} + +const std::vector& MapColumnReader::nested_definition_levels() const { + DORIS_CHECK(_key_reader != nullptr); + return _key_reader->nested_definition_levels(); +} + +const std::vector& MapColumnReader::nested_repetition_levels() const { + DORIS_CHECK(_key_reader != nullptr); + return _key_reader->nested_repetition_levels(); +} + +int64_t MapColumnReader::nested_levels_written() const { + DORIS_CHECK(_key_reader != nullptr); + return _key_reader->nested_levels_written(); +} + +bool MapColumnReader::is_or_has_repeated_child() const { + return true; +} + +void MapColumnReader::advance_nested_build_level_cursor_past_parent( + int16_t parent_repetition_level) { + DORIS_CHECK(_key_reader != nullptr); + DORIS_CHECK(_value_reader != nullptr); + ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); + _key_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); + _value_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.h b/be/src/format_v2/parquet/reader/map_column_reader.h new file mode 100644 index 00000000000000..1a8ca9c70d8c5b --- /dev/null +++ b/be/src/format_v2/parquet/reader/map_column_reader.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/reader/column_reader.h" + +namespace doris::format::parquet { + +// 2. build_nested_column() -> +class MapColumnReader final : public ParquetColumnReader { +public: + MapColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, + std::unique_ptr key_reader, + std::unique_ptr value_reader, + ParquetColumnReaderProfile profile = {}) + : ParquetColumnReader(schema, type, profile), + _key_reader(std::move(key_reader)), + _value_reader(std::move(value_reader)) {} + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status load_nested_batch(int64_t rows) override; + Status load_nested_levels_batch(int64_t rows) override; + Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) override; + Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; + const std::vector& nested_definition_levels() const override; + const std::vector& nested_repetition_levels() const override; + int64_t nested_levels_written() const override; + bool is_or_has_repeated_child() const override; + void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; + +private: + Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, + int64_t* values_processed); + + std::unique_ptr _key_reader; // key column reader (always read fully) + std::unique_ptr + _value_reader; // value column reader (can be pruned by projection) +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp b/be/src/format_v2/parquet/reader/nested_column_materializer.cpp new file mode 100644 index 00000000000000..e06b7eaaf317e7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/nested_column_materializer.cpp @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/nested_column_materializer.h" + +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" + +namespace doris::format::parquet { + +ColumnArray* array_column_from_output(MutableColumnPtr& column) { + if (auto* nullable_column = check_and_get_column(*column)) { + return assert_cast(&nullable_column->get_nested_column()); + } + return assert_cast(column.get()); +} + +ColumnMap* map_column_from_output(MutableColumnPtr& column) { + if (auto* nullable_column = check_and_get_column(*column)) { + return assert_cast(&nullable_column->get_nested_column()); + } + return assert_cast(column.get()); +} + +ColumnStruct* struct_column_from_output(MutableColumnPtr& column) { + if (auto* nullable_column = check_and_get_column(*column)) { + return assert_cast(&nullable_column->get_nested_column()); + } + return assert_cast(column.get()); +} + +NullMap* null_map_from_nullable_output(MutableColumnPtr& column) { + if (auto* nullable_column = check_and_get_column(*column)) { + return &nullable_column->get_null_map_data(); + } + return nullptr; +} + +void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts) { + offsets.reserve(offsets.size() + entry_counts.size()); + uint64_t current_offset = offsets.empty() ? 0 : offsets.back(); + for (const auto entry_count : entry_counts) { + current_offset += entry_count; + offsets.push_back(current_offset); + } +} + +void append_parent_nulls(NullMap* dst, const NullMap& src) { + if (dst == nullptr) { + return; // target column is not nullable; no null marker is needed + } + dst->insert(src.begin(), src.end()); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.h b/be/src/format_v2/parquet/reader/nested_column_materializer.h new file mode 100644 index 00000000000000..90fac01eb2f5e5 --- /dev/null +++ b/be/src/format_v2/parquet/reader/nested_column_materializer.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "core/column/column.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" + +namespace doris::format::parquet { + +// ============================================================================ +// ============================================================================ + +ColumnArray* array_column_from_output(MutableColumnPtr& column); + +ColumnMap* map_column_from_output(MutableColumnPtr& column); + +ColumnStruct* struct_column_from_output(MutableColumnPtr& column); + +NullMap* null_map_from_nullable_output(MutableColumnPtr& column); + +// offsets[i] = offsets[i-1] + entry_counts[i]. +void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts); + +void append_parent_nulls(NullMap* dst, const NullMap& src); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp new file mode 100644 index 00000000000000..fd261ef5219d27 --- /dev/null +++ b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp @@ -0,0 +1,803 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/parquet_leaf_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/data_type/data_type_nullable.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "core/string_ref.h" +#include "runtime/runtime_profile.h" +#include "util/simd/bits.h" + +namespace doris::format::parquet { +namespace { + +DecodedTimeUnit decoded_time_unit(ParquetTimeUnit time_unit) { + switch (time_unit) { + case ParquetTimeUnit::MILLIS: + return DecodedTimeUnit::MILLIS; + case ParquetTimeUnit::MICROS: + return DecodedTimeUnit::MICROS; + case ParquetTimeUnit::NANOS: + return DecodedTimeUnit::NANOS; + case ParquetTimeUnit::UNKNOWN: + default: + return DecodedTimeUnit::UNKNOWN; + } +} + +Status decoded_fixed_value_size(const std::string& column_name, DecodedValueKind value_kind, + size_t* value_size) { + switch (value_kind) { + case DecodedValueKind::BOOL: + *value_size = sizeof(bool); + return Status::OK(); + case DecodedValueKind::INT32: + *value_size = sizeof(int32_t); + return Status::OK(); + case DecodedValueKind::UINT32: + *value_size = sizeof(uint32_t); + return Status::OK(); + case DecodedValueKind::INT64: + *value_size = sizeof(int64_t); + return Status::OK(); + case DecodedValueKind::UINT64: + *value_size = sizeof(uint64_t); + return Status::OK(); + case DecodedValueKind::INT96: + *value_size = 12; + return Status::OK(); + case DecodedValueKind::FLOAT: + *value_size = sizeof(float); + return Status::OK(); + case DecodedValueKind::DOUBLE: + *value_size = sizeof(double); + return Status::OK(); + case DecodedValueKind::BINARY: + case DecodedValueKind::FIXED_BINARY: + return Status::InvalidArgument("Parquet binary value kind has no fixed value size for {}", + column_name); + } + return Status::InternalError("Unknown decoded value kind for column {}", column_name); +} + +Status get_binary_chunks(const std::string& column_name, + ::parquet::internal::RecordReader& record_reader, + std::vector>* chunks) { + if (auto* dictionary_reader = + dynamic_cast<::parquet::internal::DictionaryRecordReader*>(&record_reader); + dictionary_reader != nullptr) { + auto chunked = dictionary_reader->GetResult(); + if (chunked == nullptr) { + return Status::Corruption( + "Parquet dictionary record reader returned null result for column {}", + column_name); + } + *chunks = chunked->chunks(); + return Status::OK(); + } + auto* binary_reader = dynamic_cast<::parquet::internal::BinaryRecordReader*>(&record_reader); + if (binary_reader == nullptr) { + return Status::InternalError("Parquet binary record reader is not available for column {}", + column_name); + } + *chunks = binary_reader->GetBuilderChunks(); + return Status::OK(); +} + +Status append_dictionary_binary_values(const std::string& column_name, + const ::arrow::DictionaryArray& dictionary_array, + std::vector* values) { + DORIS_CHECK(values != nullptr); + const auto& dictionary = dictionary_array.dictionary(); + if (dictionary == nullptr) { + return Status::Corruption("Parquet dictionary array has null dictionary for column {}", + column_name); + } + auto append_value = [&](int64_t dictionary_index) -> Status { + if (dictionary_index < 0 || dictionary_index >= dictionary->length()) { + return Status::Corruption("Invalid parquet dictionary index {} for column {}", + dictionary_index, column_name); + } + if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(dictionary.get())) { + if (binary_array->IsNull(dictionary_index)) { + values->emplace_back(static_cast(nullptr), 0); + return Status::OK(); + } + int32_t length = 0; + const uint8_t* value = binary_array->GetValue(dictionary_index, &length); + values->emplace_back(reinterpret_cast(value), length); + return Status::OK(); + } + if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(dictionary.get())) { + if (fixed_array->IsNull(dictionary_index)) { + values->emplace_back(static_cast(nullptr), 0); + return Status::OK(); + } + values->emplace_back( + reinterpret_cast(fixed_array->GetValue(dictionary_index)), + fixed_array->byte_width()); + return Status::OK(); + } + return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", + column_name); + }; + for (int64_t row_idx = 0; row_idx < dictionary_array.length(); ++row_idx) { + if (dictionary_array.IsNull(row_idx)) { + values->emplace_back(static_cast(nullptr), 0); + continue; + } + RETURN_IF_ERROR(append_value(dictionary_array.GetValueIndex(row_idx))); + } + return Status::OK(); +} + +Status build_binary_values(const std::string& column_name, + const std::vector>& chunks, + int64_t records_read, const NullMap* null_map, + bool read_dense_for_nullable, std::vector* binary_values) { + std::vector compact_values; + auto* values = read_dense_for_nullable ? &compact_values : binary_values; + values->reserve(records_read); + for (const auto& chunk : chunks) { + if (chunk == nullptr) { + return Status::Corruption( + "Parquet binary record reader returned null chunk for column {}", column_name); + } + if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(chunk.get())) { + for (int64_t row_idx = 0; row_idx < binary_array->length(); ++row_idx) { + if (binary_array->IsNull(row_idx)) { + values->emplace_back(static_cast(nullptr), 0); + continue; + } + int32_t length = 0; + const uint8_t* value = binary_array->GetValue(row_idx, &length); + values->emplace_back(reinterpret_cast(value), length); + } + } else if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(chunk.get())) { + for (int64_t row_idx = 0; row_idx < fixed_array->length(); ++row_idx) { + if (fixed_array->IsNull(row_idx)) { + values->emplace_back(static_cast(nullptr), 0); + continue; + } + values->emplace_back(reinterpret_cast(fixed_array->GetValue(row_idx)), + fixed_array->byte_width()); + } + } else if (auto* dictionary_array = dynamic_cast<::arrow::DictionaryArray*>(chunk.get())) { + RETURN_IF_ERROR( + append_dictionary_binary_values(column_name, *dictionary_array, values)); + } else { + return Status::InternalError("Unexpected Arrow binary array type for column {}", + column_name); + } + } + if (read_dense_for_nullable) { + if (null_map == nullptr || null_map->size() != static_cast(records_read)) { + return Status::Corruption( + "Invalid dense nullable parquet null map for column {}: rows={}, null_map={}", + column_name, records_read, null_map == nullptr ? 0 : null_map->size()); + } + const int64_t non_null_count = static_cast(simd::count_zero_num( + reinterpret_cast(null_map->data()), null_map->size())); + if (compact_values.size() != static_cast(non_null_count)) { + return Status::Corruption( + "Invalid dense nullable parquet binary values for column {}: values={}, " + "records={}, nulls={}", + column_name, compact_values.size(), records_read, + records_read - non_null_count); + } + binary_values->reserve(records_read); + size_t value_idx = 0; + for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { + if ((*null_map)[record_idx] != 0) { + binary_values->emplace_back(static_cast(nullptr), 0); + continue; + } + binary_values->emplace_back(compact_values[value_idx++]); + } + return Status::OK(); + } + if (binary_values->size() != static_cast(records_read)) { + return Status::Corruption( + "Invalid parquet binary record read result for column {}: rows={}, records={}", + column_name, binary_values->size(), records_read); + } + return Status::OK(); +} + +float half_to_float(uint16_t value) { + const uint32_t sign = (value & 0x8000U) << 16; + const uint32_t exponent = (value & 0x7C00U) >> 10; + const uint32_t mantissa = value & 0x03FFU; + + if (exponent == 0) { + if (mantissa == 0) { + return std::bit_cast(sign); + } + const float subnormal = std::ldexp(static_cast(mantissa), -24); + return sign == 0 ? subnormal : -subnormal; + } + if (exponent == 0x1FU) { + return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); + } + return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); +} + +Status build_float16_values(const std::string& column_name, + const ParquetTypeDescriptor& type_descriptor, + const std::vector& binary_values, int64_t row_count, + std::vector* float_values) { + if (type_descriptor.fixed_length != 2) { + return Status::Corruption("Invalid parquet Float16 length for column {}: {}", column_name, + type_descriptor.fixed_length); + } + if (binary_values.size() != static_cast(row_count)) { + return Status::Corruption( + "Invalid parquet Float16 value count for column {}: values={}, rows={}", + column_name, binary_values.size(), row_count); + } + float_values->resize(static_cast(row_count)); + for (int64_t row = 0; row < row_count; ++row) { + const auto& binary_value = binary_values[static_cast(row)]; + if (binary_value.data == nullptr && binary_value.size == 0) { + (*float_values)[static_cast(row)] = 0; + continue; + } + if (binary_value.data == nullptr || binary_value.size != 2) { + return Status::Corruption( + "Invalid parquet Float16 value for column {} at row {}: data={}, size={}", + column_name, row, binary_value.data == nullptr ? "null" : "non-null", + binary_value.size); + } + uint16_t raw_value = 0; + std::memcpy(&raw_value, binary_value.data, sizeof(raw_value)); + (*float_values)[static_cast(row)] = half_to_float(raw_value); + } + return Status::OK(); +} + +} // namespace + +Status ParquetLeafReader::collect_batch(::parquet::internal::RecordReader& record_reader, + ParquetLeafBatch* batch) const { + DORIS_CHECK(batch != nullptr); + batch->_def_levels = nullptr; + batch->_rep_levels = nullptr; + batch->_fixed_values = nullptr; + batch->_binary_chunks.clear(); + batch->_value_kind = decoded_value_kind(_type_descriptor); + batch->_consumed_level_count = record_reader.levels_position(); + batch->_decoded_level_count = record_reader.levels_written(); + if (_descriptor->max_definition_level() > 0) { + batch->_def_levels = record_reader.def_levels(); + } + if (_descriptor->max_repetition_level() > 0) { + batch->_rep_levels = record_reader.rep_levels(); + } + batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); + batch->_values_written = record_reader.values_written(); + + if (!batch->is_binary_value()) { + batch->_fixed_values = record_reader.values(); + return Status::OK(); + } + + RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &batch->_binary_chunks)); + batch->_values_written = 0; + for (const auto& chunk : batch->_binary_chunks) { + if (chunk == nullptr) { + return Status::Corruption( + "Parquet binary record reader returned null chunk for column {}", _name); + } + batch->_values_written += chunk->length(); + } + return Status::OK(); +} + +Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader& record_reader, + ParquetLeafBatch* batch) const { + DORIS_CHECK(batch != nullptr); + batch->_def_levels = nullptr; + batch->_rep_levels = nullptr; + batch->_fixed_values = nullptr; + batch->_binary_chunks.clear(); + batch->_value_kind = decoded_value_kind(_type_descriptor); + batch->_consumed_level_count = record_reader.levels_position(); + batch->_decoded_level_count = record_reader.levels_written(); + if (_descriptor->max_definition_level() > 0) { + batch->_def_levels = record_reader.def_levels(); + } + if (_descriptor->max_repetition_level() > 0) { + batch->_rep_levels = record_reader.rep_levels(); + } + batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); + + // Arrow's RecordReader::Reset() does not reset ByteArray/FLBA builders. GetBuilderChunks() + // (or DictionaryRecordReader::GetResult()) is the documented reset operation and must be + // called before the next ReadRecords(). Otherwise a levels-only skip followed by a normal read + // observes values from both batches; for example, skipping ARRAY ["a", "b"] and then + // reading ["c"] would report one current level but three values. Release the chunks here and + // let the temporary vector destroy them immediately. We deliberately do not inspect or copy + // their payload into a Doris Column, so the levels-only contract still avoids Doris-side value + // materialization. + if (batch->is_binary_value()) { + std::vector> discarded_chunks; + RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &discarded_chunks)); + } + + // COUNT(col) and nested skip only need top-level shape. Fixed-width values remain owned by the + // RecordReader and are cleared by Reset(); binary values were released above solely to reset + // the Arrow builder. + batch->_values_written = 0; + return Status::OK(); +} + +// - FLOAT16: binary -> half_to_float -> float_values +Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t row_count, + const NullMap* null_map, MutableColumnPtr& column) const { + std::vector binary_values; + std::vector spaced_values; + std::vector float_values; + DecodedColumnView view; + view.value_kind = batch._value_kind; + view.time_unit = decoded_time_unit(_type_descriptor.time_unit); + view.row_count = row_count; + view.logical_integer_bit_width = _type_descriptor.integer_bit_width; + view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; + view.decimal_precision = _type_descriptor.decimal_precision; + view.decimal_scale = _type_descriptor.decimal_scale; + view.fixed_length = _type_descriptor.fixed_length; + view.timestamp_is_adjusted_to_utc = _type_descriptor.timestamp_is_adjusted_to_utc; + view.timezone = _timezone; + view.enable_strict_mode = _enable_strict_mode; + view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); + const bool read_dense_for_nullable = batch._read_dense_for_nullable && view.null_map != nullptr; + + if (_type_descriptor.extra_type_info == ParquetExtraTypeInfo::FLOAT16) { + RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, + read_dense_for_nullable, &binary_values)); + RETURN_IF_ERROR(build_float16_values(_name, _type_descriptor, binary_values, row_count, + &float_values)); + view.value_kind = DecodedValueKind::FLOAT; + view.values = reinterpret_cast(float_values.data()); + } else if (batch.is_binary_value()) { + RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, + read_dense_for_nullable, &binary_values)); + view.binary_values = &binary_values; + } else if (read_dense_for_nullable) { + RETURN_IF_ERROR(build_spaced_fixed_values(batch, row_count, null_map, &spaced_values)); + view.values = spaced_values.data(); + } else { + view.values = batch._fixed_values; + } + + if (_decoded_value_appender != nullptr) { + return _decoded_value_appender(column, view); + } + + { + SCOPED_TIMER(_profile.materialization_time); + if (!_type->is_nullable()) { + if (auto* nullable_column = check_and_get_column(*column); + nullable_column != nullptr) { + auto& nested_column = nullable_column->get_nested_column(); + auto& tmp_null_map = nullable_column->get_null_map_data(); + const auto old_nested_size = nested_column.size(); + const auto old_null_map_size = tmp_null_map.size(); + auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); + if (!st.ok()) { + nested_column.resize(old_nested_size); + return st; + } + tmp_null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); + memset(tmp_null_map.data() + old_null_map_size, 0, + tmp_null_map.size() - old_null_map_size); + } else { + RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); + } + } else { + RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); + } + } + return Status::OK(); +} + +bool ParquetLeafBatch::is_binary_value() const { + return _value_kind == DecodedValueKind::BINARY || _value_kind == DecodedValueKind::FIXED_BINARY; +} + +Status ParquetLeafReader::build_spaced_fixed_values(const ParquetLeafBatch& batch, + int64_t row_count, const NullMap* null_map, + std::vector* spaced_values) const { + DORIS_CHECK(null_map != nullptr); + DORIS_CHECK(spaced_values != nullptr); + size_t value_size = 0; + RETURN_IF_ERROR(decoded_fixed_value_size(_name, batch._value_kind, &value_size)); + spaced_values->resize(static_cast(row_count) * value_size); + const auto non_null_count = static_cast(simd::count_zero_num( + reinterpret_cast(null_map->data()), null_map->size())); + if (batch._values_written != non_null_count) { + return Status::Corruption( + "Invalid dense nullable parquet values for column {}: values={}, records={}, " + "nulls={}", + _name, batch._values_written, row_count, row_count - non_null_count); + } + auto* dst = spaced_values->data(); + int64_t value_idx = 0; + for (int64_t record_idx = 0; record_idx < row_count; ++record_idx) { + if ((*null_map)[record_idx] != 0) { + continue; // NULL row: skip it and keep the target slot zeroed + } + std::memcpy(dst + static_cast(record_idx) * value_size, + batch._fixed_values + static_cast(value_idx) * value_size, value_size); + ++value_idx; + } + return Status::OK(); +} + +ParquetLeafReader::ParquetLeafReader( + const ::parquet::ColumnDescriptor* descriptor, ParquetTypeDescriptor type_descriptor, + DataTypePtr type, std::string name, + std::shared_ptr<::parquet::internal::RecordReader> record_reader, + ParquetColumnReaderProfile profile, const cctz::time_zone* timezone, + bool enable_strict_mode, + std::function decoded_value_appender) + : _descriptor(descriptor), + _type_descriptor(type_descriptor), + _type(std::move(type)), + _name(std::move(name)), + _record_reader(std::move(record_reader)), + _profile(profile), + _timezone(timezone), + _enable_strict_mode(enable_strict_mode), + _decoded_value_appender(std::move(decoded_value_appender)) {} + +Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch, + int64_t* rows_read) const { + if (batch == nullptr || rows_read == nullptr) { + return Status::InvalidArgument("Invalid parquet leaf batch result pointer for column {}", + _name); + } + if (_record_reader == nullptr) { + return Status::InternalError("Parquet record reader is not initialized for column {}", + _name); + } + + try { + _record_reader->Reset(); + _record_reader->Reserve(batch_rows); + { + SCOPED_TIMER(_profile.arrow_read_records_time); + *rows_read = _record_reader->ReadRecords(batch_rows); + } + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to read parquet records for column {}: {}", _name, + e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to read parquet records for column {}: {}", _name, + e.what()); + } + if (*rows_read < 0 || *rows_read > batch_rows) { + return Status::Corruption("Invalid parquet record read result for column {}: {}", _name, + *rows_read); + } + return collect_batch(*_record_reader, batch); +} + +Status ParquetLeafReader::build_null_map(const ParquetLeafBatch& batch, int64_t records_read, + NullMap* null_map) const { + if (_descriptor->max_definition_level() == 0) { + return Status::OK(); + } + auto* def_levels = batch.def_levels(); + if (def_levels == nullptr && records_read > 0) { + return Status::Corruption( + "Parquet record reader returned null definition levels for nullable column {}", + _name); + } + const int16_t max_definition_level = _descriptor->max_definition_level(); + null_map->resize(records_read); + auto* __restrict dst = null_map->data(); + const auto* __restrict src = def_levels; + for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { + dst[record_idx] = src[record_idx] != max_definition_level; + } + return Status::OK(); +} + +Status ParquetLeafReader::read_nested_batch(int64_t batch_rows, int16_t value_slot_definition_level, + ParquetNestedScalarBatch* batch, + int16_t value_slot_repetition_level) const { + ParquetLeafBatch leaf_batch; + int64_t records_read = 0; + RETURN_IF_ERROR(read_batch(batch_rows, &leaf_batch, &records_read)); + return build_nested_batch_from_leaf_batch(leaf_batch, records_read, value_slot_definition_level, + batch, value_slot_repetition_level); +} + +Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, + ParquetNestedScalarBatch* batch) const { + if (batch == nullptr) { + return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); + } + if (_record_reader == nullptr) { + return Status::InternalError("Parquet record reader is not initialized for column {}", + _name); + } + + int64_t records_read = 0; + ParquetLeafBatch leaf_batch; + try { + _record_reader->Reset(); + _record_reader->Reserve(batch_rows); + { + SCOPED_TIMER(_profile.arrow_read_records_time); + records_read = _record_reader->ReadRecords(batch_rows); + } + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to read parquet levels for column {}: {}", _name, + e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to read parquet levels for column {}: {}", _name, + e.what()); + } + if (records_read < 0 || records_read > batch_rows) { + return Status::Corruption("Invalid parquet level read result for column {}: {}", _name, + records_read); + } + RETURN_IF_ERROR(collect_levels_batch(*_record_reader, &leaf_batch)); + return build_nested_levels_batch_from_leaf_batch(leaf_batch, records_read, batch); +} + +Status ParquetLeafReader::build_nested_batch_from_leaf_batch( + const ParquetLeafBatch& leaf_batch, int64_t records_read, + int16_t value_slot_definition_level, ParquetNestedScalarBatch* batch, + int16_t value_slot_repetition_level) const { + if (batch == nullptr) { + return Status::InvalidArgument("Nested scalar batch is null for column {}", _name); + } + *batch = ParquetNestedScalarBatch(); + batch->value_slot_definition_level = value_slot_definition_level; + batch->value_slot_repetition_level = value_slot_repetition_level; + + batch->records_read = records_read; + if (_type->is_nullable() && leaf_batch.read_dense_for_nullable()) { + return Status::NotSupported( + "Dense nullable parquet nested reader is not supported for column {}", _name); + } + batch->levels_written = leaf_batch.consumed_level_count(); + const int64_t values_written = leaf_batch.values_written(); + if (batch->levels_written > leaf_batch.decoded_level_count()) { + return Status::Corruption( + "Invalid nested parquet level position for column {}: position={}, levels={}", + _name, batch->levels_written, leaf_batch.decoded_level_count()); + } + if (batch->levels_written == 0 && batch->records_read > 0 && + values_written == batch->records_read && _descriptor->max_definition_level() == 0 && + _descriptor->max_repetition_level() == 0) { + batch->levels_written = batch->records_read; + } + if (batch->levels_written < batch->records_read || values_written < 0 || + values_written > batch->levels_written) { + return Status::Corruption( + "Invalid nested parquet read result for column {}: rows={}, levels={}, values={}", + _name, batch->records_read, batch->levels_written, values_written); + } + if (batch->levels_written == 0) { + return Status::OK(); + } + + auto* def_levels = leaf_batch.def_levels(); + if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { + return Status::Corruption( + "Nested parquet reader returned null definition levels for column {}", _name); + } + batch->def_levels.resize(static_cast(batch->levels_written)); + if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { + std::fill(batch->def_levels.begin(), batch->def_levels.end(), + _descriptor->max_definition_level()); + } else { + std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); + } + + auto* rep_levels = leaf_batch.rep_levels(); + if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { + return Status::Corruption( + "Nested parquet reader returned null repetition levels for column {}", _name); + } + batch->rep_levels.resize(static_cast(batch->levels_written)); + if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { + std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); + } else { + std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); + } + + const int16_t leaf_definition_level = _descriptor->max_definition_level(); + // Arrow's RecordReader may emit value placeholders for null ancestors that are below the + // Doris materialization threshold. Those slots must still advance the payload value index; + // otherwise the next defined child level points at the placeholder instead of its real value. + auto count_value_slots = [&](int16_t slot_definition_level) { + int64_t slot_count = 0; + for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { + if (batch->def_levels[level_idx] >= slot_definition_level && + batch->rep_levels[level_idx] <= value_slot_repetition_level) { + ++slot_count; + } + } + return slot_count; + }; + + const int64_t value_slot_count = count_value_slots(value_slot_definition_level); + int16_t payload_slot_definition_level = value_slot_definition_level; + int64_t payload_value_slot_count = value_slot_count; + while (payload_slot_definition_level > 0 && payload_value_slot_count < values_written) { + --payload_slot_definition_level; + payload_value_slot_count = count_value_slots(payload_slot_definition_level); + } + + int64_t leaf_value_count = 0; + for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { + if (batch->def_levels[level_idx] < value_slot_definition_level || + batch->rep_levels[level_idx] > value_slot_repetition_level) { + continue; + } + if (batch->def_levels[level_idx] == leaf_definition_level) { + ++leaf_value_count; + } + } + + enum class ValueLayout { LEVELS, VALUE_SLOTS, LEAF_VALUES, PAYLOAD_VALUE_SLOTS }; + ValueLayout value_layout = ValueLayout::LEAF_VALUES; + if (values_written == batch->levels_written) { + value_layout = ValueLayout::LEVELS; + } else if (values_written == value_slot_count) { + value_layout = ValueLayout::VALUE_SLOTS; + } else if (values_written == leaf_value_count) { + value_layout = ValueLayout::LEAF_VALUES; + } else if (values_written == payload_value_slot_count) { + value_layout = ValueLayout::PAYLOAD_VALUE_SLOTS; + } else { + return Status::Corruption( + "Nested parquet reader returned inconsistent value count for column {}: values={}, " + "levels={}, slots={}, leaf_values={}, payload_slots={}, " + "payload_slot_definition_level={}", + _name, values_written, batch->levels_written, value_slot_count, leaf_value_count, + payload_value_slot_count, payload_slot_definition_level); + } + + batch->value_indices.resize(static_cast(batch->levels_written), -1); + NullMap value_nulls(static_cast(values_written), 1); + int64_t value_idx = 0; + const int16_t decoded_slot_definition_level = value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS + ? payload_slot_definition_level + : value_slot_definition_level; + for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { + if (batch->def_levels[level_idx] < decoded_slot_definition_level || + batch->rep_levels[level_idx] > value_slot_repetition_level) { + continue; + } + const bool has_leaf_value = batch->def_levels[level_idx] == leaf_definition_level; + int64_t decoded_value_idx = -1; + if (value_layout == ValueLayout::LEVELS) { + decoded_value_idx = level_idx; + } else if (value_layout == ValueLayout::VALUE_SLOTS) { + decoded_value_idx = value_idx++; + } else if (value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS) { + decoded_value_idx = value_idx++; + } else { + if (!has_leaf_value) { + continue; + } + decoded_value_idx = value_idx++; + } + DORIS_CHECK(decoded_value_idx >= 0); + DORIS_CHECK(decoded_value_idx < values_written); + if (has_leaf_value) { + batch->value_indices[static_cast(level_idx)] = decoded_value_idx; + value_nulls[static_cast(decoded_value_idx)] = 0; + } + } + if (value_layout != ValueLayout::LEVELS && value_idx != values_written) { + return Status::Corruption( + "Nested parquet reader value cursor stopped early for column {}: values={}, " + "visited={}", + _name, values_written, value_idx); + } + + const auto value_type = remove_nullable(_type); + batch->values_column = value_type->create_column(); + if (values_written > 0) { + ParquetLeafReader value_reader(_descriptor, _type_descriptor, value_type, _name, + _record_reader, _profile, _timezone, _enable_strict_mode); + RETURN_IF_ERROR(value_reader.append_values(leaf_batch, values_written, &value_nulls, + batch->values_column)); + } + return Status::OK(); +} + +Status ParquetLeafReader::build_nested_levels_batch_from_leaf_batch( + const ParquetLeafBatch& leaf_batch, int64_t records_read, + ParquetNestedScalarBatch* batch) const { + if (batch == nullptr) { + return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); + } + *batch = ParquetNestedScalarBatch(); + batch->records_read = records_read; + batch->levels_written = leaf_batch.consumed_level_count(); + if (batch->levels_written > leaf_batch.decoded_level_count()) { + return Status::Corruption( + "Invalid nested parquet level position for column {}: position={}, levels={}", + _name, batch->levels_written, leaf_batch.decoded_level_count()); + } + + // Required flat leaves do not have physical def/rep level buffers. Synthesize one level slot + // per top-level row so the COUNT(col) aggregation code can use the same shape loop. + if (batch->levels_written == 0 && batch->records_read > 0 && + _descriptor->max_definition_level() == 0 && _descriptor->max_repetition_level() == 0) { + batch->levels_written = batch->records_read; + } + if (batch->levels_written < batch->records_read) { + return Status::Corruption( + "Invalid nested parquet levels result for column {}: rows={}, levels={}", _name, + batch->records_read, batch->levels_written); + } + if (batch->levels_written == 0) { + return Status::OK(); + } + + auto* def_levels = leaf_batch.def_levels(); + if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { + return Status::Corruption( + "Nested parquet reader returned null definition levels for column {}", _name); + } + batch->def_levels.resize(static_cast(batch->levels_written)); + if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { + std::fill(batch->def_levels.begin(), batch->def_levels.end(), + _descriptor->max_definition_level()); + } else { + std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); + } + + auto* rep_levels = leaf_batch.rep_levels(); + if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { + return Status::Corruption( + "Nested parquet reader returned null repetition levels for column {}", _name); + } + batch->rep_levels.resize(static_cast(batch->levels_written)); + if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { + std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); + } else { + std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); + } + return Status::OK(); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h new file mode 100644 index 00000000000000..b396b35fd1f32c --- /dev/null +++ b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column.h" +#include "core/column/column_nullable.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/parquet_type.h" + +namespace parquet { +class ColumnDescriptor; + +namespace internal { +class RecordReader; +} // namespace internal +} // namespace parquet + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace arrow { +class Array; +} // namespace arrow + +namespace doris::format::parquet { + +struct ParquetLeafReaderTestAccess; + +// Read result for a nested scalar leaf, separating Dremel-encoded shape from actual values. +// The COUNT(col) aggregation fast path consumes only records_read, levels_written, def_levels, and rep_levels. +// That path does not populate value_indices or values_column, so callers must not call build_nested_column() afterwards. +struct ParquetNestedScalarBatch { + int64_t records_read = 0; + int64_t levels_written = 0; + int16_t value_slot_definition_level = 0; + int16_t value_slot_repetition_level = std::numeric_limits::max(); + std::vector def_levels; + std::vector rep_levels; + std::vector value_indices; + MutableColumnPtr values_column; + + bool empty() const { return levels_written == 0; } +}; + +class ParquetLeafBatch { +public: + int64_t consumed_level_count() const { return _consumed_level_count; } + int64_t decoded_level_count() const { return _decoded_level_count; } + int64_t values_written() const { return _values_written; } + bool read_dense_for_nullable() const { return _read_dense_for_nullable; } + const int16_t* def_levels() const { return _def_levels; } + const int16_t* rep_levels() const { return _rep_levels; } + const std::vector>& binary_chunks() const { + return _binary_chunks; + } + +private: + friend class ParquetLeafReader; + + bool is_binary_value() const; + + DecodedValueKind _value_kind = DecodedValueKind::INT32; + int64_t _consumed_level_count = 0; + int64_t _decoded_level_count = 0; + int64_t _values_written = 0; + const int16_t* _def_levels = nullptr; + const int16_t* _rep_levels = nullptr; + const uint8_t* _fixed_values = nullptr; + bool _read_dense_for_nullable = false; + std::vector> _binary_chunks; +}; + +// read_batch() -> build_null_map() + append_values() +// read_nested_batch() +class ParquetLeafReader { +public: + ParquetLeafReader(const ::parquet::ColumnDescriptor* descriptor, + ParquetTypeDescriptor type_descriptor, DataTypePtr type, std::string name, + std::shared_ptr<::parquet::internal::RecordReader> record_reader, + ParquetColumnReaderProfile profile = {}, + const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, + std::function + decoded_value_appender = nullptr); + + Status read_batch(int64_t batch_rows, ParquetLeafBatch* batch, int64_t* rows_read) const; + + Status build_null_map(const ParquetLeafBatch& batch, int64_t records_read, + NullMap* null_map) const; + + Status append_values(const ParquetLeafBatch& batch, int64_t row_count, const NullMap* null_map, + MutableColumnPtr& column) const; + + // LEVELS / VALUE_SLOTS / LEAF_VALUES / PAYLOAD_VALUE_SLOTS. + Status read_nested_batch( + int64_t batch_rows, int16_t value_slot_definition_level, + ParquetNestedScalarBatch* batch, + int16_t value_slot_repetition_level = std::numeric_limits::max()) const; + + // COUNT(col) and nested-skip shape-only read path. It still calls Arrow + // RecordReader::ReadRecords() to advance the Parquet cursor and obtain def/rep levels, but + // Doris only copies levels: + // - it does not build value_indices or values_column + // - it does not enter DataTypeSerde::read_column_from_decoded_values() + // - for Binary/FLBA, it releases and immediately discards Arrow builder chunks because that is + // the RecordReader's required reset operation; it never copies them into a Doris Column + // This lets COUNT(col) on MAP/ARRAY/STRUCT evaluate top-level NULL state and lets skip advance + // nested shape without Doris-side STRING/BINARY materialization. Arrow RecordReader does not + // expose a public levels-only API, so ReadRecords may still perform required page decoding. + Status read_nested_levels_batch(int64_t batch_rows, ParquetNestedScalarBatch* batch) const; + +private: + friend struct ParquetLeafReaderTestAccess; + + Status collect_batch(::parquet::internal::RecordReader& record_reader, + ParquetLeafBatch* batch) const; + + // Levels-only variant of collect_batch(). It snapshots only def/rep level state and does not + // expose value buffers. Binary chunks are released only to reset Arrow's builder and are + // immediately discarded. Used by COUNT(col) and nested skip. + Status collect_levels_batch(::parquet::internal::RecordReader& record_reader, + ParquetLeafBatch* batch) const; + + Status build_spaced_fixed_values(const ParquetLeafBatch& batch, int64_t row_count, + const NullMap* null_map, + std::vector* spaced_values) const; + + Status build_nested_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, + int64_t records_read, + int16_t value_slot_definition_level, + ParquetNestedScalarBatch* batch, + int16_t value_slot_repetition_level) const; + Status build_nested_levels_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, + int64_t records_read, + ParquetNestedScalarBatch* batch) const; + + const ::parquet::ColumnDescriptor* _descriptor = + nullptr; // Arrow column descriptor (physical_type, max_dl, max_rl) + ParquetTypeDescriptor + _type_descriptor; // type encoding information (decimal precision, timestamp unit, etc.) + DataTypePtr _type; // Doris target type + std::string _name; // column name for error messages + std::shared_ptr<::parquet::internal::RecordReader> + _record_reader; // Arrow physical column reader (shared ownership) + ParquetColumnReaderProfile _profile; // profile counters + const cctz::time_zone* _timezone = nullptr; // timezone for timestamp conversion + bool _enable_strict_mode = false; // strict mode for type mismatch errors + std::function _decoded_value_appender; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp new file mode 100644 index 00000000000000..4e9a363b13c7cb --- /dev/null +++ b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/row_position_column_reader.h" + +#include + +#include "core/assert_cast.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "format_v2/parquet/parquet_column_schema.h" + +namespace doris::format::parquet { + +RowPositionColumnReader::RowPositionColumnReader(int64_t row_group_first_row, + ParquetColumnReaderProfile profile) + : ParquetColumnReader(ParquetColumnSchema {.name = format::ROW_POSITION_COLUMN_NAME}, + std::make_shared(), profile), + _row_group_first_row(row_group_first_row) {} + +int RowPositionColumnReader::file_column_id() const { + return format::ROW_POSITION_COLUMN_ID; +} + +int RowPositionColumnReader::parquet_leaf_column_id() const { + return -1; +} + +const DataTypePtr& RowPositionColumnReader::type() const { + return _type; +} + +const std::string& RowPositionColumnReader::name() const { + return _name; +} + +Status RowPositionColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + if (column.get() == nullptr || rows_read == nullptr) { + return Status::InvalidArgument("Invalid parquet row position read result pointer"); + } + if (rows < 0) { + return Status::InvalidArgument("Invalid parquet row position read rows {}", rows); + } + auto* vector_column = assert_cast(column.get()); + auto& data = vector_column->get_data(); + const auto old_size = data.size(); + data.resize(old_size + rows); + for (int64_t row = 0; row < rows; ++row) { + data[old_size + row] = _row_group_first_row + _next_row_position + row; + } + _next_row_position += rows; + *rows_read = rows; + return Status::OK(); +} + +Status RowPositionColumnReader::skip(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + _next_row_position += rows; + return Status::OK(); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/row_position_column_reader.h b/be/src/format_v2/parquet/reader/row_position_column_reader.h new file mode 100644 index 00000000000000..934100317ec4fd --- /dev/null +++ b/be/src/format_v2/parquet/reader/row_position_column_reader.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "format_v2/parquet/reader/column_reader.h" + +namespace doris::format::parquet { + +class RowPositionColumnReader final : public ParquetColumnReader { +public: + explicit RowPositionColumnReader(int64_t row_group_first_row, + ParquetColumnReaderProfile profile = {}); + + int file_column_id() const override; + int parquet_leaf_column_id() const override; + const DataTypePtr& type() const override; + const std::string& name() const override; + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + +private: + int64_t _row_group_first_row = 0; // first file row of the current row group + int64_t _next_row_position = 0; // next row position to emit +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp new file mode 100644 index 00000000000000..784e4cdc900fb7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp @@ -0,0 +1,568 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/scalar_column_reader.h" + +#include +#include +#include + +#include +#include +#include + +#include "core/column/column.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "util/simd/bits.h" + +namespace doris::format::parquet { +namespace { + +class ParquetNestedScalarValueCursor { +public: + explicit ParquetNestedScalarValueCursor(const ParquetNestedScalarBatch* batch) { reset(batch); } + + void reset(const ParquetNestedScalarBatch* batch) { + DORIS_CHECK(batch != nullptr); + _batch = batch; + } + + Status value_index(const std::string& column_name, int64_t level_idx, int64_t* value_idx) { + DORIS_CHECK(_batch != nullptr); + DORIS_CHECK(value_idx != nullptr); + DORIS_CHECK(level_idx < _batch->levels_written); + DORIS_CHECK(level_idx >= 0); + DORIS_CHECK(static_cast(level_idx) < _batch->value_indices.size()); + const int64_t computed_value_idx = _batch->value_indices[static_cast(level_idx)]; + if (computed_value_idx < 0) { + return Status::Corruption("Nested parquet value is absent for column {}", column_name); + } + DORIS_CHECK(_batch->values_column.get() != nullptr); + if (computed_value_idx >= _batch->values_column->size()) { + return Status::Corruption("Nested parquet value index is out of range for column {}", + column_name); + } + *value_idx = computed_value_idx; + return Status::OK(); + } + +private: + const ParquetNestedScalarBatch* _batch = nullptr; +}; + +Status append_scalar_batch_value(const ScalarColumnReader& column_reader, + const ParquetNestedScalarBatch& batch, int64_t level_idx, + ParquetNestedScalarValueCursor* value_cursor, + MutableColumnPtr& column) { + DORIS_CHECK(value_cursor != nullptr); + int64_t value_idx = -1; + RETURN_IF_ERROR(value_cursor->value_index(column_reader.name(), level_idx, &value_idx)); + auto* nullable_column = check_and_get_column(*column); + if (nullable_column != nullptr) { + nullable_column->get_nested_column().insert_from(*batch.values_column, + static_cast(value_idx)); + nullable_column->get_null_map_data().push_back(0); + return Status::OK(); + } + column->insert_from(*batch.values_column, static_cast(value_idx)); + return Status::OK(); +} + +Status append_arrow_binary_dictionary_value(const std::string& column_name, + const ::arrow::Array& dictionary, + int64_t dictionary_index, + std::vector* values) { + DORIS_CHECK(values != nullptr); + if (dictionary_index < 0 || dictionary_index >= dictionary.length()) { + return Status::Corruption("Invalid parquet dictionary index {} for column {}", + dictionary_index, column_name); + } + if (auto* binary_array = dynamic_cast(&dictionary)) { + if (binary_array->IsNull(dictionary_index)) { + values->emplace_back(static_cast(nullptr), 0); + return Status::OK(); + } + int32_t length = 0; + const uint8_t* value = binary_array->GetValue(dictionary_index, &length); + values->emplace_back(reinterpret_cast(value), length); + return Status::OK(); + } + if (auto* fixed_array = dynamic_cast(&dictionary)) { + if (fixed_array->IsNull(dictionary_index)) { + values->emplace_back(static_cast(nullptr), 0); + return Status::OK(); + } + values->emplace_back(reinterpret_cast(fixed_array->GetValue(dictionary_index)), + fixed_array->byte_width()); + return Status::OK(); + } + return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", + column_name); +} + +} // namespace + +ScalarColumnReader::ScalarColumnReader( + const ParquetColumnSchema& column_schema, + std::shared_ptr<::parquet::internal::RecordReader> record_reader, + const ParquetPageSkipPlan* page_skip_plan, const cctz::time_zone* timezone, + bool enable_strict_mode, ParquetColumnReaderProfile profile) + : ParquetColumnReader(column_schema, column_schema.type, profile), + _descriptor(column_schema.descriptor), + _type_descriptor(column_schema.type_descriptor), + _record_reader(std::move(record_reader)), + _page_skip_plan(page_skip_plan), + _timezone(timezone), + _enable_strict_mode(enable_strict_mode), + _nested_batch(std::make_unique()) {} + +ScalarColumnReader::~ScalarColumnReader() = default; + +Status ScalarColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + if (column.get() == nullptr || rows_read == nullptr) { + return Status::InvalidArgument("Invalid parquet column read result pointer for column {}", + _name); + } + if (_record_reader == nullptr) { + return Status::InternalError("Parquet record reader is not initialized for column {}", + _name); + } + auto reader = leaf_reader(); + ParquetLeafBatch leaf_batch; + RETURN_IF_ERROR(reader.read_batch(rows, &leaf_batch, rows_read)); + + NullMap null_map; + RETURN_IF_ERROR(reader.build_null_map(leaf_batch, *rows_read, &null_map)); + const auto value_kind = decoded_value_kind(_type_descriptor); + const bool is_binary_value = + value_kind == DecodedValueKind::BINARY || value_kind == DecodedValueKind::FIXED_BINARY; + if (!is_binary_value && leaf_batch.read_dense_for_nullable() && !null_map.empty()) { + const int64_t non_null_count = static_cast(simd::count_zero_num( + reinterpret_cast(null_map.data()), null_map.size())); + const int64_t null_count = *rows_read - non_null_count; + if (leaf_batch.values_written() != non_null_count) { + return Status::Corruption( + "Invalid dense nullable parquet record read result for column {}: values={}, " + "records={}, nulls={}", + _name, leaf_batch.values_written(), *rows_read, null_count); + } + } else if (!is_binary_value && !leaf_batch.read_dense_for_nullable() && + leaf_batch.values_written() != *rows_read) { + return Status::Corruption( + "Invalid parquet record read result for column {}: values={}, records={}", _name, + leaf_batch.values_written(), *rows_read); + } + + RETURN_IF_ERROR(reader.append_values(leaf_batch, *rows_read, &null_map, column)); + advance_rows_read(*rows_read); + update_reader_read_rows(*rows_read); + return Status::OK(); +} + +Status ScalarColumnReader::skip_records(int64_t rows) { + if (_record_reader == nullptr) { + return Status::InternalError("Parquet record reader is not initialized for column {}", + _name); + } + if (rows <= 0) { + return Status::OK(); + } + int64_t skipped_rows = 0; + try { + _record_reader->Reset(); + while (skipped_rows < rows) { + const int64_t skipped = _record_reader->SkipRecords(rows - skipped_rows); + if (skipped <= 0) { + return Status::Corruption( + "Failed to skip parquet records for column {}: skipped {} of {} rows", + _name, skipped_rows, rows); + } + skipped_rows += skipped; + } + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to skip parquet records for column {}: {}", _name, + e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to skip parquet records for column {}: {}", _name, + e.what()); + } + update_reader_skip_rows(rows); + return Status::OK(); +} + +int64_t ScalarColumnReader::page_filtered_rows_to_skip(int64_t rows) const { + if (_page_skip_plan == nullptr || rows <= 0) { + return 0; + } + const int64_t skip_end = _row_group_rows_read + rows; + int64_t filtered_rows = 0; + for (const auto& range : _page_skip_plan->skipped_ranges) { + const int64_t range_end = range.start + range.length; + if (range_end <= _row_group_rows_read) { + continue; + } + if (range.start >= skip_end) { + break; + } + const int64_t start = std::max(range.start, _row_group_rows_read); + const int64_t end = std::min(range_end, skip_end); + if (start < end) { + // Scheduler gap skips are derived from page-index selected_ranges. A page-filtered + // range can only overlap such a gap when the whole data page is outside every selected + // range, so partial overlap would mean the planner and scheduler are out of sync. + DORIS_CHECK(start == range.start); + DORIS_CHECK(end == range_end); + filtered_rows += end - start; + } + } + return filtered_rows; +} + +void ScalarColumnReader::advance_rows_read(int64_t rows) { + DORIS_CHECK(rows >= 0); + _row_group_rows_read += rows; +} + +Status ScalarColumnReader::skip(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + + const int64_t page_filtered_rows = page_filtered_rows_to_skip(rows); + DORIS_CHECK(page_filtered_rows <= rows); + const int64_t record_reader_skip_rows = rows - page_filtered_rows; + RETURN_IF_ERROR(skip_records(record_reader_skip_rows)); + advance_rows_read(rows); + return Status::OK(); +} + +Status ScalarColumnReader::select_with_dictionary_filter(const SelectionVector& sel, + uint16_t selected_rows, int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(column.get() != nullptr); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); + *used_filter = false; + row_filter->clear(); + row_filter->reserve(selected_rows); + + const auto ranges = selection_to_ranges(sel, selected_rows); + int64_t cursor = 0; + for (const auto& range : ranges) { + if (range.start < cursor || range.start + range.length > batch_rows) { + return Status::InvalidArgument( + "Invalid parquet dictionary selection range [{}, {}) for column {}", + range.start, range.start + range.length, _name); + } + RETURN_IF_ERROR(skip(range.start - cursor)); + + int64_t range_rows_read = 0; + RETURN_IF_ERROR(read_range_with_dictionary_filter(range.length, dictionary_filter, column, + row_filter, &range_rows_read, + used_filter)); + if (!*used_filter) { + return Status::OK(); + } + if (range_rows_read != range.length) { + return Status::Corruption( + "Parquet dictionary selected read returned {} rows, expected {} rows for " + "column {}", + range_rows_read, range.length, _name); + } + cursor = range.start + range.length; + } + RETURN_IF_ERROR(skip(batch_rows - cursor)); + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + return Status::OK(); +} + +Status ScalarColumnReader::read_range_with_dictionary_filter( + int64_t rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, + IColumn::Filter* row_filter, int64_t* rows_read, bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(rows_read != nullptr); + DORIS_CHECK(used_filter != nullptr); + DORIS_CHECK(_record_reader != nullptr); + if (!_record_reader->read_dictionary()) { + *used_filter = false; + return Status::OK(); + } + + ParquetLeafBatch leaf_batch; + RETURN_IF_ERROR(leaf_reader().read_batch(rows, &leaf_batch, rows_read)); + int64_t matched_rows = 0; + RETURN_IF_ERROR(append_dictionary_filtered_values(leaf_batch.binary_chunks(), dictionary_filter, + column, row_filter, &matched_rows, + used_filter)); + if (!*used_filter) { + return Status::Corruption( + "Parquet dictionary reader did not return dictionary batches for column {}", _name); + } + if (row_filter->size() < static_cast(*rows_read)) { + return Status::Corruption( + "Parquet dictionary filter produced too few row decisions for column {}: " + "filter={}, rows={}", + _name, row_filter->size(), *rows_read); + } + advance_rows_read(*rows_read); + update_reader_read_rows(*rows_read); + return Status::OK(); +} + +Status ScalarColumnReader::append_dictionary_filtered_values( + const std::vector>& chunks, + const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, + IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(matched_rows != nullptr); + DORIS_CHECK(used_filter != nullptr); + *matched_rows = 0; + *used_filter = false; + + std::vector selected_values; + for (const auto& chunk : chunks) { + DORIS_CHECK(chunk != nullptr); + const auto* dict_array = dynamic_cast(chunk.get()); + if (dict_array == nullptr) { + // The caller has already consumed rows from a DictionaryRecordReader. Falling back to a + // normal selected read would desynchronize the Parquet stream, so absence of a + // DictionaryArray is reported as corruption by read_range_with_dictionary_filter(). + return Status::OK(); + } + *used_filter = true; + const auto& dictionary = dict_array->dictionary(); + if (dictionary == nullptr) { + return Status::Corruption("Parquet dictionary array has null dictionary for column {}", + _name); + } + + // Dictionary predicates are evaluated once against the dictionary page and produce a + // dictionary-entry bitmap. DATA_PAGE rows then only need an integer-index lookup. NULL rows + // do not have a dictionary entry and cannot satisfy the supported equality/IN predicates. + for (int64_t row = 0; row < dict_array->length(); ++row) { + bool keep = false; + if (!dict_array->IsNull(row)) { + const int64_t dictionary_index = dict_array->GetValueIndex(row); + if (dictionary_index >= 0 && + dictionary_index < static_cast(dictionary_filter.size())) { + keep = dictionary_filter[static_cast(dictionary_index)] != 0; + } + if (keep) { + RETURN_IF_ERROR(append_arrow_binary_dictionary_value( + _name, *dictionary, dictionary_index, &selected_values)); + ++*matched_rows; + } + } + row_filter->push_back(keep ? 1 : 0); + } + } + + if (!*used_filter) { + return Status::OK(); + } + return append_decoded_binary_values(selected_values, column); +} + +Status ScalarColumnReader::append_decoded_binary_values(const std::vector& values, + MutableColumnPtr& column) const { + DecodedColumnView view; + view.value_kind = decoded_value_kind(_type_descriptor); + view.row_count = static_cast(values.size()); + view.logical_integer_bit_width = _type_descriptor.integer_bit_width; + view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; + view.fixed_length = _type_descriptor.fixed_length; + view.binary_values = &values; + + SCOPED_TIMER(_profile.materialization_time); + if (!_type->is_nullable()) { + if (auto* nullable_column = check_and_get_column(*column); + nullable_column != nullptr) { + auto& nested_column = nullable_column->get_nested_column(); + auto& null_map = nullable_column->get_null_map_data(); + const auto old_nested_size = nested_column.size(); + const auto old_null_map_size = null_map.size(); + auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); + if (!st.ok()) { + nested_column.resize(old_nested_size); + return st; + } + null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); + memset(null_map.data() + old_null_map_size, 0, null_map.size() - old_null_map_size); + return Status::OK(); + } + return _type->get_serde()->read_column_from_decoded_values(*column, view); + } + + NullMap null_map(values.size(), 0); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + return _type->get_serde()->read_column_from_decoded_values(*column, view); +} + +// The value index stream must advance on those null slots, otherwise later payload values shift. +Status ScalarColumnReader::load_nested_batch(int64_t rows) { + DORIS_CHECK(_nested_batch != nullptr); + reset_nested_build_level_cursor(); + const int16_t materialized_slot_definition_level = + static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); + RETURN_IF_ERROR(leaf_reader().read_nested_batch(rows, materialized_slot_definition_level, + _nested_batch.get(), _repetition_level)); + advance_rows_read(_nested_batch->records_read); + update_reader_read_rows(_nested_batch->records_read); + return Status::OK(); +} + +Status ScalarColumnReader::load_nested_levels_batch(int64_t rows) { + DORIS_CHECK(_nested_batch != nullptr); + reset_nested_build_level_cursor(); + RETURN_IF_ERROR(leaf_reader().read_nested_levels_batch(rows, _nested_batch.get())); + advance_rows_read(_nested_batch->records_read); + update_reader_read_rows(_nested_batch->records_read); + return Status::OK(); +} + +Status ScalarColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) { + if (column.get() == nullptr || values_read == nullptr) { + return Status::InvalidArgument("Invalid parquet nested scalar build result for column {}", + _name); + } + DORIS_CHECK(_nested_batch != nullptr); + ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); + // The levels-only loader intentionally does not populate value-slot metadata or payload + // buffers. Derive the logical slot threshold from the schema, exactly as load_nested_batch() + // does, so this consumer works for both loaded batch forms. + const int16_t materialized_slot_definition_level = + static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); + *values_read = 0; + int64_t level_idx = nested_build_level_cursor(); + while (level_idx < _nested_batch->levels_written && *values_read < length_upper_bound) { + const int64_t current_level_idx = level_idx; + const int16_t def_level = _nested_batch->def_levels[current_level_idx]; + const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; + ++level_idx; + if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { + continue; + } + if (def_level == _definition_level) { + RETURN_IF_ERROR(append_scalar_batch_value(*this, *_nested_batch, current_level_idx, + &value_cursor, column)); + } else { + if (!_type->is_nullable() && def_level >= _nullable_definition_level) { + return Status::Corruption( + "Parquet scalar column {} contains null for non-nullable field", _name); + } + column->insert_default(); + } + ++*values_read; + } + set_nested_build_level_cursor(level_idx); + return Status::OK(); +} + +Status ScalarColumnReader::consume_nested_column(int64_t length_upper_bound, + int64_t* values_consumed) { + if (values_consumed == nullptr) { + return Status::InvalidArgument("Invalid parquet nested scalar consume result for column {}", + _name); + } + DORIS_CHECK(_nested_batch != nullptr); + // A levels-only batch intentionally has no value-slot metadata. Reconstruct the same logical + // slot threshold used by load_nested_batch(): a nullable leaf owns a slot at one definition + // level below a non-null value, while a required leaf owns a slot only at its full definition + // level. For example, an empty ARRAY boundary must not be consumed as a STRING value. + const int16_t materialized_slot_definition_level = + static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); + *values_consumed = 0; + int64_t level_idx = nested_build_level_cursor(); + while (level_idx < _nested_batch->levels_written && *values_consumed < length_upper_bound) { + const int64_t current_level_idx = level_idx; + const int16_t def_level = _nested_batch->def_levels[current_level_idx]; + const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; + ++level_idx; + if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { + continue; + } + RETURN_IF_ERROR(validate_nested_value(current_level_idx, false)); + ++*values_consumed; + } + set_nested_build_level_cursor(level_idx); + return Status::OK(); +} + +Status ScalarColumnReader::append_nested_value(int64_t level_idx, MutableColumnPtr& column) const { + if (column.get() == nullptr) { + return Status::InvalidArgument("Invalid parquet nested scalar append result for column {}", + _name); + } + DORIS_CHECK(_nested_batch != nullptr); + DORIS_CHECK(level_idx >= 0); + DORIS_CHECK(level_idx < _nested_batch->levels_written); + ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); + const int16_t def_level = _nested_batch->def_levels[level_idx]; + if (def_level == _definition_level) { + return append_scalar_batch_value(*this, *_nested_batch, level_idx, &value_cursor, column); + } + if (!_type->is_nullable()) { + return Status::Corruption("Parquet MAP column {} contains null for non-nullable value", + _name); + } + column->insert_default(); + return Status::OK(); +} + +Status ScalarColumnReader::validate_nested_value(int64_t level_idx, bool require_non_null) const { + DORIS_CHECK(_nested_batch != nullptr); + DORIS_CHECK(level_idx >= 0); + DORIS_CHECK(level_idx < _nested_batch->levels_written); + const int16_t def_level = _nested_batch->def_levels[level_idx]; + if (def_level == _definition_level) { + return Status::OK(); + } + if (require_non_null || !_type->is_nullable()) { + return Status::Corruption("Parquet scalar column {} contains null for non-nullable field", + _name); + } + return Status::OK(); +} + +const std::vector& ScalarColumnReader::nested_definition_levels() const { + DORIS_CHECK(_nested_batch != nullptr); + return _nested_batch->def_levels; +} + +const std::vector& ScalarColumnReader::nested_repetition_levels() const { + DORIS_CHECK(_nested_batch != nullptr); + return _nested_batch->rep_levels; +} + +int64_t ScalarColumnReader::nested_levels_written() const { + DORIS_CHECK(_nested_batch != nullptr); + return _nested_batch->levels_written; +} + +bool ScalarColumnReader::is_or_has_repeated_child() const { + return _repetition_level > 0; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.h b/be/src/format_v2/parquet/reader/scalar_column_reader.h new file mode 100644 index 00000000000000..5342baa803eca0 --- /dev/null +++ b/be/src/format_v2/parquet/reader/scalar_column_reader.h @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "core/string_ref.h" +#include "format_v2/parquet/parquet_type.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/parquet_leaf_reader.h" + +namespace parquet { +class ColumnDescriptor; + +namespace internal { +class RecordReader; +} // namespace internal +} // namespace parquet + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris::format::parquet { + +struct ScalarColumnReaderTestAccess; + +// load_nested_batch() / build_nested_column() +class ScalarColumnReader final : public ParquetColumnReader { + friend class MapColumnReader; + friend struct ScalarColumnReaderTestAccess; + +public: + ScalarColumnReader(const ParquetColumnSchema& column_schema, + std::shared_ptr<::parquet::internal::RecordReader> record_reader, + const ParquetPageSkipPlan* page_skip_plan = nullptr, + const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, + ParquetColumnReaderProfile profile = {}); + ~ScalarColumnReader() override; + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, + int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, IColumn::Filter* row_filter, + bool* used_filter) override; + + Status load_nested_batch(int64_t rows) override; + Status load_nested_levels_batch(int64_t rows) override; + Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) override; + Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; + const std::vector& nested_definition_levels() const override; + const std::vector& nested_repetition_levels() const override; + int64_t nested_levels_written() const override; + bool is_or_has_repeated_child() const override; + +private: + Status append_nested_value(int64_t level_idx, MutableColumnPtr& column) const; + Status validate_nested_value(int64_t level_idx, bool require_non_null) const; + Status read_range_with_dictionary_filter(int64_t rows, const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter); + Status append_dictionary_filtered_values( + const std::vector>& chunks, + const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, + IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const; + Status append_decoded_binary_values(const std::vector& values, + MutableColumnPtr& column) const; + + const ::parquet::ColumnDescriptor* descriptor() const { return _descriptor; } + + ParquetLeafReader leaf_reader() const { + return ParquetLeafReader(_descriptor, _type_descriptor, _type, _name, _record_reader, + _profile, _timezone, _enable_strict_mode); + } + + void advance_rows_read(int64_t rows); + Status skip_records(int64_t rows); + int64_t page_filtered_rows_to_skip(int64_t rows) const; + + const ::parquet::ColumnDescriptor* _descriptor = nullptr; // Arrow column descriptor + ParquetTypeDescriptor _type_descriptor; // type encoding information + std::shared_ptr<::parquet::internal::RecordReader> + _record_reader; // Arrow physical column reader + const ParquetPageSkipPlan* _page_skip_plan = + nullptr; // page-index pruning result (may be nullptr) + const cctz::time_zone* _timezone = nullptr; // timezone + bool _enable_strict_mode = false; // strict mode + int64_t _row_group_rows_read = 0; // rows read in the current row group (cursor) + std::unique_ptr _nested_batch; // intermediate result for nested reads +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.cpp b/be/src/format_v2/parquet/reader/struct_column_reader.cpp new file mode 100644 index 00000000000000..5abe7abe75e9a2 --- /dev/null +++ b/be/src/format_v2/parquet/reader/struct_column_reader.cpp @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/struct_column_reader.h" + +#include +#include +#include +#include + +#include "core/column/column_struct.h" +#include "format_v2/parquet/reader/nested_column_materializer.h" +#include "format_v2/parquet/reader/scalar_column_reader.h" + +namespace doris::format::parquet { + +ParquetColumnReader* StructColumnReader::shape_source_reader() const { + for (const auto& child : _children) { + auto* child_reader = child.get(); + DORIS_CHECK(child_reader != nullptr); + if (!child_reader->is_or_has_repeated_child()) { + return child_reader; + } + } + if (_children.empty()) { + return nullptr; + } + return _children[0].get(); +} + +Status StructColumnReader::advance_child_past_null_parent(ParquetColumnReader* child_reader, + int64_t parent_level_idx) const { + DORIS_CHECK(child_reader != nullptr); + const int64_t next_child_cursor = parent_level_idx + 1; + if (auto* scalar_child = dynamic_cast(child_reader)) { + if (next_child_cursor > scalar_child->nested_levels_written()) { + return Status::Corruption( + "Parquet STRUCT child {} ended before null parent row in column {}", + scalar_child->name(), _name); + } + scalar_child->set_nested_build_level_cursor( + std::max(scalar_child->nested_build_level_cursor(), next_child_cursor)); + return Status::OK(); + } + if (auto* struct_child = dynamic_cast(child_reader); + struct_child != nullptr && !struct_child->is_or_has_repeated_child()) { + if (next_child_cursor > struct_child->nested_levels_written()) { + return Status::Corruption( + "Parquet STRUCT child {} ended before null parent row in column {}", + struct_child->name(), _name); + } + struct_child->set_nested_build_level_cursor( + std::max(struct_child->nested_build_level_cursor(), next_child_cursor)); + for (auto& grandchild : struct_child->_children) { + RETURN_IF_ERROR(struct_child->advance_child_past_null_parent(grandchild.get(), + parent_level_idx)); + } + return Status::OK(); + } + + int64_t child_cursor = child_reader->nested_build_level_cursor(); + const auto& child_rep_levels = child_reader->nested_repetition_levels(); + const int64_t child_levels_written = child_reader->nested_levels_written(); + while (child_cursor < child_levels_written) { + const int16_t child_rep_level = child_rep_levels[child_cursor]; + ++child_cursor; + if (!child_reader->is_or_has_repeated_child() || child_rep_level <= _repetition_level) { + break; + } + } + child_reader->set_nested_build_level_cursor(child_cursor); + return Status::OK(); +} + +Status StructColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + RETURN_IF_ERROR(load_nested_batch(rows)); + return build_nested_column(rows, column, rows_read); +} + +Status StructColumnReader::skip(int64_t rows) { + return skip_nested_rows(rows); +} + +Status StructColumnReader::load_nested_batch(int64_t rows) { + reset_nested_build_level_cursor(); + for (auto& child_reader : _children) { + DORIS_CHECK(child_reader != nullptr); + RETURN_IF_ERROR(child_reader->load_nested_batch(rows)); + } + return Status::OK(); +} + +Status StructColumnReader::load_nested_levels_batch(int64_t rows) { + reset_nested_build_level_cursor(); + for (auto& child_reader : _children) { + DORIS_CHECK(child_reader != nullptr); + RETURN_IF_ERROR(child_reader->load_nested_levels_batch(rows)); + } + return Status::OK(); +} + +Status StructColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) { + if (column.get() == nullptr) { + return Status::InvalidArgument("Invalid parquet struct build result pointer for column {}", + _name); + } + return _consume_or_build_nested_column(length_upper_bound, &column, values_read); +} + +Status StructColumnReader::consume_nested_column(int64_t length_upper_bound, + int64_t* values_consumed) { + return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); +} + +Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, + MutableColumnPtr* column, + int64_t* values_processed) { + if (values_processed == nullptr) { + return Status::InvalidArgument( + "Invalid parquet struct process result pointer for column {}", _name); + } + if (_children.empty()) { + if (column != nullptr) { + (*column)->resize((*column)->size() + static_cast(length_upper_bound)); + } + *values_processed = length_upper_bound; + return Status::OK(); + } + ColumnStruct* struct_column = nullptr; + NullMap* parent_null_map = nullptr; + if (column != nullptr) { + struct_column = struct_column_from_output(*column); + DORIS_CHECK(struct_column != nullptr); + parent_null_map = null_map_from_nullable_output(*column); + } + auto* shape_reader = shape_source_reader(); + DORIS_CHECK(shape_reader != nullptr); + const auto& def_levels = shape_reader->nested_definition_levels(); + const auto& rep_levels = shape_reader->nested_repetition_levels(); + const int64_t levels_written = shape_reader->nested_levels_written(); + + NullMap parent_nulls; + std::vector parent_level_indices; + *values_processed = 0; + int64_t level_idx = nested_build_level_cursor(); + while (level_idx < levels_written) { + const int64_t current_level_idx = level_idx; + const int16_t def_level = def_levels[level_idx]; + const int16_t rep_level = rep_levels[level_idx]; + const bool starts_parent = + !shape_reader->is_or_has_repeated_child() || rep_level <= _repetition_level; + if (starts_parent && *values_processed >= length_upper_bound) { + break; + } + ++level_idx; + if (def_level < _repeated_ancestor_definition_level) { + continue; + } + if (shape_reader->is_or_has_repeated_child() && rep_level > _repetition_level) { + continue; + } + const bool parent_is_null = def_level < _nullable_definition_level; + if (parent_is_null && !_type->is_nullable()) { + return Status::Corruption( + "Parquet STRUCT column {} contains null for non-nullable struct", _name); + } + parent_nulls.push_back(parent_is_null); + parent_level_indices.push_back(current_level_idx); + ++*values_processed; + } + set_nested_build_level_cursor(level_idx); + + std::vector child_columns; + if (column != nullptr) { + child_columns.reserve(struct_column->get_columns().size()); + for (size_t child_idx = 0; child_idx < struct_column->get_columns().size(); ++child_idx) { + child_columns.push_back(struct_column->get_column_ptr(child_idx)->assert_mutable()); + } + } + for (size_t child_idx = 0; child_idx < _children.size(); ++child_idx) { + const int output_idx = _child_output_indices[child_idx]; + if (column != nullptr && output_idx < 0) { + continue; + } + // STRUCT owns row alignment. Child readers consume only present parent rows from their + // level streams; null STRUCT parents become default placeholders in every child column. + // This mirrors Arrow's separation between struct validity and child array materialization, + // and avoids asking scalar/list/map children to invent values for an absent parent. + int64_t pending_present_rows = 0; + int64_t total_child_rows = 0; + auto flush_present_rows = [&]() -> Status { + if (pending_present_rows == 0) { + return Status::OK(); + } + int64_t child_rows = 0; + if (column != nullptr) { + RETURN_IF_ERROR(_children[child_idx]->build_nested_column( + pending_present_rows, child_columns[output_idx], &child_rows)); + } else { + RETURN_IF_ERROR(_children[child_idx]->consume_nested_column(pending_present_rows, + &child_rows)); + } + if (child_rows != pending_present_rows) { + return Status::Corruption( + "Parquet STRUCT child {} built {} rows, expected {} for column {}", + _children[child_idx]->name(), child_rows, pending_present_rows, _name); + } + total_child_rows += child_rows; + pending_present_rows = 0; + return Status::OK(); + }; + for (size_t parent_idx = 0; parent_idx < parent_nulls.size(); ++parent_idx) { + const auto parent_is_null = parent_nulls[parent_idx]; + if (!parent_is_null) { + ++pending_present_rows; + continue; + } + RETURN_IF_ERROR(flush_present_rows()); + if (column != nullptr) { + child_columns[output_idx]->insert_default(); + } + RETURN_IF_ERROR(advance_child_past_null_parent(_children[child_idx].get(), + parent_level_indices[parent_idx])); + ++total_child_rows; + } + RETURN_IF_ERROR(flush_present_rows()); + if (total_child_rows != *values_processed) { + return Status::Corruption( + "Parquet STRUCT child {} built {} rows, expected {} for column {}", + _children[child_idx]->name(), total_child_rows, *values_processed, _name); + } + } + if (column != nullptr) { + for (size_t child_idx = 0; child_idx < child_columns.size(); ++child_idx) { + struct_column->get_column_ptr(child_idx) = std::move(child_columns[child_idx]); + } + append_parent_nulls(parent_null_map, parent_nulls); + } + return Status::OK(); +} + +const std::vector& StructColumnReader::nested_definition_levels() const { + auto* shape_reader = shape_source_reader(); + DORIS_CHECK(shape_reader != nullptr); + return shape_reader->nested_definition_levels(); +} + +const std::vector& StructColumnReader::nested_repetition_levels() const { + auto* shape_reader = shape_source_reader(); + DORIS_CHECK(shape_reader != nullptr); + return shape_reader->nested_repetition_levels(); +} + +int64_t StructColumnReader::nested_levels_written() const { + auto* shape_reader = shape_source_reader(); + DORIS_CHECK(shape_reader != nullptr); + return shape_reader->nested_levels_written(); +} + +bool StructColumnReader::is_or_has_repeated_child() const { + auto* shape_reader = shape_source_reader(); + return shape_reader != nullptr && shape_reader->is_or_has_repeated_child(); +} + +void StructColumnReader::advance_nested_build_level_cursor_past_parent( + int16_t parent_repetition_level) { + ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); + for (auto& child : _children) { + DORIS_CHECK(child != nullptr); + child->advance_nested_build_level_cursor_past_parent(parent_repetition_level); + } +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.h b/be/src/format_v2/parquet/reader/struct_column_reader.h new file mode 100644 index 00000000000000..3c2d6904cb36f4 --- /dev/null +++ b/be/src/format_v2/parquet/reader/struct_column_reader.h @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/reader/column_reader.h" + +namespace doris::format::parquet { + +class StructColumnReader final : public ParquetColumnReader { +public: + StructColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, + std::vector> children, + std::vector child_output_indices, + ParquetColumnReaderProfile profile = {}) + : ParquetColumnReader(schema, type, profile), + _children(std::move(children)), + _child_output_indices(std::move(child_output_indices)) { + DCHECK_EQ(_children.size(), _child_output_indices.size()); + } + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status load_nested_batch(int64_t rows) override; + Status load_nested_levels_batch(int64_t rows) override; + Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) override; + Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; + const std::vector& nested_definition_levels() const override; + const std::vector& nested_repetition_levels() const override; + int64_t nested_levels_written() const override; + bool is_or_has_repeated_child() const override; + void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; + +private: + Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, + int64_t* values_processed); + ParquetColumnReader* shape_source_reader() const; + Status advance_child_past_null_parent(ParquetColumnReader* child_reader, + int64_t parent_level_idx) const; + + std::vector> _children; // projected child readers + std::vector _child_output_indices; // child reader -> struct output position mapping +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h new file mode 100644 index 00000000000000..589154d4acc0e4 --- /dev/null +++ b/be/src/format_v2/parquet/selection_vector.h @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" + +namespace doris::format::parquet { + +struct RowRange { + int64_t start = 0; + int64_t length = 0; +}; + +struct ParquetPageSkipPlan { + int leaf_column_id = -1; + // Page ordinal is the data-page ordinal in the column chunk. It intentionally excludes + // dictionary pages, matching Arrow PageReader::set_data_page_filter(). + std::vector skipped_pages; + std::vector skipped_page_compressed_sizes; + // Row ranges covered by skipped data pages. ScalarColumnReader uses these ranges to avoid + // calling RecordReader::SkipRecords() again for pages already skipped by Arrow. + std::vector skipped_ranges; + + bool empty() const { return skipped_ranges.empty(); } + + bool should_skip_page(size_t page_idx) const { + return page_idx < skipped_pages.size() && skipped_pages[page_idx] != 0; + } + + int64_t skipped_page_compressed_size(size_t page_idx) const { + DCHECK_LT(page_idx, skipped_page_compressed_sizes.size()); + return skipped_page_compressed_sizes[page_idx]; + } +}; + +class SelectionVector { +public: + using Index = uint16_t; + + SelectionVector() = default; + + explicit SelectionVector(size_t count) { resize(count); } + + SelectionVector(Index* data, size_t count) { initialize(data, count); } + + void initialize(Index* data, size_t count) { + _owned.clear(); + _data = data; + _size = count; + } + + void resize(size_t count) { + _owned.resize(count); + _data = _owned.data(); + _size = count; + for (size_t idx = 0; idx < count; ++idx) { + _data[idx] = static_cast(idx); + } + } + + void clear() { + _owned.clear(); + _data = nullptr; + _size = 0; + } + + size_t size() const { return _size; } + + bool is_set() const { return _data != nullptr; } + + Index* data() { return _data; } + + const Index* data() const { return _data; } + + size_t get_index(size_t idx) const { + if (_data == nullptr) { + return idx; + } + return _data[idx]; + } + + void set_index(size_t idx, Index value) { _data[idx] = value; } + + Status verify(size_t count, int64_t batch_rows) const { + if (batch_rows < 0) { + return Status::InvalidArgument("Negative parquet selection batch rows {}", batch_rows); + } + if (std::cmp_greater(count, batch_rows)) { + return Status::InvalidArgument("Parquet selection count {} exceeds batch rows {}", + count, batch_rows); + } + if (_data != nullptr && count > _size) { + return Status::InvalidArgument("Parquet selection count {} exceeds vector size {}", + count, _size); + } + size_t previous = 0; + for (size_t idx = 0; idx < count; ++idx) { + const size_t current = get_index(idx); + if (std::cmp_greater_equal(current, batch_rows)) { + return Status::InvalidArgument( + "Parquet selection index {} out of range [0, {}) at position {}", current, + batch_rows, idx); + } + if (idx > 0 && current <= previous) { + return Status::InvalidArgument( + "Parquet selection index {} is not strictly greater than previous {} at " + "position {}", + current, previous, idx); + } + previous = current; + } + return Status::OK(); + } + +private: + std::vector _owned; + Index* _data = nullptr; + size_t _size = 0; +}; + +inline std::vector selection_to_ranges(const SelectionVector& selection, + uint16_t selected_rows) { + std::vector ranges; + if (selected_rows == 0) { + return ranges; + } + + int64_t range_start = selection.get_index(0); + int64_t previous = selection.get_index(0); + for (uint16_t selection_idx = 1; selection_idx < selected_rows; ++selection_idx) { + const int64_t current = selection.get_index(selection_idx); + if (current == previous + 1) { + previous = current; + continue; + } + ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + range_start = current; + previous = current; + } + ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + return ranges; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/schema_projection.cpp b/be/src/format_v2/schema_projection.cpp new file mode 100644 index 00000000000000..342f4c91898c92 --- /dev/null +++ b/be/src/format_v2/schema_projection.cpp @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/schema_projection.h" + +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" + +namespace doris::format { +namespace { + +// Rebuild the complex DataType for one already-pruned semantic ColumnDefinition node. +// +// The caller has already matched the projection against ColumnDefinition::children and preserved +// the file-local child order. This helper only mirrors those projected semantic children into the +// node type. It intentionally does not understand physical format wrappers. In particular, a MAP +// node is expected to have semantic children [key, value], even if the underlying format stores a +// wrapper such as Parquet key_value/entry. +Status rebuild_semantic_projected_type(const DataTypePtr& original_type, + const std::vector& projected_children, + DataTypePtr* projected_type) { + DORIS_CHECK(original_type != nullptr); + DORIS_CHECK(projected_type != nullptr); + + DataTypePtr nested_projected_type; + const auto primitive_type = remove_nullable(original_type)->get_primitive_type(); + switch (primitive_type) { + case TYPE_STRUCT: { + DataTypes child_types; + Strings child_names; + child_types.reserve(projected_children.size()); + child_names.reserve(projected_children.size()); + for (const auto& child : projected_children) { + child_types.push_back(child.type); + child_names.push_back(child.name); + } + nested_projected_type = std::make_shared(child_types, child_names); + break; + } + case TYPE_ARRAY: + DORIS_CHECK(projected_children.size() == 1); + nested_projected_type = std::make_shared(projected_children[0].type); + break; + case TYPE_MAP: { + DORIS_CHECK(remove_nullable(original_type)->get_primitive_type() == TYPE_MAP); + const auto* original_map_type = + assert_cast(remove_nullable(original_type).get()); + DataTypePtr key_type = original_map_type->get_key_type(); + DataTypePtr value_type; + for (const auto& child : projected_children) { + // Partial MAP projection only prunes the value subtree. The key stream must remain + // complete because it defines entry existence and offsets when materializing ColumnMap; + // the projected DataTypeMap also preserves the original key type instead of rebuilding + // it from children. If a caller includes key in the semantic child list, ignore it + // here; the presence of a value child still decides the projected value shape. + if (child.file_local_id() == 0 || child.name == "key") { + continue; + } + if (child.file_local_id() == 1 || child.name == "value") { + value_type = child.type; + } + } + if (value_type == nullptr) { + return Status::NotSupported("MAP projection for type {} contains no value child", + original_type->get_name()); + } + nested_projected_type = std::make_shared(key_type, value_type); + break; + } + default: + return Status::InvalidArgument("Cannot project children from non-complex type {}", + original_type->get_name()); + } + + *projected_type = original_type->is_nullable() ? make_nullable(nested_projected_type) + : nested_projected_type; + return Status::OK(); +} + +} // namespace + +Status project_column_definition(const ColumnDefinition& field, const LocalColumnIndex& projection, + ColumnDefinition* projected_field) { + if (projected_field == nullptr) { + return Status::InvalidArgument("projected_field is null"); + } + *projected_field = field; + if (projection.project_all_children || projection.children.empty()) { + return Status::OK(); + } + + projected_field->children.clear(); + for (const auto& child_projection : projection.children) { + if (child_projection.local_id() == -1) { + return Status::InvalidArgument("Empty projection path for field {}", field.name); + } + const auto child_it = + std::ranges::find_if(field.children, [&](const ColumnDefinition& child) { + return child.file_local_id() == child_projection.local_id(); + }); + if (child_it == field.children.end()) { + return Status::InvalidArgument("Invalid projection child id {} for field {}", + child_projection.local_id(), field.name); + } + } + for (const auto& child : field.children) { + const auto child_projection_it = + std::ranges::find_if(projection.children, [&](const LocalColumnIndex& child_proj) { + return child_proj.local_id() == child.file_local_id(); + }); + if (child_projection_it == projection.children.end()) { + continue; + } + ColumnDefinition projected_child; + RETURN_IF_ERROR(project_column_definition(child, *child_projection_it, &projected_child)); + projected_field->children.push_back(std::move(projected_child)); + } + if (projected_field->children.empty()) { + return Status::NotSupported("Projection for field {} contains no children", field.name); + } + + return rebuild_semantic_projected_type(field.type, projected_field->children, + &projected_field->type); +} + +} // namespace doris::format diff --git a/be/src/format_v2/schema_projection.h b/be/src/format_v2/schema_projection.h new file mode 100644 index 00000000000000..c2125d66931631 --- /dev/null +++ b/be/src/format_v2/schema_projection.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "format_v2/file_reader.h" + +namespace doris::format { + +// Build a projected file-local semantic schema node from a full schema node and a nested +// LocalColumnIndex projection. +// +// This module is deliberately about semantic ColumnDefinition trees, not physical file-format +// trees. FileReader::get_schema() returns file-local columns after type conversion to Doris +// DataType, and their children must follow Doris semantics: +// +// STRUCT children = fields +// ARRAY children = [element] +// MAP children = [key, value] +// +// Format-specific wrappers, such as Parquet MAP key_value/entry nodes, are intentionally hidden +// from this API. A format reader that needs those wrappers for its physical reader tree should +// translate the semantic projection back to its physical layout internally. +// +// The function does three things: +// - Copies `field` metadata to `projected_field`. +// - Recursively prunes children according to `projection.children`, matching children by +// ColumnDefinition::file_local_id() rather than vector ordinal. The root projection id is not +// interpreted here because the caller has already selected `field`. +// - Rebuilds the node DataType from the projected semantic children so the returned definition is +// self-consistent. STRUCT uses projected child names/types, ARRAY uses the projected element +// type, and MAP preserves the original key type while rebuilding the projected value type. +// +// A full projection copies `field` unchanged. Partial MAP projection only uses the value child for +// type rebuilding. MAP is materialized as offsets + keys + values, so the reader must still read +// the complete key stream to build entry shape and offsets. If the semantic projection includes +// the key child, it is ignored here; key-only MAP projections are rejected because they do not +// define a value shape. +Status project_column_definition(const ColumnDefinition& field, const LocalColumnIndex& projection, + ColumnDefinition* projected_field); + +} // namespace doris::format diff --git a/be/src/format_v2/table/hive_reader.cpp b/be/src/format_v2/table/hive_reader.cpp new file mode 100644 index 00000000000000..783ccbf95034fb --- /dev/null +++ b/be/src/format_v2/table/hive_reader.cpp @@ -0,0 +1,246 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/hive_reader.h" + +#include +#include +#include + +#include "common/consts.h" +#include "format_v2/column_mapper.h" +#include "format_v2/file_reader.h" +#include "runtime/runtime_state.h" + +namespace doris::format::hive { +namespace { + +TFileFormatType::type format_type_from_context(const format::ProjectedColumnBuildContext& context) { + DORIS_CHECK(context.scan_params != nullptr); + if (context.range != nullptr && context.range->__isset.format_type) { + return context.range->format_type; + } + return context.scan_params->format_type; +} + +bool use_column_position_mapping(const format::ProjectedColumnBuildContext& context) { + if (context.runtime_state == nullptr || context.scan_params == nullptr) { + return false; + } + switch (format_type_from_context(context)) { + case TFileFormatType::FORMAT_PARQUET: + return !context.runtime_state->query_options().hive_parquet_use_column_names; + case TFileFormatType::FORMAT_ORC: + return !context.runtime_state->query_options().hive_orc_use_column_names; + default: + return false; + } +} + +bool is_file_column_position_slot(const TFileScanSlotInfo& slot_info, + const std::string& column_name) { + if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) || + column_name == BeConsts::ICEBERG_ROWID_COL) { + return false; + } + if (slot_info.__isset.is_file_slot) { + return slot_info.is_file_slot; + } + return true; +} + +bool is_hive1_orc_column_name(std::string_view name) { + if (name.size() <= 4 || name.substr(0, 4) != "_col") { + return false; + } + return std::ranges::all_of(name.substr(4), [](unsigned char c) { return std::isdigit(c); }); +} + +bool is_hive1_orc_file_schema(const std::vector& file_schema) { + bool has_data_column = false; + for (const auto& field : file_schema) { + if (field.column_type != format::ColumnType::DATA_COLUMN) { + continue; + } + has_data_column = true; + if (!is_hive1_orc_column_name(field.name)) { + return false; + } + } + return has_data_column; +} + +bool is_file_column_for_hive1_position_mapping(const format::ColumnDefinition& column) { + if (column.column_type != format::ColumnType::DATA_COLUMN || column.is_partition_key) { + return false; + } + return !column.name.starts_with(BeConsts::GLOBAL_ROWID_COL) && + column.name != BeConsts::ICEBERG_ROWID_COL; +} + +void add_name_mapping(std::vector* name_mapping, const std::string& alias) { + DORIS_CHECK(name_mapping != nullptr); + if (std::ranges::find(*name_mapping, alias) == name_mapping->end()) { + name_mapping->push_back(alias); + } +} + +} // namespace + +Status HiveReader::prepare_split(const format::SplitReadOptions& options) { + if (options.current_split_format != _format) { + return Status::InternalError( + "Hive scan expects all splits to use the same file format, " + "initialized_format={}, current_split_format={}", + static_cast(_format), static_cast(options.current_split_format)); + } + return format::TableReader::prepare_split(options); +} + +format::TableColumnMappingMode HiveReader::mapping_mode() const { + // Hive-specific behavior: choose the column matching mode based on file format and the + // matching session variable. + // - hive_orc_use_column_names / hive_parquet_use_column_names == true + // => BY_NAME (modern Hive default, match by column name) + // - those options == false + // => BY_INDEX (mainly for Hive1 ORC `_col0` / `_col1`, match by top-level position; + // Parquet exposes the same switch for consistency) + // TableReader updates `_format` in prepare_split(), so this is evaluated per split. + DORIS_CHECK(_runtime_state != nullptr); + const auto& query_options = _runtime_state->query_options(); + bool use_column_names = true; + if (_format == format::FileFormat::ORC) { + use_column_names = query_options.hive_orc_use_column_names; + } else if (_format == format::FileFormat::PARQUET) { + use_column_names = query_options.hive_parquet_use_column_names; + } else if (_format == format::FileFormat::CSV || _format == format::FileFormat::TEXT || + _format == format::FileFormat::JSON) { + // Hive CSV/TEXT/JSON readers synthesize a file-local schema from FE-provided file slots + // because these formats do not carry embedded column names or field ids. The scan params' + // format-specific attributes still tell the physical reader how to read values, while the + // table-level mapper can safely match the synthesized file schema by table column name. + use_column_names = true; + } else { + DORIS_CHECK(false) << "HiveReader does not support this file reader format"; + } + + return use_column_names ? format::TableColumnMappingMode::BY_NAME + : format::TableColumnMappingMode::BY_INDEX; +} + +Status HiveReader::annotate_projected_column(const TFileScanSlotInfo& slot_info, + format::ProjectedColumnBuildContext* context, + format::ColumnDefinition* column) const { + RETURN_IF_ERROR(format::TableReader::annotate_projected_column(slot_info, context, column)); + DORIS_CHECK(context != nullptr); + DORIS_CHECK(column != nullptr); + if (!use_column_position_mapping(*context) || + !is_file_column_position_slot(slot_info, column->name)) { + return Status::OK(); + } + const auto* scan_params = context->scan_params; + DORIS_CHECK(scan_params != nullptr); + if (!scan_params->__isset.column_idxs || + context->next_file_column_idx >= scan_params->column_idxs.size()) { + return Status::InvalidArgument( + "Hive positional column mapping is missing file index for column '{}', " + "required file slot ordinal={}, column_idxs_size={}", + column->name, context->next_file_column_idx, + scan_params->__isset.column_idxs ? scan_params->column_idxs.size() : 0); + } + const auto file_index = scan_params->column_idxs[context->next_file_column_idx]; + if (file_index < 0) { + return Status::InvalidArgument( + "Hive positional column mapping has negative file index {} for column '{}'", + file_index, column->name); + } + column->identifier = Field::create_field(file_index); + ++context->next_file_column_idx; + return Status::OK(); +} + +Status HiveReader::validate_projected_columns( + const format::ProjectedColumnBuildContext& context) const { + if (!use_column_position_mapping(context)) { + return Status::OK(); + } + DORIS_CHECK(context.scan_params != nullptr); + if (context.scan_params->__isset.column_idxs && + context.next_file_column_idx != context.scan_params->column_idxs.size()) { + return Status::InvalidArgument( + "Hive positional column mapping has unused file indexes: consumed={}, " + "column_idxs_size={}", + context.next_file_column_idx, context.scan_params->column_idxs.size()); + } + return Status::OK(); +} + +Status HiveReader::annotate_file_schema(std::vector* file_schema) { + DORIS_CHECK(file_schema != nullptr); + if (_format != format::FileFormat::ORC || _runtime_state == nullptr || + !_runtime_state->query_options().hive_orc_use_column_names || + !is_hive1_orc_file_schema(*file_schema)) { + return Status::OK(); + } + + DORIS_CHECK(_scan_params != nullptr); + if (!_scan_params->__isset.column_idxs) { + return Status::InvalidArgument( + "Hive ORC Hive1-style name mapping is missing positional column indexes"); + } + + size_t next_file_column_idx = 0; + for (const auto& table_column : _projected_columns) { + if (!is_file_column_for_hive1_position_mapping(table_column)) { + continue; + } + if (next_file_column_idx >= _scan_params->column_idxs.size()) { + return Status::InvalidArgument( + "Hive ORC Hive1-style name mapping is missing file index for column '{}', " + "required file slot ordinal={}, column_idxs_size={}", + table_column.name, next_file_column_idx, _scan_params->column_idxs.size()); + } + const auto file_index = _scan_params->column_idxs[next_file_column_idx]; + if (file_index < 0) { + return Status::InvalidArgument( + "Hive ORC Hive1-style name mapping has negative file index {} for column '{}'", + file_index, table_column.name); + } + ++next_file_column_idx; + if (static_cast(file_index) >= file_schema->size()) { + continue; + } + + auto& file_column = (*file_schema)[static_cast(file_index)]; + add_name_mapping(&file_column.name_mapping, table_column.name); + if (table_column.has_identifier_name()) { + add_name_mapping(&file_column.name_mapping, table_column.get_identifier_name()); + } + for (const auto& alias : table_column.name_mapping) { + add_name_mapping(&file_column.name_mapping, alias); + } + } + if (next_file_column_idx != _scan_params->column_idxs.size()) { + return Status::InvalidArgument( + "Hive ORC Hive1-style name mapping has unused file indexes: consumed={}, " + "column_idxs_size={}", + next_file_column_idx, _scan_params->column_idxs.size()); + } + return Status::OK(); +} + +} // namespace doris::format::hive diff --git a/be/src/format_v2/table/hive_reader.h b/be/src/format_v2/table/hive_reader.h new file mode 100644 index 00000000000000..8dbd99b9c971a6 --- /dev/null +++ b/be/src/format_v2/table/hive_reader.h @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "format_v2/table_reader.h" + +namespace doris::format::hive { +// now hive self only support mixed with orc/parquet files in table and different partitions. +// But if mixed with orc/parquet files in table and same partition, will failed when read. +// now fe will plan table format for all files dirctly, and BE could not handle mixed files also. +class HiveReader final : public format::TableReader { +public: + ENABLE_FACTORY_CREATOR(HiveReader); + ~HiveReader() final = default; + + Status prepare_split(const format::SplitReadOptions& options) override; + format::TableColumnMappingMode mapping_mode() const override; + Status annotate_projected_column(const TFileScanSlotInfo& slot_info, + format::ProjectedColumnBuildContext* context, + format::ColumnDefinition* column) const override; + Status validate_projected_columns( + const format::ProjectedColumnBuildContext& context) const override; +#ifdef BE_TEST + Status TEST_annotate_file_schema(std::vector* file_schema) { + return annotate_file_schema(file_schema); + } +#endif + +protected: + Status annotate_file_schema(std::vector* file_schema) override; +}; + +} // namespace doris::format::hive diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp new file mode 100644 index 00000000000000..f37d24bd888240 --- /dev/null +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/hudi_reader.h" + +#include + +#include "exprs/vexpr_context.h" +#include "format_v2/column_mapper.h" +#include "format_v2/jni/hudi_jni_reader.h" +#include "format_v2/table/schema_history_util.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::hudi { + +Status HudiReader::prepare_split(const format::SplitReadOptions& options) { + _split_schema_id = -1; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.hudi_params && + options.current_range.table_format_params.hudi_params.__isset.schema_id) { + _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + } + return format::TableReader::prepare_split(options); +} + +format::TableColumnMappingMode HudiReader::mapping_mode() const { + return format::can_map_by_history_schema(_scan_params, _split_schema_id) + ? format::TableColumnMappingMode::BY_FIELD_ID + : format::TableColumnMappingMode::BY_NAME; +} + +Status HudiReader::annotate_file_schema(std::vector* file_schema) { + DORIS_CHECK(file_schema != nullptr); + if (mapping_mode() != format::TableColumnMappingMode::BY_FIELD_ID) { + return Status::OK(); + } + return format::annotate_file_schema_from_history(_scan_params, _split_schema_id, file_schema); +} + +Status HudiHybridReader::init(format::TableReadOptions&& options) { + return format::TableReader::init(std::move(options)); +} + +Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) { + RETURN_IF_ERROR(_ensure_current_split_reader(options)); + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->prepare_split(options); +} + +Status HudiHybridReader::get_block(Block* block, bool* eos) { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->get_block(block, eos); +} + +bool HudiHybridReader::current_split_pruned() const { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->current_split_pruned(); +} + +Status HudiHybridReader::abort_split() { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->abort_split(); +} + +Status HudiHybridReader::close() { + Status close_status = Status::OK(); + if (_native_reader != nullptr) { + close_status = _native_reader->close(); + } + if (_jni_reader != nullptr) { + auto status = _jni_reader->close(); + if (!status.ok() && close_status.ok()) { + close_status = std::move(status); + } + } + _current_split_reader = nullptr; + return close_status; +} + +void HudiHybridReader::set_batch_size(size_t batch_size) { + format::TableReader::set_batch_size(batch_size); + if (_native_reader != nullptr) { + _native_reader->set_batch_size(_batch_size); + } + if (_jni_reader != nullptr) { + _jni_reader->set_batch_size(_batch_size); + } +} + +Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { + DORIS_CHECK(_scan_params != nullptr); + if (_is_jni_split(*_scan_params, options.current_range)) { + if (_jni_reader == nullptr) { + _jni_reader = std::make_unique(); + RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); + } + _current_split_reader = _jni_reader.get(); + } else { + format::FileFormat file_format; + RETURN_IF_ERROR(_to_file_format(*_scan_params, options.current_range, &file_format)); + if (_native_reader == nullptr) { + _native_reader = format::hudi::HudiReader::create_unique(); + RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); + } + _current_split_reader = _native_reader.get(); + } + return Status::OK(); +} + +Status HudiHybridReader::_init_child_reader(format::TableReader* reader, + format::FileFormat file_format) { + DORIS_CHECK(reader != nullptr); + VExprContextSPtrs conjuncts; + RETURN_IF_ERROR(_clone_conjuncts(&conjuncts)); + RETURN_IF_ERROR(reader->init({ + .projected_columns = _projected_columns, + .conjuncts = std::move(conjuncts), + .format = file_format, + .scan_params = _scan_params, + .io_ctx = _io_ctx, + .runtime_state = _runtime_state, + .scanner_profile = _scanner_profile, + .push_down_agg_type = _push_down_agg_type, + .condition_cache_digest = _condition_cache_digest, + })); + // Zero means no adaptive prediction has been produced yet. Preserve the child's normal + // runtime default until FileScannerV2 supplies the first positive prediction. + if (_batch_size > 0) { + reader->set_batch_size(_batch_size); + } + return Status::OK(); +} + +Status HudiHybridReader::_clone_conjuncts(VExprContextSPtrs* conjuncts) const { + DORIS_CHECK(conjuncts != nullptr); + conjuncts->clear(); + conjuncts->reserve(_conjuncts.size()); + for (const auto& conjunct : _conjuncts) { + VExprSPtr root; + RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); + conjuncts->push_back(VExprContext::create_shared(std::move(root))); + } + return Status::OK(); +} + +TFileFormatType::type HudiHybridReader::_range_format_type(const TFileScanRangeParams& params, + const TFileRangeDesc& range) { + return range.__isset.format_type ? range.format_type : params.format_type; +} + +bool HudiHybridReader::_is_jni_split(const TFileScanRangeParams& params, + const TFileRangeDesc& range) { + return _range_format_type(params, range) == TFileFormatType::FORMAT_JNI; +} + +Status HudiHybridReader::_to_file_format(const TFileScanRangeParams& params, + const TFileRangeDesc& range, + format::FileFormat* file_format) { + DORIS_CHECK(file_format != nullptr); + const auto format_type = _range_format_type(params, range); + switch (format_type) { + case TFileFormatType::FORMAT_PARQUET: + *file_format = format::FileFormat::PARQUET; + return Status::OK(); + case TFileFormatType::FORMAT_ORC: + *file_format = format::FileFormat::ORC; + return Status::OK(); + default: + return Status::NotSupported("Unsupported native Hudi file format {}", + to_string(format_type)); + } +} + +} // namespace doris::format::hudi diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h new file mode 100644 index 00000000000000..e22c6bd866f061 --- /dev/null +++ b/be/src/format_v2/table/hudi_reader.h @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "format_v2/table_reader.h" + +namespace doris::format::hudi { + +class HudiReader final : public format::TableReader { +public: + ENABLE_FACTORY_CREATOR(HudiReader); + ~HudiReader() final = default; + + Status prepare_split(const format::SplitReadOptions& options) override; + +#ifdef BE_TEST + void TEST_set_scan_params(TFileScanRangeParams* params) { _scan_params = params; } + format::TableColumnMappingMode TEST_mapping_mode() const { return mapping_mode(); } + Status TEST_annotate_file_schema(std::vector* file_schema) { + return annotate_file_schema(file_schema); + } +#endif + +protected: + format::TableColumnMappingMode mapping_mode() const override; + Status annotate_file_schema(std::vector* file_schema) override; + +private: + int64_t _split_schema_id = -1; +}; + +// Hudi MOR scans can contain both JNI splits that need log-file merge semantics and native +// data-file splits without delta logs in the same SplitSource. FileScannerV2 owns one table reader +// for the scanner lifetime, so this reader keeps native and JNI child readers internally and +// dispatches each split to the matching child reader. +class HudiHybridReader final : public format::TableReader { +public: + ~HudiHybridReader() override = default; + + Status init(format::TableReadOptions&& options) override; + Status prepare_split(const format::SplitReadOptions& options) override; + Status get_block(Block* block, bool* eos) override; + bool current_split_pruned() const override; + Status abort_split() override; + Status close() override; + void set_batch_size(size_t batch_size) override; + +#ifdef BE_TEST + void TEST_install_batch_size_children() { + _native_reader = std::make_unique(); + _jni_reader = std::make_unique(); + } + std::pair TEST_child_batch_sizes() const { + return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; + } +#endif + +private: + Status _ensure_current_split_reader(const format::SplitReadOptions& options); + Status _init_child_reader(format::TableReader* reader, format::FileFormat file_format); + Status _clone_conjuncts(VExprContextSPtrs* conjuncts) const; + static TFileFormatType::type _range_format_type(const TFileScanRangeParams& params, + const TFileRangeDesc& range); + static bool _is_jni_split(const TFileScanRangeParams& params, const TFileRangeDesc& range); + static Status _to_file_format(const TFileScanRangeParams& params, const TFileRangeDesc& range, + format::FileFormat* file_format); + + std::unique_ptr _native_reader; // handle native parquet/orc splits + std::unique_ptr _jni_reader; // handle MOR JNI splits + format::TableReader* _current_split_reader = nullptr; +}; + +} // namespace doris::format::hudi diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp new file mode 100644 index 00000000000000..b19b1f8bf19553 --- /dev/null +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -0,0 +1,998 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_reader.h" + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/define_primitive_type.h" +#include "core/field.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/deletion_vector_reader.h" +#include "format_v2/expr/cast.h" +#include "format_v2/expr/equality_delete_predicate.h" +#include "format_v2/orc/orc_reader.h" +#include "format_v2/parquet/parquet_reader.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/table_reader.h" +#include "io/file_factory.h" +#include "util/url_coding.h" + +namespace doris::format::iceberg { + +static constexpr const char* ROW_LINEAGE_ROW_ID = "_row_id"; +static constexpr int32_t ROW_LINEAGE_ROW_ID_FIELD_ID = 2147483540; + +template +static std::string join_values_for_debug(const std::vector& values) { + std::ostringstream out; + out << "["; + for (size_t idx = 0; idx < values.size(); ++idx) { + if (idx > 0) { + out << ", "; + } + out << values[idx]; + } + out << "]"; + return out.str(); +} + +static bool is_projected_row_lineage_row_id(const format::ColumnDefinition& column) { + // Iceberg row lineage columns can be bound by field id when a mapper has already been built, + // but customize_file_scan_request() is also exercised directly by scan-request tests before the + // mapper exists. In that path, inspect the projected table schema so row-position dependencies + // are still added for `_row_id`. + return column.name == ROW_LINEAGE_ROW_ID || + (column.has_identifier_field_id() && + column.get_identifier_field_id() == ROW_LINEAGE_ROW_ID_FIELD_ID); +} + +static bool is_projected_iceberg_rowid(const format::ColumnDefinition& column) { + return column.name == BeConsts::ICEBERG_ROWID_COL; +} + +static Status build_missing_equality_delete_key_expr(const format::ColumnDefinition& table_field, + const DataTypePtr& delete_key_type, + VExprSPtr* key_expr) { + DORIS_CHECK(delete_key_type != nullptr); + DORIS_CHECK(key_expr != nullptr); + if (!table_field.initial_default_value.has_value()) { + // A newly added optional field without an initial default is logically NULL in older + // files. EqualityDeletePredicate treats NULL == NULL as a match. + *key_expr = VLiteral::create_shared(make_nullable(delete_key_type), Field()); + return Status::OK(); + } + + Field initial_default; + if (table_field.initial_default_value_is_base64 || + table_field.type->get_primitive_type() == TYPE_VARBINARY) { + // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its + // Doris mapping. Keep the VARBINARY fallback for scan descriptors produced before that + // marker existed. Decode before parsing so STRING/CHAR and VARBINARY all compare against + // the raw bytes stored in equality-delete files. + std::string decoded_default; + if (!base64_decode(*table_field.initial_default_value, &decoded_default)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + table_field.name); + } + if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { + initial_default = Field::create_field(StringView(decoded_default)); + } else { + DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); + initial_default = Field::create_field(decoded_default); + } + } else { + // An added field's initial default is its logical value in every older data file that lacks + // the physical column. FE normalizes the string for the current Doris table type. + RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( + *table_field.initial_default_value, initial_default)); + } + + auto literal = VLiteral::create_shared(table_field.type, initial_default); + if (table_field.type->equals(*delete_key_type)) { + *key_expr = std::move(literal); + return Status::OK(); + } + auto cast_expr = Cast::create_shared(delete_key_type); + cast_expr->add_child(std::move(literal)); + *key_expr = std::move(cast_expr); + return Status::OK(); +} + +static std::string iceberg_delete_file_debug_string(const TIcebergDeleteFileDesc& delete_file) { + std::ostringstream out; + out << "TIcebergDeleteFileDesc{path=" << (delete_file.__isset.path ? delete_file.path : "null") + << ", content=" << (delete_file.__isset.content ? delete_file.content : -1) + << ", file_format=" + << (delete_file.__isset.file_format ? static_cast(delete_file.file_format) : -1) + << ", position_lower_bound=" + << (delete_file.__isset.position_lower_bound ? delete_file.position_lower_bound : -1) + << ", position_upper_bound=" + << (delete_file.__isset.position_upper_bound ? delete_file.position_upper_bound : -1) + << ", field_ids=" + << (delete_file.__isset.field_ids ? join_values_for_debug(delete_file.field_ids) : "[]") + << ", content_offset=" + << (delete_file.__isset.content_offset ? delete_file.content_offset : -1) + << ", content_size_in_bytes=" + << (delete_file.__isset.content_size_in_bytes ? delete_file.content_size_in_bytes : -1) + << "}"; + return out.str(); +} + +static std::string iceberg_delete_files_debug_string( + const std::vector& delete_files) { + std::ostringstream out; + out << "["; + for (size_t idx = 0; idx < delete_files.size(); ++idx) { + if (idx > 0) { + out << ", "; + } + out << iceberg_delete_file_debug_string(delete_files[idx]); + } + out << "]"; + return out.str(); +} + +static std::string iceberg_params_debug_string(const std::optional& params) { + if (!params.has_value()) { + return "null"; + } + const auto& iceberg_params = *params; + std::ostringstream out; + out << "TIcebergFileDesc{format_version=" + << (iceberg_params.__isset.format_version ? iceberg_params.format_version : -1) + << ", content=" << (iceberg_params.__isset.content ? iceberg_params.content : -1) + << ", original_file_path=" + << (iceberg_params.__isset.original_file_path ? iceberg_params.original_file_path : "null") + << ", row_count=" << (iceberg_params.__isset.row_count ? iceberg_params.row_count : -1) + << ", partition_spec_id=" + << (iceberg_params.__isset.partition_spec_id ? iceberg_params.partition_spec_id : 0) + << ", has_partition_data_json=" << iceberg_params.__isset.partition_data_json + << ", first_row_id=" + << (iceberg_params.__isset.first_row_id ? iceberg_params.first_row_id : -1) + << ", last_updated_sequence_number=" + << (iceberg_params.__isset.last_updated_sequence_number + ? iceberg_params.last_updated_sequence_number + : -1) + << ", delete_file_count=" + << (iceberg_params.__isset.delete_files ? iceberg_params.delete_files.size() : 0) + << ", delete_files=" + << (iceberg_params.__isset.delete_files + ? iceberg_delete_files_debug_string(iceberg_params.delete_files) + : "[]") + << "}"; + return out.str(); +} + +IcebergTableReader::PositionDeleteRowsCollector::PositionDeleteRowsCollector( + PositionDeleteFile* rows_by_data_file) + : _rows_by_data_file(rows_by_data_file) { + DORIS_CHECK(_rows_by_data_file != nullptr); +} + +Status IcebergTableReader::PositionDeleteRowsCollector::collect(const Block& block, + size_t read_rows) { + if (read_rows == 0) { + return Status::OK(); + } + const auto& file_path_column = assert_cast( + *remove_nullable((block.get_by_position(ICEBERG_FILE_PATH_BLOCK_POSITION).column))); + const auto& pos_column = assert_cast( + *remove_nullable(block.get_by_position(ICEBERG_ROW_POS_BLOCK_POSITION).column)); + for (size_t row = 0; row < read_rows; ++row) { + const auto file_path = file_path_column.get_data_at(row).to_string(); + (*_rows_by_data_file)[file_path].push_back(pos_column.get_element(row)); + } + return Status::OK(); +} + +Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options) { + _row_lineage_columns = {}; + _iceberg_params.reset(); + _delete_predicates_initialized = false; + _position_delete_rows_storage.clear(); + _equality_delete_filters.clear(); + _split_cache = options.cache; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.iceberg_params) { + const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; + _iceberg_params = iceberg_params; + if (iceberg_params.__isset.first_row_id) { + _row_lineage_columns.first_row_id = iceberg_params.first_row_id; + } + if (iceberg_params.__isset.last_updated_sequence_number) { + _row_lineage_columns.last_updated_sequence_number = + iceberg_params.last_updated_sequence_number; + } + } + RETURN_IF_ERROR(TableReader::prepare_split(options)); + if (current_split_pruned()) { + return Status::OK(); + } + if (_is_table_level_count_active()) { + return Status::OK(); + } + RETURN_IF_ERROR(_init_delete_predicates(options.current_range.table_format_params)); + return Status::OK(); +} + +std::string IcebergTableReader::debug_string() const { + size_t position_delete_file_count = 0; + size_t equality_delete_file_count = 0; + size_t deletion_vector_file_count = 0; + if (_iceberg_params.has_value() && _iceberg_params->__isset.delete_files) { + for (const auto& delete_file : _iceberg_params->delete_files) { + if (!delete_file.__isset.content) { + continue; + } + if (delete_file.content == POSITION_DELETE) { + ++position_delete_file_count; + } else if (delete_file.content == EQUALITY_DELETE) { + ++equality_delete_file_count; + } else if (delete_file.content == DELETION_VECTOR) { + ++deletion_vector_file_count; + } + } + } + + std::ostringstream equality_filters; + equality_filters << "["; + for (size_t idx = 0; idx < _equality_delete_filters.size(); ++idx) { + if (idx > 0) { + equality_filters << ", "; + } + const auto& filter = _equality_delete_filters[idx]; + equality_filters << "EqualityDeleteFilter{field_ids=" + << join_values_for_debug(filter.field_ids) << ", key_types=["; + for (size_t type_idx = 0; type_idx < filter.key_types.size(); ++type_idx) { + if (type_idx > 0) { + equality_filters << ", "; + } + equality_filters << (filter.key_types[type_idx] == nullptr + ? "null" + : filter.key_types[type_idx]->get_name()); + } + equality_filters << "], delete_block_rows=" << filter.delete_block.rows() + << ", delete_block_columns=" << filter.delete_block.columns() << "}"; + } + equality_filters << "]"; + + std::ostringstream out; + out << "IcebergTableReader{base=" << format::TableReader::debug_string() + << ", iceberg_params=" << iceberg_params_debug_string(_iceberg_params) + << ", row_lineage_first_row_id=" << _row_lineage_columns.first_row_id + << ", row_lineage_last_updated_sequence_number=" + << _row_lineage_columns.last_updated_sequence_number + << ", need_row_lineage_row_id=" << _need_row_lineage_row_id() + << ", need_iceberg_rowid=" << _need_iceberg_rowid() + << ", row_position_block_position=" << _row_position_block_position + << ", delete_predicates_initialized=" << _delete_predicates_initialized + << ", position_delete_file_count=" << position_delete_file_count + << ", equality_delete_file_count=" << equality_delete_file_count + << ", deletion_vector_file_count=" << deletion_vector_file_count + << ", position_delete_rows_storage_count=" << _position_delete_rows_storage.size() + << ", equality_delete_filter_count=" << _equality_delete_filters.size() + << ", equality_delete_filters=" << equality_filters.str() << "}"; + return out.str(); +} + +Status IcebergTableReader::materialize_virtual_columns(Block* table_block) { + for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size(); + ++column_idx) { + const auto& mapping = _data_reader.column_mapper->mappings()[column_idx]; + switch (mapping.virtual_column_type) { + case format::TableVirtualColumnType::ROW_ID: + RETURN_IF_ERROR(_materialize_row_lineage_row_id(table_block, column_idx)); + break; + case format::TableVirtualColumnType::LAST_UPDATED_SEQUENCE_NUMBER: + RETURN_IF_ERROR( + _materialize_row_lineage_last_updated_sequence_number(table_block, column_idx)); + break; + case format::TableVirtualColumnType::ICEBERG_ROWID: + RETURN_IF_ERROR(_materialize_iceberg_rowid(table_block, column_idx)); + break; + case format::TableVirtualColumnType::INVALID: + break; + } + } + return Status::OK(); +} + +Status IcebergTableReader::customize_file_scan_request(format::FileScanRequest* file_request) { + RETURN_IF_ERROR(TableReader::customize_file_scan_request(file_request)); + if ((_row_lineage_columns.first_row_id >= 0 && _need_row_lineage_row_id()) || + _need_iceberg_rowid()) { + RETURN_IF_ERROR(_append_row_position_output_column(file_request)); + } + RETURN_IF_ERROR(_append_equality_delete_predicates(file_request)); + return Status::OK(); +} + +bool IcebergTableReader::_supports_aggregate_pushdown(TPushAggOp::type agg_type) const { + if (!TableReader::_supports_aggregate_pushdown(agg_type)) { + return false; + } + return _equality_delete_filters.empty(); +} + +Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, + DeleteFileDesc* desc, + bool* has_delete_file) { + DORIS_CHECK(desc != nullptr); + DORIS_CHECK(has_delete_file != nullptr); + *has_delete_file = false; + if (!t_desc.__isset.iceberg_params) { + return Status::OK(); + } + const auto& iceberg_params = t_desc.iceberg_params; + if (!iceberg_params.__isset.format_version || + iceberg_params.format_version < MIN_SUPPORT_DELETE_FILES_VERSION || + !iceberg_params.__isset.delete_files || iceberg_params.delete_files.empty()) { + return Status::OK(); + } + + const TIcebergDeleteFileDesc* deletion_vector = nullptr; + for (const auto& delete_file : iceberg_params.delete_files) { + if (!delete_file.__isset.content || delete_file.content != DELETION_VECTOR) { + continue; + } + if (deletion_vector != nullptr) { + return Status::DataQualityError("This iceberg data file has multiple DVs."); + } + deletion_vector = &delete_file; + } + if (deletion_vector == nullptr) { + return Status::OK(); + } + if (!deletion_vector->__isset.content_offset || + !deletion_vector->__isset.content_size_in_bytes) { + return Status::InternalError("Deletion vector is missing content offset or length"); + } + + const std::string data_file_path = iceberg_params.__isset.original_file_path + ? iceberg_params.original_file_path + : _data_file_path(); + desc->key = build_iceberg_deletion_vector_cache_key(data_file_path, *deletion_vector); + desc->path = deletion_vector->path; + desc->start_offset = deletion_vector->content_offset; + desc->size = deletion_vector->content_size_in_bytes; + desc->file_size = -1; + desc->format = DeleteFileDesc::Format::ICEBERG; + *has_delete_file = true; + return Status::OK(); +} + +Status IcebergTableReader::_init_delete_predicates(const TTableFormatFileDesc& t_desc) { + if (!t_desc.__isset.iceberg_params || _delete_predicates_initialized) { + _delete_predicates_initialized = true; + return Status::OK(); + } + const auto& iceberg_params = t_desc.iceberg_params; + if (!iceberg_params.__isset.format_version || + iceberg_params.format_version < MIN_SUPPORT_DELETE_FILES_VERSION || + !iceberg_params.__isset.delete_files || iceberg_params.delete_files.empty()) { + _delete_predicates_initialized = true; + return Status::OK(); + } + + std::vector position_delete_files; + std::vector equality_delete_files; + for (const auto& delete_file : iceberg_params.delete_files) { + if (!delete_file.__isset.content) { + continue; + } + if (delete_file.content == POSITION_DELETE) { + position_delete_files.push_back(delete_file); + } else if (delete_file.content == EQUALITY_DELETE) { + equality_delete_files.push_back(delete_file); + } + } + // Per Iceberg scan planning, position delete files apply only when there is no deletion vector + // for the data file. DVs and position deletes now intentionally use different in-memory + // representations, so use the Roaring pointer as the DV sentinel. + if (_deletion_vector != nullptr) { + position_delete_files.clear(); + } + // Initialize position and equality delete predicates. Position delete files contain row + // positions of deleted rows, which can be directly added to `_delete_rows`. Equality delete + // files contain values of deleted rows, which require reading the files and building + // predicates for later filtering. + if (!position_delete_files.empty()) { + RETURN_IF_ERROR(_init_position_delete_rows(position_delete_files)); + } + if (!equality_delete_files.empty()) { + RETURN_IF_ERROR(_init_equality_delete_predicates(equality_delete_files)); + } + + _delete_predicates_initialized = true; + return Status::OK(); +} + +std::shared_ptr IcebergTableReader::_delete_file_system_properties( + const TFileScanRangeParams& scan_params) { + auto system_properties = std::make_shared(); + system_properties->system_type = + scan_params.__isset.file_type ? scan_params.file_type : TFileType::FILE_LOCAL; + system_properties->properties = scan_params.properties; + system_properties->hdfs_params = scan_params.hdfs_params; + if (scan_params.__isset.broker_addresses) { + system_properties->broker_addresses.assign(scan_params.broker_addresses.begin(), + scan_params.broker_addresses.end()); + } + return system_properties; +} + +std::unique_ptr IcebergTableReader::_delete_file_description( + const TFileRangeDesc& range) { + auto file_description = std::make_unique(); + file_description->path = range.path; + file_description->file_size = range.__isset.file_size ? range.file_size : -1; + file_description->range_start_offset = range.__isset.start_offset ? range.start_offset : 0; + file_description->range_size = range.__isset.size ? range.size : -1; + if (range.__isset.fs_name) { + file_description->fs_name = range.fs_name; + } + return file_description; +} + +std::string IcebergTableReader::_data_file_path() const { + if (_iceberg_params.has_value() && _iceberg_params->__isset.original_file_path) { + return _iceberg_params->original_file_path; + } + DORIS_CHECK(_current_task != nullptr); + DORIS_CHECK(_current_task->data_file != nullptr); + return _current_task->data_file->path; +} + +Status IcebergTableReader::_append_row_position_output_column(format::FileScanRequest* request) { + const auto row_position_column_id = format::LocalColumnId(format::ROW_POSITION_COLUMN_ID); + _append_file_scan_column(request, row_position_column_id, &request->non_predicate_columns); + _row_position_block_position = request->local_positions.at(row_position_column_id).value(); + return Status::OK(); +} + +const format::ColumnDefinition* IcebergTableReader::_find_equality_delete_data_field( + const EqualityDeleteFilter& filter, size_t key_idx) const { + DORIS_CHECK(key_idx < filter.field_ids.size()); + DORIS_CHECK(key_idx < filter.field_names.size()); + if (mapping_mode() != format::TableColumnMappingMode::BY_NAME) { + const int field_id = filter.field_ids[key_idx]; + const auto field_it = std::ranges::find_if( + _data_reader.file_schema, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field.get_identifier_field_id() == field_id; + }); + return field_it == _data_reader.file_schema.end() ? nullptr : &*field_it; + } + + // Equality keys are hidden scan dependencies and need not appear in the query projection. + // Resolve their current name and aliases from the full table schema supplied by FE, falling + // back to the delete-file name when history metadata is unavailable. Reuse ColumnMapper's + // exact BY_NAME rules so case, string identifiers, and aliases on either side stay consistent. + auto table_field = _find_equality_delete_table_field(filter, key_idx); + return format::find_column_by_name(*table_field, _data_reader.file_schema); +} + +std::optional IcebergTableReader::_find_equality_delete_table_field( + const EqualityDeleteFilter& filter, size_t key_idx) const { + DORIS_CHECK(key_idx < filter.field_ids.size()); + DORIS_CHECK(key_idx < filter.field_names.size()); + const int field_id = filter.field_ids[key_idx]; + auto table_field = _find_current_table_column_by_field_id(field_id, filter.key_types[key_idx]); + if (!table_field.has_value()) { + const auto projected_field = std::ranges::find_if( + _projected_columns, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field.get_identifier_field_id() == field_id; + }); + if (projected_field != _projected_columns.end()) { + // Older scan descriptors and focused unit tests may omit history_schema_info. Keep the + // projected metadata as a compatibility fallback, but never require projection when + // the complete current schema is available. + table_field = *projected_field; + } + } + if (!table_field.has_value()) { + table_field = format::ColumnDefinition { + .identifier = {}, + .name = filter.field_names[key_idx], + .type = filter.key_types[key_idx], + }; + } + return table_field; +} + +std::string IcebergTableReader::_delete_file_cache_key(const char* prefix, + const std::string& path) const { + DORIS_CHECK(prefix != nullptr); + std::string fs_name; + if (_current_task != nullptr && _current_task->data_file != nullptr) { + fs_name = _current_task->data_file->fs_name; + } + // Delete descriptors can reuse the same path text in different filesystem namespaces. Encode + // both variable-length strings so neither an fs/path boundary nor equality field-id suffixes + // can be reinterpreted as path content; scan-level credentials/properties are shared here. + std::ostringstream key; + key << prefix << fs_name.size() << ':' << fs_name << ':' << path.size() << ':' << path; + return key.str(); +} + +void IcebergTableReader::_append_equality_delete_row_count_carrier( + format::FileScanRequest* request) { + DORIS_CHECK(request != nullptr); + // Columnar readers establish a filter batch's row count from predicate columns. If all + // equality keys are missing, the predicate consists only of NULL literals and the filter block + // would otherwise have zero rows. Read one physical column eagerly as a row-count carrier; + // normal final materialization ignores this hidden dependency. + const auto carrier_it = std::ranges::find_if( + _data_reader.file_schema, [](const format::ColumnDefinition& field) { + return field.column_type == format::ColumnType::DATA_COLUMN; + }); + DORIS_CHECK(carrier_it != _data_reader.file_schema.end()); + _append_file_scan_column(request, format::LocalColumnId(carrier_it->file_local_id()), + &request->predicate_columns); +} + +Status IcebergTableReader::_append_equality_delete_predicates(format::FileScanRequest* request) { + DORIS_CHECK(request != nullptr); + for (const auto& filter : _equality_delete_filters) { + auto delete_predicate = + std::make_shared(filter.delete_block, filter.field_ids); + DCHECK_EQ(filter.field_ids.size(), filter.key_types.size()); + bool has_missing_key = false; + for (size_t idx = 0; idx < filter.field_ids.size(); ++idx) { + const auto* field = _find_equality_delete_data_field(filter, idx); + if (field == nullptr) { + auto table_field = _find_equality_delete_table_field(filter, idx); + DORIS_CHECK(table_field.has_value()); + VExprSPtr key_expr; + RETURN_IF_ERROR(build_missing_equality_delete_key_expr( + *table_field, filter.key_types[idx], &key_expr)); + delete_predicate->add_child(key_expr); + has_missing_key = true; + continue; + } + const auto field_column_id = format::LocalColumnId(field->file_local_id()); + _append_file_scan_column(request, field_column_id, &request->predicate_columns); + const auto block_position = request->local_positions.at(field_column_id).value(); + auto slot = VSlotRef::create_shared(cast_set(block_position), + cast_set(block_position), -1, field->type, + field->name); + if (field->type->equals(*filter.key_types[idx])) { + delete_predicate->add_child(std::move(slot)); + } else { + auto cast_expr = Cast::create_shared(filter.key_types[idx]); + cast_expr->add_child(std::move(slot)); + delete_predicate->add_child(std::move(cast_expr)); + } + } + if (has_missing_key && request->predicate_columns.empty()) { + _append_equality_delete_row_count_carrier(request); + } + request->delete_conjuncts.push_back( + VExprContext::create_shared(std::move(delete_predicate))); + } + return Status::OK(); +} + +Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + std::unique_ptr* reader) { + DORIS_CHECK(delete_io_ctx != nullptr); + DORIS_CHECK(reader != nullptr); + if (!delete_file.__isset.file_format) { + return Status::InternalError("Iceberg delete file is missing file format"); + } + if (delete_file.file_format != TFileFormatType::FORMAT_PARQUET && + delete_file.file_format != TFileFormatType::FORMAT_ORC) { + return Status::NotSupported("Unsupported Iceberg delete file format {}", + delete_file.file_format); + } + auto delete_range = build_iceberg_delete_file_range(delete_file.path); + if (_current_task != nullptr && _current_task->data_file != nullptr && + !_current_task->data_file->fs_name.empty()) { + delete_range.__set_fs_name(_current_task->data_file->fs_name); + } + auto system_properties = _delete_file_system_properties(scan_params); + auto file_description = _delete_file_description(delete_range); + std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); + const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && + scan_params.enable_mapping_timestamp_tz; + if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { + *reader = std::make_unique( + system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz); + } else { + *reader = std::make_unique(system_properties, file_description, + io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz); + } + RETURN_IF_ERROR((*reader)->init(_runtime_state)); + return Status::OK(); +} + +Status IcebergTableReader::_read_position_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + PositionDeleteRowsCollector* collector) { + DORIS_CHECK(collector != nullptr); + std::unique_ptr reader; + RETURN_IF_ERROR(_create_delete_file_reader(delete_file, scan_params, delete_io_ctx, &reader)); + DORIS_CHECK(reader != nullptr); + + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + format::ColumnDefinition* file_path_field = nullptr; + format::ColumnDefinition* pos_field = nullptr; + for (auto& field : schema) { + if (field.name == ICEBERG_FILE_PATH) { + file_path_field = &field; + } else if (field.name == ICEBERG_ROW_POS) { + pos_field = &field; + } + } + if (file_path_field == nullptr || pos_field == nullptr) { + return Status::InternalError("Position delete file is missing required columns"); + } + + auto request = std::make_shared(); + request->non_predicate_columns = { + format::LocalColumnIndex::top_level( + format::LocalColumnId(file_path_field->file_local_id())), + format::LocalColumnIndex::top_level(format::LocalColumnId(pos_field->file_local_id()))}; + request->local_positions = { + {format::LocalColumnId(file_path_field->file_local_id()), + format::LocalIndex(ICEBERG_FILE_PATH_BLOCK_POSITION)}, + {format::LocalColumnId(pos_field->file_local_id()), + format::LocalIndex(ICEBERG_ROW_POS_BLOCK_POSITION)}, + }; + RETURN_IF_ERROR(reader->open(request)); + + bool eof = false; + auto build_position_delete_block = [](const format::ColumnDefinition& file_path_field, + const format::ColumnDefinition& pos_field) -> Block { + Block block; + block.insert( + {file_path_field.type->create_column(), file_path_field.type, ICEBERG_FILE_PATH}); + block.insert({pos_field.type->create_column(), pos_field.type, ICEBERG_ROW_POS}); + return block; + }; + while (!eof) { + Block block = build_position_delete_block(*file_path_field, *pos_field); + size_t read_rows = 0; + RETURN_IF_ERROR(reader->get_block(&block, &read_rows, &eof)); + RETURN_IF_ERROR(collector->collect(block, read_rows)); + } + return reader->close(); +} + +Status IcebergTableReader::_init_position_delete_rows( + const std::vector& delete_files) { + DORIS_CHECK(_split_cache != nullptr); + TFileScanRangeParams delete_scan_params = + _scan_params == nullptr ? TFileScanRangeParams() : *_scan_params; + format::DeleteRows position_delete_rows; + IcebergDeleteFileIOContext delete_io_ctx(_runtime_state); + for (const auto& delete_file : delete_files) { + Status read_status = Status::OK(); + // A position delete file normally references many data files. Cache the complete + // path-to-position map once; caching only the current data file would still rescan the + // shared delete file for every subsequent split. + auto* rows_by_data_file = + _split_cache->get( + _delete_file_cache_key("iceberg_v2_position_delete_", delete_file.path), + [&]() -> PositionDeleteRowsCollector::PositionDeleteFile* { + auto result = std::make_unique< + PositionDeleteRowsCollector::PositionDeleteFile>(); + PositionDeleteRowsCollector collector(result.get()); + read_status = _read_position_delete_file( + delete_file, delete_scan_params, &delete_io_ctx, &collector); + if (!read_status.ok()) { + return nullptr; + } + for (auto& [_, rows] : *result) { + std::ranges::sort(rows); + } + return result.release(); + }); + RETURN_IF_ERROR(read_status); + DORIS_CHECK(rows_by_data_file != nullptr); + const auto rows_it = rows_by_data_file->find(_data_file_path()); + if (rows_it == rows_by_data_file->end()) { + continue; + } + auto first = rows_it->second.begin(); + auto last = rows_it->second.end(); + // Bounds are inclusive Iceberg position statistics supplied by FE. Apply them after the + // cached per-data-file vector is sorted so irrelevant positions are sliced without a scan. + if (delete_file.__isset.position_lower_bound) { + first = std::lower_bound(first, last, delete_file.position_lower_bound); + } + if (delete_file.__isset.position_upper_bound) { + last = std::upper_bound(first, last, delete_file.position_upper_bound); + } + position_delete_rows.insert(position_delete_rows.end(), first, last); + } + if (position_delete_rows.empty()) { + return Status::OK(); + } + // Position delete files and deletion vectors both become row-position deletes for the + // common TableReader DeletePredicate path. Keep the merged rows in a member vector because + // DeletePredicate stores a reference to the vector used by _delete_rows. + _position_delete_rows_storage.insert(_position_delete_rows_storage.end(), + position_delete_rows.begin(), position_delete_rows.end()); + std::sort(_position_delete_rows_storage.begin(), _position_delete_rows_storage.end()); + _position_delete_rows_storage.erase( + std::unique(_position_delete_rows_storage.begin(), _position_delete_rows_storage.end()), + _position_delete_rows_storage.end()); + _delete_rows = &_position_delete_rows_storage; + return Status::OK(); +} + +Status IcebergTableReader::_init_equality_delete_predicates( + const std::vector& delete_files) { + DORIS_CHECK(_split_cache != nullptr); + TFileScanRangeParams delete_scan_params = + _scan_params == nullptr ? TFileScanRangeParams() : *_scan_params; + IcebergDeleteFileIOContext delete_io_ctx(_runtime_state); + for (const auto& delete_file : delete_files) { + RETURN_IF_ERROR( + _read_equality_delete_file(delete_file, delete_scan_params, &delete_io_ctx)); + } + return Status::OK(); +} + +Status IcebergTableReader::_resolve_equality_delete_fields( + const TIcebergDeleteFileDesc& delete_file, + const std::vector& schema, + std::vector* delete_fields, EqualityDeleteFilter* result) const { + DORIS_CHECK(delete_fields != nullptr); + DORIS_CHECK(result != nullptr); + for (const auto field_id : delete_file.field_ids) { + const auto field_it = + std::ranges::find_if(schema, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field_id == field.get_identifier_field_id(); + }); + if (field_it == schema.end()) { + return Status::InternalError("Can not find field id {} in equality delete file {}", + field_id, delete_file.path); + } + if (!field_it->children.empty()) { + return Status::NotSupported( + "Iceberg equality delete does not support complex column {}", field_it->name); + } + delete_fields->push_back(*field_it); + result->field_ids.push_back(field_id); + result->field_names.push_back(field_it->name); + result->key_types.push_back(field_it->type); + } + return Status::OK(); +} + +Status IcebergTableReader::_load_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + EqualityDeleteFilter* result) { + DORIS_CHECK(result != nullptr); + std::unique_ptr reader; + RETURN_IF_ERROR(_create_delete_file_reader(delete_file, scan_params, delete_io_ctx, &reader)); + DORIS_CHECK(reader != nullptr); + + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + std::vector delete_fields; + RETURN_IF_ERROR(_resolve_equality_delete_fields(delete_file, schema, &delete_fields, result)); + + auto request = std::make_shared(); + auto build_block = [](const std::vector& fields) -> Block { + Block block; + for (const auto& field : fields) { + block.insert({field.type->create_column(), field.type, field.name}); + } + return block; + }; + for (size_t idx = 0; idx < delete_fields.size(); ++idx) { + const auto local_column_id = format::LocalColumnId(delete_fields[idx].file_local_id()); + request->non_predicate_columns.push_back( + format::LocalColumnIndex::top_level(local_column_id)); + request->local_positions.emplace(local_column_id, format::LocalIndex(idx)); + } + RETURN_IF_ERROR(reader->open(request)); + + MutableBlock mutable_delete_block(build_block(delete_fields)); + bool eof = false; + while (!eof) { + Block block = build_block(delete_fields); + size_t read_rows = 0; + RETURN_IF_ERROR(reader->get_block(&block, &read_rows, &eof)); + if (read_rows > 0) { + RETURN_IF_ERROR(mutable_delete_block.merge(block)); + } + } + RETURN_IF_ERROR(reader->close()); + result->delete_block = mutable_delete_block.to_block(); + return Status::OK(); +} + +Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx) { + if (!delete_file.__isset.field_ids || delete_file.field_ids.empty()) { + return Status::InternalError("Iceberg equality delete file is missing field ids"); + } + std::ostringstream cache_key; + cache_key << _delete_file_cache_key("iceberg_v2_equality_delete_", delete_file.path); + cache_key << ':' << delete_file.field_ids.size(); + for (const auto field_id : delete_file.field_ids) { + cache_key << ':' << field_id; + } + Status read_status = Status::OK(); + // Include the ordered equality ids in the key because the same physical delete file can be + // projected with different key layouts. The cached block and its key metadata are immutable + // after construction and therefore safe to copy into each split-local predicate. + auto* cached_filter = _split_cache->get( + cache_key.str(), [&]() -> EqualityDeleteFilter* { + auto result = std::make_unique(); + read_status = _load_equality_delete_file(delete_file, scan_params, delete_io_ctx, + result.get()); + if (!read_status.ok()) { + return nullptr; + } + return result.release(); + }); + RETURN_IF_ERROR(read_status); + DORIS_CHECK(cached_filter != nullptr); + _equality_delete_filters.push_back(*cached_filter); + return Status::OK(); +} + +Status IcebergTableReader::_materialize_row_lineage_row_id(Block* table_block, size_t column_idx) { + if (_row_lineage_columns.first_row_id < 0) { + return Status::OK(); + } + DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns()); + const auto& row_position_column = assert_cast( + *_data_reader.block_template.get_by_position(_row_position_block_position).column); + DORIS_CHECK(row_position_column.size() == table_block->rows()); + auto column = IColumn::mutate( + table_block->get_by_position(column_idx).column->convert_to_full_column_if_const()); + auto* nullable_column = assert_cast(column.get()); + auto& null_map = nullable_column->get_null_map_data(); + auto& data = assert_cast(*nullable_column->get_nested_column_ptr()).get_data(); + DORIS_CHECK(null_map.size() == row_position_column.size()); + DORIS_CHECK(data.size() == row_position_column.size()); + for (size_t row = 0; row < row_position_column.size(); ++row) { + if (null_map[row]) { + null_map[row] = 0; + data[row] = _row_lineage_columns.first_row_id + row_position_column.get_element(row); + } + } + table_block->replace_by_position(column_idx, std::move(column)); + return Status::OK(); +} + +Status IcebergTableReader::_materialize_iceberg_rowid(Block* table_block, size_t column_idx) { + DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns()); + const auto& row_position_column = assert_cast( + *_data_reader.block_template.get_by_position(_row_position_block_position).column); + DORIS_CHECK(row_position_column.size() == table_block->rows()); + + const auto& type = table_block->get_by_position(column_idx).type; + auto column = type->create_column(); + auto* nullable_column = check_and_get_column(column.get()); + auto* struct_column = nullable_column != nullptr + ? check_and_get_column( + nullable_column->get_nested_column_ptr().get()) + : check_and_get_column(column.get()); + DORIS_CHECK(struct_column != nullptr); + DORIS_CHECK(struct_column->tuple_size() >= 4); + + const auto rows = row_position_column.size(); + const auto file_path = _data_file_path(); + const int32_t partition_spec_id = + _iceberg_params.has_value() && _iceberg_params->__isset.partition_spec_id + ? _iceberg_params->partition_spec_id + : 0; + const std::string partition_data_json = + _iceberg_params.has_value() && _iceberg_params->__isset.partition_data_json + ? _iceberg_params->partition_data_json + : ""; + + auto& file_path_column = struct_column->get_column(0); + auto& row_pos_column = struct_column->get_column(1); + auto& spec_id_column = struct_column->get_column(2); + auto& partition_data_column = struct_column->get_column(3); + file_path_column.reserve(rows); + row_pos_column.reserve(rows); + spec_id_column.reserve(rows); + partition_data_column.reserve(rows); + for (size_t row = 0; row < rows; ++row) { + file_path_column.insert_data(file_path.data(), file_path.size()); + const int64_t row_pos = row_position_column.get_element(row); + row_pos_column.insert_data(reinterpret_cast(&row_pos), sizeof(row_pos)); + spec_id_column.insert_data(reinterpret_cast(&partition_spec_id), + sizeof(partition_spec_id)); + partition_data_column.insert_data(partition_data_json.data(), partition_data_json.size()); + } + if (nullable_column != nullptr) { + nullable_column->get_null_map_data().resize_fill(rows, 0); + } + table_block->replace_by_position(column_idx, std::move(column)); + return Status::OK(); +} + +Status IcebergTableReader::_materialize_row_lineage_last_updated_sequence_number( + Block* table_block, size_t column_idx) { + if (_row_lineage_columns.last_updated_sequence_number < 0) { + return Status::OK(); + } + auto column = IColumn::mutate( + table_block->get_by_position(column_idx).column->convert_to_full_column_if_const()); + auto* nullable_column = assert_cast(column.get()); + auto& null_map = nullable_column->get_null_map_data(); + auto& data = assert_cast(*nullable_column->get_nested_column_ptr()).get_data(); + DORIS_CHECK(null_map.size() == table_block->rows()); + DORIS_CHECK(data.size() == table_block->rows()); + for (size_t row = 0; row < table_block->rows(); ++row) { + if (null_map[row]) { + null_map[row] = 0; + data[row] = _row_lineage_columns.last_updated_sequence_number; + } + } + table_block->replace_by_position(column_idx, std::move(column)); + return Status::OK(); +} + +bool IcebergTableReader::_need_row_lineage_row_id() const { + if (_data_reader.column_mapper != nullptr) { + for (const auto& mapping : _data_reader.column_mapper->mappings()) { + if (mapping.virtual_column_type == format::TableVirtualColumnType::ROW_ID) { + return true; + } + } + } + return std::ranges::any_of(_projected_columns, is_projected_row_lineage_row_id); +} + +bool IcebergTableReader::_need_iceberg_rowid() const { + if (_data_reader.column_mapper != nullptr) { + for (const auto& mapping : _data_reader.column_mapper->mappings()) { + if (mapping.virtual_column_type == format::TableVirtualColumnType::ICEBERG_ROWID) { + return true; + } + } + } + return std::ranges::any_of(_projected_columns, is_projected_iceberg_rowid); +} + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h new file mode 100644 index 00000000000000..79e0174a3d7eaa --- /dev/null +++ b/be/src/format_v2/table/iceberg_reader.h @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "format/table/iceberg_delete_file_reader_helper.h" +#include "format_v2/file_reader.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { +class Block; +namespace io { +struct FileDescription; +struct FileSystemProperties; +} // namespace io +} // namespace doris + +namespace doris::format { +struct DeleteFileDesc; +} +namespace doris::format::iceberg { + +// Iceberg table-level reader. +// It reuses TableReader for split orchestration, dynamic partition pruning and table-block +// finalization, while composing a FileReader for physical data-file reads instead of inheriting +// from a concrete file-format reader. +class IcebergTableReader : public format::TableReader { +public: + ~IcebergTableReader() override = default; + Status init(format::TableReadOptions&& options) override { + RETURN_IF_ERROR(format::TableReader::init(std::move(options))); + _mapper_options.mode = format::TableColumnMappingMode::BY_FIELD_ID; + return Status::OK(); + } + + Status prepare_split(const format::SplitReadOptions& options) override; + std::string debug_string() const override; + format::TableColumnMappingMode mapping_mode() const override { + return !_data_reader.file_schema.empty() && _has_field_id(_data_reader.file_schema) + ? format::TableColumnMappingMode::BY_FIELD_ID + : format::TableColumnMappingMode::BY_NAME; + } + +protected: + Status materialize_virtual_columns(Block* table_block) override; + + Status customize_file_scan_request(format::FileScanRequest* file_request) override; + + bool _supports_aggregate_pushdown(TPushAggOp::type agg_type) const override; + + Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, + bool* has_delete_file) override; + + Status _init_delete_predicates(const TTableFormatFileDesc& t_desc); + +private: + struct EqualityDeleteFilter; + + bool _has_field_id(const std::vector& schema) const { + for (const auto& field : schema) { + // TopN lazy materialization asks the file reader to synthesize GLOBAL_ROWID in the + // first-phase scan. That virtual column is not an Iceberg data field and therefore has + // no Iceberg field id. Do not let it downgrade schema-evolution reads to BY_NAME, + // otherwise old data files whose physical names predate a rename (for example, + // table column `new_new_id` stored as file column `id`) are materialized as defaults. + if (field.column_type != format::ColumnType::DATA_COLUMN) { + continue; + } + if (!field.has_identifier_field_id()) { + return false; + } + if (!_has_field_id(field.children)) { + return false; + } + } + return true; + } + static constexpr int MIN_SUPPORT_DELETE_FILES_VERSION = 2; + static constexpr int POSITION_DELETE = 1; + static constexpr int EQUALITY_DELETE = 2; + static constexpr int DELETION_VECTOR = 3; + + struct RowLineageColumns { + int64_t first_row_id = -1; + int64_t last_updated_sequence_number = -1; + }; + + static constexpr const char* ICEBERG_FILE_PATH = "file_path"; + static constexpr const char* ICEBERG_ROW_POS = "pos"; + static constexpr size_t ICEBERG_FILE_PATH_BLOCK_POSITION = 0; + static constexpr size_t ICEBERG_ROW_POS_BLOCK_POSITION = 1; + + class PositionDeleteRowsCollector final { + public: + using PositionDeleteFile = std::unordered_map; + + explicit PositionDeleteRowsCollector(PositionDeleteFile* rows_by_data_file); + + Status collect(const Block& block, size_t read_rows); + + private: + PositionDeleteFile* _rows_by_data_file = nullptr; + }; + + static std::shared_ptr _delete_file_system_properties( + const TFileScanRangeParams& scan_params); + + static std::unique_ptr _delete_file_description( + const TFileRangeDesc& range); + + std::string _data_file_path() const; + + // Append row position column to file scan request for position delete handling. + Status _append_row_position_output_column(format::FileScanRequest* request); + // Append equality delete predicates to file scan request based on the delete files in iceberg + // params. DeleteVector and position delete files use the common DeleteRows path in TableReader. + Status _append_equality_delete_predicates(format::FileScanRequest* request); + const format::ColumnDefinition* _find_equality_delete_data_field( + const EqualityDeleteFilter& filter, size_t key_idx) const; + std::optional _find_equality_delete_table_field( + const EqualityDeleteFilter& filter, size_t key_idx) const; + void _append_equality_delete_row_count_carrier(format::FileScanRequest* request); + std::string _delete_file_cache_key(const char* prefix, const std::string& path) const; + + Status _init_equality_delete_predicates( + const std::vector& delete_files); + + // Read equality/position delete files. + Status _create_delete_file_reader(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + std::unique_ptr* reader); + Status _read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx); + Status _load_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + EqualityDeleteFilter* result); + Status _resolve_equality_delete_fields(const TIcebergDeleteFileDesc& delete_file, + const std::vector& schema, + std::vector* delete_fields, + EqualityDeleteFilter* result) const; + Status _read_position_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + PositionDeleteRowsCollector* collector); + + // Read position delete files and collect deleted row positions to update DeletePredicate. + Status _init_position_delete_rows(const std::vector& delete_files); + + // Materialize row lineage virtual columns based on the position delete file. + Status _materialize_iceberg_rowid(Block* table_block, size_t column_idx); + Status _materialize_row_lineage_row_id(Block* table_block, size_t column_idx); + Status _materialize_row_lineage_last_updated_sequence_number(Block* table_block, + size_t column_idx); + + RowLineageColumns _row_lineage_columns; + size_t _row_position_block_position = 0; + std::optional _iceberg_params; + bool _delete_predicates_initialized = false; + format::DeleteRows _position_delete_rows_storage; + struct EqualityDeleteFilter { + std::vector field_ids; + // Delete-file names are retained for Iceberg tables imported from formats that did not + // persist field ids. In BY_NAME mode they are the fallback binding key. + std::vector field_names; + std::vector key_types; + Block delete_block; + }; + std::vector _equality_delete_filters; + // Scanner-shared cache supplied in SplitReadOptions. Parsed delete files outlive one data-file + // split and can be reused by every split referencing the same delete file. + ShardedKVCache* _split_cache = nullptr; + + bool _need_row_lineage_row_id() const; + bool _need_iceberg_rowid() const; +}; + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp new file mode 100644 index 00000000000000..bc2d0e64f8c067 --- /dev/null +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/paimon_reader.h" + +#include + +#include +#include + +#include "exprs/vexpr_context.h" +#include "format/table/paimon_reader.h" +#include "format_v2/column_mapper.h" +#include "format_v2/deletion_vector_reader.h" +#include "format_v2/jni/paimon_jni_reader.h" +#include "format_v2/table/schema_history_util.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::paimon { + +Status PaimonReader::prepare_split(const format::SplitReadOptions& options) { + _split_schema_id = -1; + const auto& paimon_params = options.current_range.table_format_params.paimon_params; + if (paimon_params.__isset.schema_id) { + _split_schema_id = paimon_params.schema_id; + } + return format::TableReader::prepare_split(options); +} + +format::TableColumnMappingMode PaimonReader::mapping_mode() const { + return format::can_map_by_history_schema(_scan_params, _split_schema_id) + ? format::TableColumnMappingMode::BY_FIELD_ID + : format::TableColumnMappingMode::BY_NAME; +} + +Status PaimonReader::annotate_file_schema(std::vector* file_schema) { + DORIS_CHECK(file_schema != nullptr); + if (mapping_mode() != format::TableColumnMappingMode::BY_FIELD_ID) { + return Status::OK(); + } + return format::annotate_file_schema_from_history(_scan_params, _split_schema_id, file_schema); +} + +Status PaimonReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, + DeleteFileDesc* desc, bool* has_delete_file) { + DORIS_CHECK(desc != nullptr); + DORIS_CHECK(has_delete_file != nullptr); + *has_delete_file = false; + const auto& table_desc = t_desc.paimon_params; + if (!table_desc.__isset.deletion_file) { + return Status::OK(); + } + const auto& deletion_file = table_desc.deletion_file; + + desc->key = build_paimon_deletion_vector_cache_key(deletion_file); + desc->path = deletion_file.path; + desc->start_offset = deletion_file.offset; + desc->size = deletion_file.length + 4; + desc->file_size = -1; + desc->format = DeleteFileDesc::Format::PAIMON; + *has_delete_file = true; + return Status::OK(); +} + +Status PaimonHybridReader::init(format::TableReadOptions&& options) { + return format::TableReader::init(std::move(options)); +} + +Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options) { + RETURN_IF_ERROR(_ensure_current_split_reader(options)); + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->prepare_split(options); +} + +Status PaimonHybridReader::get_block(Block* block, bool* eos) { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->get_block(block, eos); +} + +bool PaimonHybridReader::current_split_pruned() const { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->current_split_pruned(); +} + +Status PaimonHybridReader::abort_split() { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->abort_split(); +} + +Status PaimonHybridReader::close() { + Status close_status = Status::OK(); + if (_native_reader != nullptr) { + close_status = _native_reader->close(); + } + if (_jni_reader != nullptr) { + auto status = _jni_reader->close(); + if (!status.ok() && close_status.ok()) { + close_status = std::move(status); + } + } + _current_split_reader = nullptr; + return close_status; +} + +void PaimonHybridReader::set_batch_size(size_t batch_size) { + format::TableReader::set_batch_size(batch_size); + if (_native_reader != nullptr) { + _native_reader->set_batch_size(_batch_size); + } + if (_jni_reader != nullptr) { + _jni_reader->set_batch_size(_batch_size); + } +} + +Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { + if (_is_jni_split(options.current_range)) { + DCHECK(options.current_split_format == format::FileFormat::JNI); + if (_jni_reader == nullptr) { + _jni_reader = std::make_unique(); + RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); + } + _current_split_reader = _jni_reader.get(); + } else { + format::FileFormat file_format; + RETURN_IF_ERROR(_to_file_format(options.current_range, &file_format)); + DCHECK(options.current_split_format == file_format); + DCHECK(file_format == format::FileFormat::PARQUET || + file_format == format::FileFormat::ORC); + if (_native_reader == nullptr) { + _native_reader = format::paimon::PaimonReader::create_unique(); + RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); + } + _current_split_reader = _native_reader.get(); + } + return Status::OK(); +} + +Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, + format::FileFormat file_format) { + DORIS_CHECK(reader != nullptr); + VExprContextSPtrs conjuncts; + RETURN_IF_ERROR(_clone_conjuncts(&conjuncts)); + RETURN_IF_ERROR(reader->init({ + .projected_columns = _projected_columns, + .conjuncts = std::move(conjuncts), + .format = file_format, + .scan_params = _scan_params, + .io_ctx = _io_ctx, + .runtime_state = _runtime_state, + .scanner_profile = _scanner_profile, + .push_down_agg_type = _push_down_agg_type, + .condition_cache_digest = _condition_cache_digest, + })); + // Zero means no adaptive prediction has been produced yet. Preserve the child's normal + // runtime default until FileScannerV2 supplies the first positive prediction. + if (_batch_size > 0) { + reader->set_batch_size(_batch_size); + } + return Status::OK(); +} + +Status PaimonHybridReader::_clone_conjuncts(VExprContextSPtrs* conjuncts) const { + DORIS_CHECK(conjuncts != nullptr); + conjuncts->clear(); + conjuncts->reserve(_conjuncts.size()); + for (const auto& conjunct : _conjuncts) { + VExprSPtr root; + RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); + conjuncts->push_back(VExprContext::create_shared(std::move(root))); + } + return Status::OK(); +} + +bool PaimonHybridReader::_is_jni_split(const TFileRangeDesc& range) { + return range.__isset.table_format_params && range.table_format_params.__isset.paimon_params && + range.table_format_params.paimon_params.__isset.reader_type && + range.table_format_params.paimon_params.reader_type == TPaimonReaderType::PAIMON_JNI; +} + +Status PaimonHybridReader::_to_file_format(const TFileRangeDesc& range, + format::FileFormat* file_format) { + DORIS_CHECK(file_format != nullptr); + const auto format_type = + range.__isset.format_type ? range.format_type : TFileFormatType::FORMAT_PARQUET; + switch (format_type) { + case TFileFormatType::FORMAT_PARQUET: + *file_format = format::FileFormat::PARQUET; + return Status::OK(); + case TFileFormatType::FORMAT_ORC: + *file_format = format::FileFormat::ORC; + return Status::OK(); + default: + return Status::NotSupported("Unsupported native Paimon file format {}", + to_string(format_type)); + } +} + +} // namespace doris::format::paimon diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h new file mode 100644 index 00000000000000..efaf29d5c76059 --- /dev/null +++ b/be/src/format_v2/table/paimon_reader.h @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/table_reader.h" + +namespace doris::format { +struct DeleteFileDesc; +} +namespace doris::format::paimon { + +class PaimonReader final : public format::TableReader { +public: + ENABLE_FACTORY_CREATOR(PaimonReader); + ~PaimonReader() final = default; + Status prepare_split(const format::SplitReadOptions& options) override; + +#ifdef BE_TEST + void TEST_set_scan_params(TFileScanRangeParams* params) { _scan_params = params; } + format::TableColumnMappingMode TEST_mapping_mode() const { return mapping_mode(); } + Status TEST_annotate_file_schema(std::vector* file_schema) { + return annotate_file_schema(file_schema); + } + Status TEST_parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, + bool* has_delete_file) { + return _parse_deletion_vector_file(t_desc, desc, has_delete_file); + } +#endif + +protected: + format::TableColumnMappingMode mapping_mode() const override; + Status annotate_file_schema(std::vector* file_schema) override; + + Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, + bool* has_delete_file) override; + +private: + int64_t _split_schema_id = -1; +}; + +// Paimon scans can contain both native data-file splits and serialized JNI splits in the same +// SplitSource. FileScannerV2 owns one table reader for the scanner lifetime, so this reader keeps +// native and JNI child readers internally and dispatches each split to the matching child reader. +class PaimonHybridReader final : public format::TableReader { +public: + ~PaimonHybridReader() override = default; + + Status init(format::TableReadOptions&& options) override; + Status prepare_split(const format::SplitReadOptions& options) override; + Status get_block(Block* block, bool* eos) override; + bool current_split_pruned() const override; + Status abort_split() override; + Status close() override; + void set_batch_size(size_t batch_size) override; + +#ifdef BE_TEST + static bool TEST_is_jni_split(const TFileRangeDesc& range) { return _is_jni_split(range); } + static Status TEST_to_file_format(const TFileRangeDesc& range, + format::FileFormat* file_format) { + return _to_file_format(range, file_format); + } + void TEST_install_batch_size_children() { + _native_reader = std::make_unique(); + _jni_reader = std::make_unique(); + } + std::pair TEST_child_batch_sizes() const { + return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; + } +#endif + +private: + Status _ensure_current_split_reader(const format::SplitReadOptions& options); + Status _init_child_reader(format::TableReader* reader, format::FileFormat file_format); + Status _clone_conjuncts(VExprContextSPtrs* conjuncts) const; + static bool _is_jni_split(const TFileRangeDesc& range); + static Status _to_file_format(const TFileRangeDesc& range, format::FileFormat* file_format); + + std::unique_ptr _native_reader; // handle parquet/orc native splits + std::unique_ptr _jni_reader; // handle serialized JNI splits + format::TableReader* _current_split_reader = nullptr; +}; + +} // namespace doris::format::paimon diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp new file mode 100644 index 00000000000000..c67cece6e05b8c --- /dev/null +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -0,0 +1,365 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/remote_doris_reader.h" + +#include +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type_serde/data_type_serde.h" +#include "format/arrow/arrow_utils.h" +#include "format_v2/materialized_reader_util.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" +#include "util/timezone_utils.h" + +namespace doris::format::remote_doris { +namespace { + +Status validate_remote_doris_range(const TFileRangeDesc& range) { + if (!range.__isset.table_format_params || + range.table_format_params.table_format_type != "remote_doris") { + return Status::InvalidArgument("Remote Doris v2 reader requires remote_doris table format"); + } + if (!range.table_format_params.__isset.remote_doris_params) { + return Status::InvalidArgument("Remote Doris v2 reader requires remote_doris_params"); + } + const auto& params = range.table_format_params.remote_doris_params; + if (!params.__isset.location_uri || params.location_uri.empty()) { + return Status::InvalidArgument("Remote Doris v2 reader requires location_uri"); + } + if (!params.__isset.ticket || params.ticket.empty()) { + return Status::InvalidArgument("Remote Doris v2 reader requires ticket"); + } + return Status::OK(); +} + +class FlightRemoteDorisStream final : public RemoteDorisStream { +public: + explicit FlightRemoteDorisStream(const TFileRangeDesc& range) : _range(range) {} + + Status open() { + RETURN_IF_ERROR(validate_remote_doris_range(_range)); + const auto& params = _range.table_format_params.remote_doris_params; + arrow::flight::Location location; + RETURN_DORIS_STATUS_IF_ERROR( + arrow::flight::Location::Parse(params.location_uri).Value(&location)); + arrow::flight::Ticket ticket; + RETURN_DORIS_STATUS_IF_ERROR( + arrow::flight::Ticket::Deserialize(params.ticket).Value(&ticket)); + RETURN_DORIS_STATUS_IF_ERROR( + arrow::flight::FlightClient::Connect(location).Value(&_flight_client)); + RETURN_DORIS_STATUS_IF_ERROR(_flight_client->DoGet(ticket).Value(&_stream)); + return Status::OK(); + } + + Status next(std::shared_ptr* batch) override { + DORIS_CHECK(batch != nullptr); + arrow::flight::FlightStreamChunk chunk; + RETURN_DORIS_STATUS_IF_ERROR(_stream->Next().Value(&chunk)); + *batch = chunk.data; + return Status::OK(); + } + + Status close() override { + _stream.reset(); + if (_flight_client != nullptr) { + RETURN_DORIS_STATUS_IF_ERROR(_flight_client->Close()); + _flight_client.reset(); + } + return Status::OK(); + } + +private: + const TFileRangeDesc _range; + std::unique_ptr _flight_client; + std::unique_ptr _stream; +}; + +Status create_flight_stream(const TFileRangeDesc& range, std::unique_ptr* out) { + DORIS_CHECK(out != nullptr); + auto stream = std::make_unique(range); + RETURN_IF_ERROR(stream->open()); + *out = std::move(stream); + return Status::OK(); +} + +ColumnDefinition remote_doris_child_definition(const std::string& name, DataTypePtr type, + int32_t local_id); + +std::vector synthesize_remote_doris_children(const DataTypePtr& type) { + std::vector children; + DORIS_CHECK(type != nullptr); + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_ARRAY: { + const auto* array_type = assert_cast(nested_type.get()); + children.push_back( + remote_doris_child_definition("element", array_type->get_nested_type(), 0)); + break; + } + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + children.push_back(remote_doris_child_definition("key", map_type->get_key_type(), 0)); + children.push_back(remote_doris_child_definition("value", map_type->get_value_type(), 1)); + break; + } + case TYPE_STRUCT: { + const auto* struct_type = assert_cast(nested_type.get()); + children.reserve(struct_type->get_elements().size()); + for (size_t idx = 0; idx < struct_type->get_elements().size(); ++idx) { + children.push_back(remote_doris_child_definition(struct_type->get_element_name(idx), + struct_type->get_element(idx), + cast_set(idx))); + } + break; + } + default: + break; + } + return children; +} + +ColumnDefinition remote_doris_child_definition(const std::string& name, DataTypePtr type, + int32_t local_id) { + ColumnDefinition child; + child.identifier = Field::create_field(name); + child.local_id = local_id; + child.name = name; + child.type = std::move(type); + child.children = synthesize_remote_doris_children(child.type); + return child; +} + +} // namespace + +RemoteDorisFileReader::RemoteDorisFileReader( + std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, const TFileRangeDesc& range, + const std::vector& file_slot_descs, + RemoteDorisStreamFactory stream_factory) + : FileReader(system_properties, file_description, std::move(io_ctx), profile), + _range(range), + _file_slot_descs(file_slot_descs), + _stream_factory(std::move(stream_factory)) { + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _ctz); +} + +RemoteDorisFileReader::~RemoteDorisFileReader() { + static_cast(close()); +} + +Status RemoteDorisFileReader::init(RuntimeState* state) { + (void)state; + RETURN_IF_ERROR(validate_remote_doris_range(_range)); + RETURN_IF_ERROR(_build_col_name_to_file_id()); + _eof = false; + return Status::OK(); +} + +Status RemoteDorisFileReader::get_schema(std::vector* file_schema) const { + DORIS_CHECK(file_schema != nullptr); + file_schema->clear(); + file_schema->reserve(_file_slot_descs.size()); + for (size_t idx = 0; idx < _file_slot_descs.size(); ++idx) { + const auto* slot = _file_slot_descs[idx]; + DORIS_CHECK(slot != nullptr); + file_schema->push_back({ + .identifier = Field::create_field(cast_set(idx)), + .local_id = cast_set(idx), + .name = slot->col_name(), + .type = slot->type(), + // Remote Doris exposes table slots as file columns. Complex columns still need + // structural children so TableColumnMapper can validate and project them. + .children = synthesize_remote_doris_children(slot->type()), + }); + } + return Status::OK(); +} + +Status RemoteDorisFileReader::open(std::shared_ptr request) { + RETURN_IF_ERROR(FileReader::open(std::move(request))); + RETURN_IF_ERROR(_open_stream()); + _eof = false; + return Status::OK(); +} + +Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* eof) { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + if (_stream == nullptr) { + return Status::InternalError("Remote Doris v2 reader is not open"); + } + + *rows = 0; + *eof = false; + std::shared_ptr batch; + RETURN_IF_ERROR(_stream->next(&batch)); + if (batch == nullptr) { + *eof = true; + _eof = true; + return Status::OK(); + } + + RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + _record_scan_rows(cast_set(*rows)); + RETURN_IF_ERROR( + apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + return Status::OK(); +} + +Status RemoteDorisFileReader::close() { + if (_stream != nullptr) { + RETURN_IF_ERROR(_stream->close()); + _stream.reset(); + } + _request.reset(); + _eof = true; + return Status::OK(); +} + +Status RemoteDorisFileReader::_open_stream() { + DORIS_CHECK(_stream == nullptr); + if (_stream_factory) { + RETURN_IF_ERROR(_stream_factory(_range, &_stream)); + } else { + RETURN_IF_ERROR(create_flight_stream(_range, &_stream)); + } + DORIS_CHECK(_stream != nullptr); + return Status::OK(); +} + +Status RemoteDorisFileReader::_materialize_record_batch(const arrow::RecordBatch& batch, + Block* file_block, size_t* rows) const { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + if (_request == nullptr) { + return Status::InternalError("Remote Doris v2 reader is not open"); + } + + std::vector materialized_columns(file_block->columns(), false); + for (int arrow_idx = 0; arrow_idx < batch.num_columns(); ++arrow_idx) { + const std::string& column_name = batch.schema()->field(arrow_idx)->name(); + const auto file_id_it = _col_name_to_file_id.find(column_name); + if (file_id_it == _col_name_to_file_id.end()) { + return Status::InternalError("Remote Doris returned unknown column {}", column_name); + } + const auto block_position_it = _request->local_positions.find(file_id_it->second); + if (block_position_it == _request->local_positions.end()) { + continue; + } + RETURN_IF_ERROR(_materialize_arrow_column(batch, arrow_idx, file_id_it->second, + block_position_it->second, file_block)); + materialized_columns[block_position_it->second.value()] = true; + } + + for (const auto& [file_column_id, block_position] : _request->local_positions) { + if (block_position.value() >= materialized_columns.size()) { + return Status::InternalError( + "Remote Doris requested block position {} out of range, block columns {}", + block_position.value(), materialized_columns.size()); + } + if (!materialized_columns[block_position.value()]) { + return Status::InternalError("Remote Doris did not return requested file column id {}", + file_column_id.value()); + } + } + + *rows = cast_set(batch.num_rows()); + return Status::OK(); +} + +Status RemoteDorisFileReader::_materialize_arrow_column(const arrow::RecordBatch& batch, + int arrow_column_idx, + LocalColumnId file_column_id, + const LocalIndex& block_position, + Block* file_block) const { + DORIS_CHECK(file_block != nullptr); + if (block_position.value() >= file_block->columns()) { + return Status::InternalError( + "Remote Doris block position {} out of range, block columns {}", + block_position.value(), file_block->columns()); + } + const auto column_name = batch.schema()->field(arrow_column_idx)->name(); + auto columns_guard = file_block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + try { + RETURN_IF_ERROR(columns_guard.get_datatype_by_position(block_position.value()) + ->get_serde() + ->read_column_from_arrow(*columns[block_position.value()], + batch.column(arrow_column_idx).get(), 0, + batch.num_rows(), _ctz)); + } catch (const Exception& e) { + return Status::InternalError( + "Failed to convert Remote Doris Arrow column '{}' (file_column_id={}) to Doris " + "block: {}", + column_name, file_column_id.value(), e.what()); + } + return Status::OK(); +} + +Status RemoteDorisFileReader::_build_col_name_to_file_id() { + _col_name_to_file_id.clear(); + _col_name_to_file_id.reserve(_file_slot_descs.size()); + for (size_t idx = 0; idx < _file_slot_descs.size(); ++idx) { + const auto* slot = _file_slot_descs[idx]; + DORIS_CHECK(slot != nullptr); + _col_name_to_file_id.emplace(slot->col_name(), LocalColumnId(cast_set(idx))); + } + return Status::OK(); +} + +RemoteDorisReader::RemoteDorisReader(RemoteDorisStreamFactory stream_factory) + : _stream_factory(std::move(stream_factory)) {} + +Status RemoteDorisReader::init(TableReadOptions&& options) { + if (options.file_slot_descs == nullptr) { + return Status::InvalidArgument("Remote Doris v2 reader requires file slot descriptors"); + } + return TableReader::init(std::move(options)); +} + +Status RemoteDorisReader::prepare_split(const SplitReadOptions& options) { + RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + return TableReader::prepare_split(options); +} + +Status RemoteDorisReader::create_file_reader(std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + DORIS_CHECK(_file_slot_descs != nullptr); + *reader = std::make_unique( + _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, + _current_file_range_desc, *_file_slot_descs, _stream_factory); + return Status::OK(); +} + +} // namespace doris::format::remote_doris diff --git a/be/src/format_v2/table/remote_doris_reader.h b/be/src/format_v2/table/remote_doris_reader.h new file mode 100644 index 00000000000000..b4dd2a505a95ad --- /dev/null +++ b/be/src/format_v2/table/remote_doris_reader.h @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/file_reader.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { +class Block; +class RuntimeProfile; +class RuntimeState; +class SlotDescriptor; +} // namespace doris + +namespace doris::format::remote_doris { + +// Small abstraction around Arrow Flight to keep Remote Doris v2 reader unit-testable without +// starting a Flight server. Production code uses FlightRemoteDorisStream; tests can provide +// RecordBatch-backed streams that exercise the same FileReader block materialization path. +class RemoteDorisStream { +public: + virtual ~RemoteDorisStream() = default; + virtual Status next(std::shared_ptr* batch) = 0; + virtual Status close() = 0; +}; + +using RemoteDorisStreamFactory = + std::function*)>; + +class RemoteDorisFileReader final : public FileReader { +public: + RemoteDorisFileReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + const TFileRangeDesc& range, + const std::vector& file_slot_descs, + RemoteDorisStreamFactory stream_factory = {}); + ~RemoteDorisFileReader() override; + + Status init(RuntimeState* state) override; + Status get_schema(std::vector* file_schema) const override; + Status open(std::shared_ptr request) override; + Status get_block(Block* file_block, size_t* rows, bool* eof) override; + Status close() override; + +private: + Status _open_stream(); + Status _materialize_record_batch(const arrow::RecordBatch& batch, Block* file_block, + size_t* rows) const; + Status _materialize_arrow_column(const arrow::RecordBatch& batch, int arrow_column_idx, + LocalColumnId file_column_id, const LocalIndex& block_position, + Block* file_block) const; + Status _build_col_name_to_file_id(); + + const TFileRangeDesc _range; + const std::vector _file_slot_descs; + RemoteDorisStreamFactory _stream_factory; + cctz::time_zone _ctz; + std::unique_ptr _stream; + std::unordered_map _col_name_to_file_id; +}; + +class RemoteDorisReader final : public TableReader { +public: + explicit RemoteDorisReader(RemoteDorisStreamFactory stream_factory = {}); + + Status init(TableReadOptions&& options) override; + Status prepare_split(const SplitReadOptions& options) override; + +protected: + Status create_file_reader(std::unique_ptr* reader) override; + +private: + RemoteDorisStreamFactory _stream_factory; +}; + +} // namespace doris::format::remote_doris diff --git a/be/src/format_v2/table/schema_history_util.cpp b/be/src/format_v2/table/schema_history_util.cpp new file mode 100644 index 00000000000000..10109839e6987d --- /dev/null +++ b/be/src/format_v2/table/schema_history_util.cpp @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/schema_history_util.h" + +#include +#include +#include + +#include "core/field.h" +#include "util/string_util.h" + +namespace doris::format { +namespace { + +const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { + if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { + return nullptr; + } + return field_ptr.field_ptr.get(); +} + +const schema::external::TField* find_child_field_by_name( + const std::vector& fields, const std::string& name) { + for (const auto& field_ptr : fields) { + const auto* field = get_field_ptr(field_ptr); + if (field == nullptr) { + continue; + } + if (field->__isset.name && to_lower(field->name) == to_lower(name)) { + return field; + } + if (field->__isset.name_mapping && + std::ranges::any_of(field->name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(name); + })) { + return field; + } + } + return nullptr; +} + +void annotate_column_from_field(ColumnDefinition* column, const schema::external::TField& field); + +void annotate_struct_children(ColumnDefinition* column, + const schema::external::TStructField& struct_field) { + DORIS_CHECK(column != nullptr); + if (!struct_field.__isset.fields) { + return; + } + for (auto& child : column->children) { + const auto* child_field = find_child_field_by_name(struct_field.fields, child.name); + if (child_field != nullptr) { + annotate_column_from_field(&child, *child_field); + } + } +} + +void annotate_column_from_field(ColumnDefinition* column, const schema::external::TField& field) { + DORIS_CHECK(column != nullptr); + if (field.__isset.id) { + column->identifier = Field::create_field(field.id); + } + column->name_mapping = + field.__isset.name_mapping ? field.name_mapping : std::vector {}; + if (!field.__isset.nestedField) { + return; + } + if (field.nestedField.__isset.struct_field) { + annotate_struct_children(column, field.nestedField.struct_field); + } else if (field.nestedField.__isset.array_field) { + if (column->children.empty() || !field.nestedField.array_field.__isset.item_field) { + return; + } + const auto* item_field = get_field_ptr(field.nestedField.array_field.item_field); + if (item_field != nullptr) { + annotate_column_from_field(&column->children.front(), *item_field); + } + } else if (field.nestedField.__isset.map_field) { + if (!column->children.empty() && field.nestedField.map_field.__isset.key_field) { + const auto* key_field = get_field_ptr(field.nestedField.map_field.key_field); + if (key_field != nullptr) { + annotate_column_from_field(&column->children.front(), *key_field); + } + } + if (column->children.size() > 1 && field.nestedField.map_field.__isset.value_field) { + const auto* value_field = get_field_ptr(field.nestedField.map_field.value_field); + if (value_field != nullptr) { + annotate_column_from_field(&column->children[1], *value_field); + } + } + } +} + +} // namespace + +const schema::external::TSchema* find_history_schema(const TFileScanRangeParams* params, + int64_t schema_id) { + if (params == nullptr || !params->__isset.history_schema_info) { + return nullptr; + } + for (const auto& schema : params->history_schema_info) { + if (schema.__isset.schema_id && schema.schema_id == schema_id) { + return &schema; + } + } + return nullptr; +} + +bool can_map_by_history_schema(const TFileScanRangeParams* params, int64_t split_schema_id) { + if (split_schema_id < 0 || params == nullptr || !params->__isset.current_schema_id || + !params->__isset.history_schema_info) { + return false; + } + return find_history_schema(params, split_schema_id) != nullptr; +} + +Status annotate_file_schema_from_history(const TFileScanRangeParams* params, + int64_t split_schema_id, + std::vector* file_schema) { + DORIS_CHECK(file_schema != nullptr); + const auto* schema = find_history_schema(params, split_schema_id); + DORIS_CHECK(schema != nullptr); + if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { + return Status::OK(); + } + for (auto& column : *file_schema) { + const auto* field = find_child_field_by_name(schema->root_field.fields, column.name); + if (field != nullptr) { + annotate_column_from_field(&column, *field); + } + } + return Status::OK(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/table/schema_history_util.h b/be/src/format_v2/table/schema_history_util.h new file mode 100644 index 00000000000000..3c4a80b5d4c975 --- /dev/null +++ b/be/src/format_v2/table/schema_history_util.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "format_v2/column_data.h" +#include "gen_cpp/ExternalTableSchema_types.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format { + +const schema::external::TSchema* find_history_schema(const TFileScanRangeParams* params, + int64_t schema_id); + +bool can_map_by_history_schema(const TFileScanRangeParams* params, int64_t split_schema_id); + +// Annotate a file-local schema with the field ids and name mappings from the historical table +// schema that describes the current split. TableReader has already annotated projected table +// columns from current_schema_id; this function performs the symmetric annotation for the file +// schema so TableColumnMapper can match evolved Hudi/Paimon files by field id. +Status annotate_file_schema_from_history(const TFileScanRangeParams* params, + int64_t split_schema_id, + std::vector* file_schema); + +} // namespace doris::format diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp new file mode 100644 index 00000000000000..45c930fcf2aee8 --- /dev/null +++ b/be/src/format_v2/table_reader.cpp @@ -0,0 +1,1024 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table_reader.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/primitive_type.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/paimon_reader.h" +#include "format_v2/column_mapper.h" +#include "format_v2/deletion_vector_reader.h" +#include "format_v2/delimited_text/csv_reader.h" +#include "format_v2/delimited_text/text_reader.h" +#include "format_v2/json/json_reader.h" +#include "format_v2/native/native_reader.h" +#include "format_v2/orc/orc_reader.h" +#include "format_v2/parquet/parquet_reader.h" +#include "storage/segment/condition_cache.h" +#include "util/debug_points.h" +#include "util/string_util.h" + +namespace doris::format { +namespace { + +template +std::string join_table_reader_debug_strings(const std::vector& values, Formatter formatter) { + std::ostringstream out; + out << "["; + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) { + out << ", "; + } + out << formatter(values[i]); + } + out << "]"; + return out.str(); +} + +std::string file_format_to_string(FileFormat format) { + switch (format) { + case FileFormat::PARQUET: + return "PARQUET"; + case FileFormat::ORC: + return "ORC"; + case FileFormat::CSV: + return "CSV"; + case FileFormat::JSON: + return "JSON"; + case FileFormat::TEXT: + return "TEXT"; + case FileFormat::JNI: + return "JNI"; + case FileFormat::NATIVE: + return "NATIVE"; + case FileFormat::ARROW: + return "ARROW"; + } + return "UNKNOWN"; +} + +std::string push_down_agg_to_string(TPushAggOp::type op) { + switch (op) { + case TPushAggOp::NONE: + return "NONE"; + case TPushAggOp::COUNT: + return "COUNT"; + case TPushAggOp::MINMAX: + return "MINMAX"; + case TPushAggOp::MIX: + return "MIX"; + case TPushAggOp::COUNT_ON_INDEX: + return "COUNT_ON_INDEX"; + } + return "UNKNOWN"; +} + +std::string current_file_debug_string(const std::unique_ptr& task) { + if (task == nullptr || task->data_file == nullptr) { + return "null"; + } + const auto& file = *task->data_file; + std::ostringstream out; + out << "FileDescription{path=" << file.path << ", file_size=" << file.file_size + << ", range_start_offset=" << file.range_start_offset << ", range_size=" << file.range_size + << ", mtime=" << file.mtime << ", fs_name=" << file.fs_name + << ", file_cache_admission=" << file.file_cache_admission << "}"; + return out.str(); +} + +std::string partition_values_debug_string(const std::map& partition_values) { + std::ostringstream out; + out << "{"; + size_t idx = 0; + for (const auto& [key, _] : partition_values) { + if (idx++ > 0) { + out << ", "; + } + out << key; + } + out << "}"; + return out.str(); +} + +const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { + if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { + return nullptr; + } + return field_ptr.field_ptr.get(); +} + +bool external_field_matches_name(const schema::external::TField& field, const std::string& name) { + if (field.__isset.name && to_lower(field.name) == to_lower(name)) { + return true; + } + return field.__isset.name_mapping && + std::ranges::any_of(field.name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(name); + }); +} + +DataTypePtr find_struct_child_type_by_external_field(const DataTypeStruct& struct_type, + const schema::external::TField& field) { + for (size_t field_idx = 0; field_idx < struct_type.get_elements().size(); ++field_idx) { + if (external_field_matches_name(field, struct_type.get_element_name(field_idx))) { + return struct_type.get_element(field_idx); + } + } + return nullptr; +} + +DataTypePtr restore_current_primitive_type(const schema::external::TField& field, + DataTypePtr fallback_type) { + if (!field.__isset.type) { + return fallback_type; + } + const auto primitive_type = thrift_to_type(field.type.type); + if (is_complex_type(primitive_type)) { + return fallback_type; + } + // The delete file can expose an older physical type, but initial defaults belong to the + // current table field. Restore that type from FE before parsing the default and let the table + // reader apply the normal promotion cast to the delete-key type. + return DataTypeFactory::instance().create_data_type( + primitive_type, false, field.type.__isset.precision ? field.type.precision : 0, + field.type.__isset.scale ? field.type.scale : 0, + field.type.__isset.len ? field.type.len : -1); +} + +ColumnDefinition build_schema_column_from_external_field(const schema::external::TField& field, + DataTypePtr type) { + type = restore_current_primitive_type(field, std::move(type)); + ColumnDefinition column { + .identifier = field.__isset.id ? Field::create_field(field.id) : Field {}, + .name = field.__isset.name ? field.name : "", + .name_mapping = + field.__isset.name_mapping ? field.name_mapping : std::vector {}, + .type = std::move(type), + .children = {}, + .default_expr = nullptr, + .initial_default_value = field.__isset.initial_default_value + ? std::make_optional(field.initial_default_value) + : std::nullopt, + .initial_default_value_is_base64 = field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64, + .is_partition_key = false, + }; + if (column.type == nullptr || !field.__isset.nestedField) { + return column; + } + + const auto nested_type = remove_nullable(column.type); + switch (nested_type->get_primitive_type()) { + case TYPE_STRUCT: { + if (!field.nestedField.__isset.struct_field || + !field.nestedField.struct_field.__isset.fields) { + return column; + } + const auto& struct_type = assert_cast(*nested_type); + for (const auto& child_ptr : field.nestedField.struct_field.fields) { + const auto* child_field = get_field_ptr(child_ptr); + if (child_field == nullptr || !child_field->__isset.name) { + continue; + } + auto child_type = find_struct_child_type_by_external_field(struct_type, *child_field); + if (child_type == nullptr) { + continue; + } + column.children.push_back( + build_schema_column_from_external_field(*child_field, child_type)); + } + break; + } + case TYPE_ARRAY: { + if (!field.nestedField.__isset.array_field || + !field.nestedField.array_field.__isset.item_field) { + return column; + } + const auto* item_field = get_field_ptr(field.nestedField.array_field.item_field); + if (item_field == nullptr) { + return column; + } + const auto& array_type = assert_cast(*nested_type); + auto child = + build_schema_column_from_external_field(*item_field, array_type.get_nested_type()); + child.name = "element"; + if (child.has_identifier_name()) { + child.identifier = Field::create_field(child.name); + } + column.children.push_back(std::move(child)); + break; + } + case TYPE_MAP: { + if (!field.nestedField.__isset.map_field || + !field.nestedField.map_field.__isset.key_field || + !field.nestedField.map_field.__isset.value_field) { + return column; + } + const auto& map_type = assert_cast(*nested_type); + const auto* key_field = get_field_ptr(field.nestedField.map_field.key_field); + if (key_field != nullptr) { + auto child = + build_schema_column_from_external_field(*key_field, map_type.get_key_type()); + child.name = "key"; + if (child.has_identifier_name()) { + child.identifier = Field::create_field(child.name); + } + column.children.push_back(std::move(child)); + } + const auto* value_field = get_field_ptr(field.nestedField.map_field.value_field); + if (value_field != nullptr) { + auto child = build_schema_column_from_external_field(*value_field, + map_type.get_value_type()); + child.name = "value"; + if (child.has_identifier_name()) { + child.identifier = Field::create_field(child.name); + } + column.children.push_back(std::move(child)); + } + break; + } + default: + break; + } + return column; +} + +const schema::external::TField* find_external_root_field(const TFileScanRangeParams* params, + const ColumnDefinition& column) { + if (params == nullptr || !params->__isset.history_schema_info || + params->history_schema_info.empty()) { + return nullptr; + } + const auto* schema = ¶ms->history_schema_info.front(); + if (params->__isset.current_schema_id) { + for (const auto& candidate_schema : params->history_schema_info) { + if (candidate_schema.__isset.schema_id && + candidate_schema.schema_id == params->current_schema_id) { + schema = &candidate_schema; + break; + } + } + } + if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { + return nullptr; + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field == nullptr) { + continue; + } + if (external_field_matches_name(*field, column.name)) { + return field; + } + } + return nullptr; +} + +std::string expr_context_debug_string(const VExprContextSPtr& context) { + if (context == nullptr) { + return "null"; + } + const auto root = context->root(); + if (root == nullptr) { + return "VExprContext{root=null}"; + } + std::ostringstream out; + out << "VExprContext{root_name=" << root->expr_name() << ", root_debug=" << root->debug_string() + << "}"; + return out.str(); +} + +std::string table_filter_debug_string(const TableFilter& filter) { + std::ostringstream out; + out << "TableFilter{conjunct=" << expr_context_debug_string(filter.conjunct) + << ", global_indices=" + << join_table_reader_debug_strings( + filter.global_indices, + [](GlobalIndex global_index) { return std::to_string(global_index.value()); }) + << "}"; + return out.str(); +} + +bool contains_runtime_filter(const VExprContextSPtrs& conjuncts) { + return std::ranges::any_of(conjuncts, [](const auto& conjunct) { + return conjunct != nullptr && conjunct->root() != nullptr && + conjunct->root()->is_rf_wrapper(); + }); +} + +void collect_global_indices(const VExprSPtr& expr, std::set* global_indices) { + if (expr == nullptr) { + return; + } + if (expr->is_rf_wrapper()) { + // RuntimeFilterExpr wraps a real predicate expression but its own thrift node can still + // look like SLOT_REF. Collect indices from the wrapped predicate; do not cast the wrapper + // itself to VSlotRef. + collect_global_indices(expr->get_impl(), global_indices); + return; + } + if (expr->is_slot_ref()) { + const auto* slot_ref = assert_cast(expr.get()); + DORIS_CHECK(slot_ref->column_id() >= 0); + global_indices->insert(GlobalIndex(cast_set(slot_ref->column_id()))); + } + for (const auto& child : expr->children()) { + collect_global_indices(child, global_indices); + } +} + +Status build_table_filters_from_conjunct(const VExprContextSPtr& conjunct, RuntimeState* state, + std::vector* table_filters) { + if (conjunct == nullptr) { + return Status::OK(); + } + std::set global_indices; + collect_global_indices(conjunct->root(), &global_indices); + if (!global_indices.empty()) { + TableFilter table_filter; + VExprSPtr filter_root; + RETURN_IF_ERROR(clone_table_expr_tree(conjunct->root(), &filter_root)); + table_filter.conjunct = VExprContext::create_shared(std::move(filter_root)); + for (const auto global_index : global_indices) { + table_filter.global_indices.push_back(global_index); + } + table_filters->push_back(std::move(table_filter)); + } + return Status::OK(); +} + +Status parse_deletion_vector(const char* buf, size_t buffer_size, DeleteFileDesc::Format format, + DeletionVector* deletion_vector) { + DORIS_CHECK(buf != nullptr); + DORIS_CHECK(deletion_vector != nullptr); + DORIS_CHECK(format == DeleteFileDesc::Format::PAIMON || + format == DeleteFileDesc::Format::ICEBERG); + + if (format == DeleteFileDesc::Format::PAIMON) { + RETURN_IF_ERROR(decode_paimon_deletion_vector_buffer(buf, buffer_size, deletion_vector)); + return Status::OK(); + } + + return decode_iceberg_deletion_vector_buffer(buf, buffer_size, deletion_vector); +} + +} // namespace + +std::shared_ptr create_system_properties( + const TFileScanRangeParams* scan_params) { + auto system_properties = std::make_shared(); + if (scan_params == nullptr || !scan_params->__isset.file_type) { + system_properties->system_type = TFileType::FILE_LOCAL; + return system_properties; + } + system_properties->system_type = scan_params->file_type; + system_properties->properties = scan_params->properties; + system_properties->hdfs_params = scan_params->hdfs_params; + if (scan_params->__isset.broker_addresses) { + system_properties->broker_addresses.assign(scan_params->broker_addresses.begin(), + scan_params->broker_addresses.end()); + } + return system_properties; +} + +std::string TableReader::debug_string() const { + std::ostringstream out; + out << "TableReader{format=" << file_format_to_string(_format) + << ", push_down_agg_type=" << push_down_agg_to_string(_push_down_agg_type) + << ", aggregate_pushdown_tried=" << _aggregate_pushdown_tried + << ", has_current_reader=" << (_data_reader.reader != nullptr) + << ", has_current_task=" << (_current_task != nullptr) + << ", current_file=" << current_file_debug_string(_current_task) + << ", has_delete_rows=" << (_delete_rows != nullptr) + << ", delete_row_count=" << (_delete_rows == nullptr ? 0 : _delete_rows->size()) + << ", has_deletion_vector=" << (_deletion_vector != nullptr) + << ", deletion_vector_cardinality=" + << (_deletion_vector == nullptr ? 0 : _deletion_vector->cardinality()) + << ", has_system_properties=" << (_system_properties != nullptr) << ", system_type=" + << (_system_properties == nullptr ? static_cast(TFileType::FILE_LOCAL) + : static_cast(_system_properties->system_type)) + << ", has_scan_params=" << (_scan_params != nullptr) + << ", has_io_ctx=" << (_io_ctx != nullptr) + << ", has_runtime_state=" << (_runtime_state != nullptr) + << ", has_scanner_profile=" << (_scanner_profile != nullptr) + << ", mapper_options=" << _mapper_options.debug_string() << ", projected_columns=" + << join_table_reader_debug_strings( + _projected_columns, + [](const ColumnDefinition& column) { return column.debug_string(); }) + << ", partition_values=" << partition_values_debug_string(_partition_values) + << ", table_filters=" + << join_table_reader_debug_strings( + _table_filters, + [](const TableFilter& filter) { return table_filter_debug_string(filter); }) + << ", conjunct_count=" << _conjuncts.size() << ", conjuncts=" + << join_table_reader_debug_strings(_conjuncts, + [](const VExprContextSPtr& conjunct) { + return expr_context_debug_string(conjunct); + }) + << ", file_schema=" + << join_table_reader_debug_strings( + _data_reader.file_schema, + [](const ColumnDefinition& field) { return field.debug_string(); }) + << ", file_block_layout=" + << join_table_reader_debug_strings( + _data_reader.file_block_layout, + [](const FileBlockColumn& column) { + std::ostringstream column_out; + column_out << "FileBlockColumn{file_column_id=" << column.file_column_id + << ", name=" << column.name << ", type=" + << (column.type == nullptr ? "null" : column.type->get_name()) + << "}"; + return column_out.str(); + }) + << ", block_template_columns=" << _data_reader.block_template.columns() + << ", column_mapper=" + << (_data_reader.column_mapper == nullptr ? "null" + : _data_reader.column_mapper->debug_string()) + << "}"; + return out.str(); +} + +Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info, + ProjectedColumnBuildContext* context, + ColumnDefinition* column) const { + (void)slot_info; + DORIS_CHECK(context != nullptr); + DORIS_CHECK(column != nullptr); + context->schema_column.reset(); + const auto* schema_field = find_external_root_field(context->scan_params, *column); + if (schema_field == nullptr) { + return Status::OK(); + } + context->schema_column = build_schema_column_from_external_field(*schema_field, column->type); + column->identifier = context->schema_column->identifier; + column->name_mapping = context->schema_column->name_mapping; + return Status::OK(); +} + +std::optional TableReader::_find_current_table_column_by_field_id( + int32_t field_id, DataTypePtr type) const { + if (_scan_params == nullptr || !_scan_params->__isset.history_schema_info || + _scan_params->history_schema_info.empty()) { + return std::nullopt; + } + const auto* schema = &_scan_params->history_schema_info.front(); + if (_scan_params->__isset.current_schema_id) { + for (const auto& candidate_schema : _scan_params->history_schema_info) { + if (candidate_schema.__isset.schema_id && + candidate_schema.schema_id == _scan_params->current_schema_id) { + schema = &candidate_schema; + break; + } + } + } + if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { + return std::nullopt; + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && field->__isset.id && field->id == field_id) { + return build_schema_column_from_external_field(*field, std::move(type)); + } + } + return std::nullopt; +} + +Status TableReader::init(TableReadOptions&& options) { + _scan_params = options.scan_params; + _format = options.format; + _io_ctx = options.io_ctx; + _runtime_state = options.runtime_state; + _scanner_profile = options.scanner_profile; + _file_slot_descs = options.file_slot_descs; + _push_down_agg_type = options.push_down_agg_type; + _condition_cache_digest = options.condition_cache_digest; + _projected_columns = std::move(options.projected_columns); + _system_properties = create_system_properties(_scan_params); + _mapper_options.mode = TableColumnMappingMode::BY_NAME; + _conjuncts = std::move(options.conjuncts); + + if (_scanner_profile != nullptr) { + static const char* table_profile = "TableReader"; + ADD_TIMER_WITH_LEVEL(_scanner_profile, table_profile, 1); + _profile.num_delete_files = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteFiles", + TUnit::UNIT, table_profile, 1); + _profile.num_delete_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteRows", + TUnit::UNIT, table_profile, 1); + _profile.parse_delete_file_time = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "ParseDeleteFileTime", table_profile, 1); + _profile.decoded_dv_cache_hit_count = + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "DeletionVectorDecodedCacheHitCount", + TUnit::UNIT, table_profile, 1); + _profile.decoded_dv_cache_miss_count = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "DeletionVectorDecodedCacheMissCount", TUnit::UNIT, table_profile, + 1); + _profile.dv_file_cache_hit_count = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "DeletionVectorFileCacheHitCount", TUnit::UNIT, table_profile, 1); + _profile.dv_file_cache_miss_count = + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "DeletionVectorFileCacheMissCount", + TUnit::UNIT, table_profile, 1); + _profile.dv_file_cache_peer_read_count = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "DeletionVectorFileCachePeerReadCount", TUnit::UNIT, + table_profile, 1); + _profile.exec_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "GetBlockTime", table_profile, 1); + _profile.prepare_split_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PrepareSplitTime", table_profile, 1); + _profile.finalize_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "FinalizeBlockTime", table_profile, 1); + _profile.create_reader_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CreateReaderTime", table_profile, 1); + _profile.pushdown_agg_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PushDownAggTime", table_profile, 1); + _profile.open_reader_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "OpenReaderTime", table_profile, 1); + _profile.runtime_filter_partition_prune_timer = ADD_TIMER_WITH_LEVEL( + _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", 1); + _profile.runtime_filter_partition_pruned_range_counter = ADD_COUNTER_WITH_LEVEL( + _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, 1); + } + return Status::OK(); +} + +Status TableReader::_build_table_filters_from_conjuncts() { + _table_filters.clear(); + _constant_pruning_safe_filter_count = 0; + bool in_safe_prefix = true; + for (const auto& conjunct : _conjuncts) { + DORIS_CHECK(conjunct != nullptr); + DORIS_CHECK(conjunct->root() != nullptr); + // `_table_filters` omits expressions without slot references, but such an expression still + // occupies a position in the row-level conjunct order. Record how many localized filters + // precede the first unsafe original conjunct so constant pruning cannot jump over a + // slotless non-deterministic/error-preserving barrier. + if (in_safe_prefix && !_is_safe_to_pre_execute(conjunct)) { + in_safe_prefix = false; + } + if (!in_safe_prefix) { + continue; + } + RETURN_IF_ERROR( + build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); + _constant_pruning_safe_filter_count = _table_filters.size(); + } + return Status::OK(); +} + +Status TableReader::_open_local_filter_exprs(const FileScanRequest& file_request) { + RowDescriptor row_desc; + for (const auto& conjunct : file_request.conjuncts) { + RETURN_IF_ERROR(conjunct->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(conjunct->open(_runtime_state)); + } + for (const auto& delete_conjunct : file_request.delete_conjuncts) { + RETURN_IF_ERROR(delete_conjunct->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(delete_conjunct->open(_runtime_state)); + } + return Status::OK(); +} + +bool TableReader::_should_enable_condition_cache(const FileScanRequest& file_request) const { + if (_condition_cache_digest == 0 || _push_down_agg_type == TPushAggOp::type::COUNT || + _current_file_description == std::nullopt || _data_reader.reader == nullptr) { + return false; + } + // Condition cache is populated by file readers after evaluating file-local row-level + // conjuncts. Metadata pruning can skip row groups/pages, but it does not produce a per-row + // survivor bitmap that can safely populate the cache. + if (file_request.conjuncts.empty()) { + return false; + } + // Delete files/deletion vectors are table-format state. They may change independently of the + // data file path/mtime/size used by the external cache key, so caching their result can become + // stale. Keep delete filtering enabled, but do not read or write condition cache. + if (_delete_rows != nullptr || _deletion_vector != nullptr || + !file_request.delete_conjuncts.empty()) { + return false; + } + // Runtime filters can arrive late and their payload is not guaranteed to be represented by the + // scan-local digest. Without a read-only mode, a MISS could insert a bitmap for P AND RF under + // the digest for only P. This mirrors the old FileScanner guard. + return !contains_runtime_filter(file_request.conjuncts); +} + +Status TableReader::_init_reader_condition_cache(const FileScanRequest& file_request) { + _condition_cache = nullptr; + _condition_cache_ctx = nullptr; + if (!_should_enable_condition_cache(file_request)) { + return Status::OK(); + } + + auto* cache = segment_v2::ConditionCache::instance(); + if (cache == nullptr) { + return Status::OK(); + } + const auto& file = *_current_file_description; + _condition_cache_key = segment_v2::ConditionCache::ExternalCacheKey( + file.path, file.mtime, file.file_size, _condition_cache_digest, file.range_start_offset, + file.range_size, + segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION); + + segment_v2::ConditionCacheHandle handle; + const bool condition_cache_hit = cache->lookup(_condition_cache_key, &handle); + if (condition_cache_hit) { + _condition_cache = handle.get_filter_result(); + ++_condition_cache_hit_count; + } else { + const int64_t total_rows = _data_reader.reader->get_total_rows(); + if (total_rows <= 0) { + return Status::OK(); + } + // Add one guard granule for split ranges that start in the middle of a granule. A guard + // false bit beyond the real range never overlaps real rows, but avoids boundary overflow + // when a reader marks the last partial granule. + const size_t num_granules = (total_rows + ConditionCacheContext::GRANULE_SIZE - 1) / + ConditionCacheContext::GRANULE_SIZE; + _condition_cache = std::make_shared>(num_granules + 1, false); + } + + if (_condition_cache != nullptr) { + _condition_cache_ctx = std::make_shared(); + _condition_cache_ctx->is_hit = condition_cache_hit; + _condition_cache_ctx->filter_result = _condition_cache; + _condition_cache_ctx->num_granules = _condition_cache->size(); + if (condition_cache_hit) { + _condition_cache_ctx->base_granule = handle.get_base_granule(); + } + _data_reader.reader->set_condition_cache_context(_condition_cache_ctx); + } + return Status::OK(); +} + +void TableReader::_finalize_reader_condition_cache() { + if (_condition_cache_ctx == nullptr || _condition_cache_ctx->is_hit) { + _condition_cache = nullptr; + _condition_cache_ctx = nullptr; + return; + } + // LIMIT or scanner cancellation may close a reader before all selected row ranges are visited. + // Unvisited granules remain false in a MISS bitmap, so inserting a partial bitmap would make a + // later HIT skip valid rows. Only publish cache entries after the physical reader reaches EOF. + if (!_current_reader_reached_eof) { + _condition_cache = nullptr; + _condition_cache_ctx = nullptr; + return; + } + DORIS_CHECK(_condition_cache_ctx->num_granules <= _condition_cache->size()); + _condition_cache->resize(_condition_cache_ctx->num_granules); + segment_v2::ConditionCache::instance()->insert( + _condition_cache_key, std::move(_condition_cache), _condition_cache_ctx->base_granule); + _condition_cache = nullptr; + _condition_cache_ctx = nullptr; +} + +Status TableReader::create_next_reader(bool* eos) { + SCOPED_TIMER(_profile.create_reader_timer); + DCHECK(_data_reader.reader == nullptr); + if (_current_task == nullptr) { + *eos = true; + return Status::OK(); + } + + RETURN_IF_ERROR(create_file_reader(&_data_reader.reader)); + DORIS_CHECK(_data_reader.reader != nullptr); + if (_batch_size > 0) { + _data_reader.reader->set_batch_size(_batch_size); + } + Status st = _data_reader.reader->init(_runtime_state); + if (!st.ok()) { + if (_io_ctx != nullptr && _io_ctx->should_stop && st.is()) { + *eos = true; + _data_reader.reader.reset(); + return Status::OK(); + } + return st; + } + st = open_reader(); + if (!st.ok()) { + if (_io_ctx != nullptr && _io_ctx->should_stop && st.is()) { + *eos = true; + _data_reader.reader.reset(); + return Status::OK(); + } + return st; + } + if (_data_reader.reader == nullptr) { + *eos = _current_task == nullptr; + return Status::OK(); + } + *eos = false; + return Status::OK(); +} + +Status TableReader::create_file_reader(std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + const bool enable_mapping_timestamp_tz = _scan_params != nullptr && + _scan_params->__isset.enable_mapping_timestamp_tz && + _scan_params->enable_mapping_timestamp_tz; + if (_format == FileFormat::PARQUET) { + *reader = std::make_unique( + _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, + _global_rowid_context, enable_mapping_timestamp_tz); + return Status::OK(); + } + if (_format == FileFormat::ORC) { + *reader = std::make_unique( + _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, + _global_rowid_context, enable_mapping_timestamp_tz); + return Status::OK(); + } + if (_format == FileFormat::CSV) { + if (_file_slot_descs == nullptr) { + return Status::InvalidArgument("CSV reader requires file slot descriptors"); + } + // CSV has no embedded schema. TableReader owns table-level mapping, while CsvReader needs + // only the physical file slots plus scan text parameters to build a file-local schema. + // Non-file columns such as partitions/defaults/virtual row ids are intentionally excluded + // from `_file_slot_descs` and are materialized during finalize_chunk(). + *reader = std::make_unique( + _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, + _scan_params, *_file_slot_descs, _current_range_compress_type, + _current_range_load_id); + return Status::OK(); + } + if (_format == FileFormat::TEXT) { + if (_file_slot_descs == nullptr) { + return Status::InvalidArgument("Text reader requires file slot descriptors"); + } + // Text files have no embedded schema. As with CSV, TableReader handles table-level mapping + // and only passes physical file slots to the v2 TextReader. + *reader = std::make_unique( + _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, + _scan_params, *_file_slot_descs, _current_range_compress_type, + _current_range_load_id); + return Status::OK(); + } + if (_format == FileFormat::JSON) { + if (_file_slot_descs == nullptr) { + return Status::InvalidArgument("JSON reader requires file slot descriptors"); + } + *reader = std::make_unique( + _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, + _scan_params, _current_file_range_desc, *_file_slot_descs, + _current_range_compress_type, _current_range_load_id); + return Status::OK(); + } + if (_format == FileFormat::NATIVE) { + *reader = std::make_unique( + _system_properties, _current_task->data_file, _io_ctx, _scanner_profile); + return Status::OK(); + } + return Status::NotSupported("TableReader does not support file format {}", + file_format_to_string(_format)); +} + +std::unique_ptr create_file_description(const TFileRangeDesc& range) { + auto file_description = std::make_unique(); + file_description->path = range.path; + file_description->file_size = range.__isset.file_size ? range.file_size : -1; + file_description->mtime = range.__isset.modification_time ? range.modification_time : 0; + file_description->range_start_offset = range.__isset.start_offset ? range.start_offset : 0; + file_description->range_size = range.__isset.size ? range.size : -1; + if (range.__isset.fs_name) { + file_description->fs_name = range.fs_name; + } + if (range.__isset.file_cache_admission) { + file_description->file_cache_admission = range.file_cache_admission; + } + return file_description; +} + +Status TableReader::prepare_split(const SplitReadOptions& options) { + SCOPED_TIMER(_profile.prepare_split_timer); + _current_split_pruned = false; + _all_runtime_filters_applied_for_split = options.all_runtime_filters_applied; + if (options.conjuncts.has_value()) { + _conjuncts = *options.conjuncts; + } + // Update to current split format to handle ORC/PARQUET files in one table. + _format = options.current_split_format; + _partition_values = std::move(options.partition_values); + _current_task.reset(); + _current_file_description.reset(); + _current_file_range_desc = options.current_range; + _current_range_compress_type = options.current_range.__isset.compress_type + ? options.current_range.compress_type + : TFileCompressType::UNKNOWN; + _current_range_load_id = options.current_range.__isset.load_id + ? std::make_optional(options.current_range.load_id) + : std::nullopt; + _global_rowid_context = options.global_rowid_context; + _delete_rows = nullptr; + _deletion_vector = nullptr; + _aggregate_pushdown_tried = false; + _remaining_table_level_count = -1; + _current_reader_reached_eof = false; + RETURN_IF_ERROR(_evaluate_partition_prune_conjuncts(options.partition_prune_conjuncts, + &_current_split_pruned)); + if (_current_split_pruned) { + COUNTER_UPDATE(_profile.runtime_filter_partition_pruned_range_counter, 1); + return Status::OK(); + } + _current_task = std::make_unique(); + _current_task->data_file = create_file_description(options.current_range); + _current_file_description = *_current_task->data_file; + // A table-level row count is only equivalent to scanning the split when no row predicate is + // active and no predicate can arrive later. The metadata path can return several batches for + // one split; after its first synthetic batch there is no way to recover the real rows if a + // runtime filter arrives before the next scheduler turn. + if (_push_down_agg_type == TPushAggOp::type::COUNT && options.all_runtime_filters_applied && + _conjuncts.empty() && options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.table_level_row_count) { + DORIS_CHECK(options.current_range.table_format_params.table_level_row_count >= -1); + _remaining_table_level_count = + options.current_range.table_format_params.table_level_row_count; + } + if (_is_table_level_count_active()) { + return Status::OK(); + } + return _parse_delete_predicates(options); +} + +Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, + bool* can_filter_all) { + DORIS_CHECK(can_filter_all != nullptr); + SCOPED_TIMER(_profile.runtime_filter_partition_prune_timer); + *can_filter_all = false; + if (conjuncts.empty() || _partition_values.empty()) { + return Status::OK(); + } + + VExprContextSPtrs partition_conjuncts; + for (const auto& conjunct : conjuncts) { + DORIS_CHECK(conjunct != nullptr); + DORIS_CHECK(conjunct->root() != nullptr); + // Keep only the safe prefix of the original conjunct order. If an unsafe conjunct is + // skipped, a later predicate could prune the split before the unsafe one reaches its + // normal row-level evaluation point. + if (!_is_safe_to_pre_execute(conjunct)) { + break; + } + std::set global_indices; + collect_global_indices(conjunct->root(), &global_indices); + if (global_indices.empty()) { + continue; + } + const bool partition_only = std::ranges::all_of(global_indices, [&](GlobalIndex index) { + if (index.value() >= _projected_columns.size()) { + return false; + } + const auto& column = _projected_columns[index.value()]; + return column.is_partition_key && + find_partition_value(column, _partition_values) != nullptr; + }); + if (partition_only) { + partition_conjuncts.push_back(conjunct); + } + } + if (partition_conjuncts.empty()) { + return Status::OK(); + } + + Block block; + RETURN_IF_ERROR(_build_partition_prune_block(&block)); + RowDescriptor row_desc; + for (const auto& conjunct : partition_conjuncts) { + RETURN_IF_ERROR(conjunct->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(conjunct->open(_runtime_state)); + } + IColumn::Filter result_filter(block.rows(), 1); + return VExprContext::execute_conjuncts(partition_conjuncts, nullptr, &block, &result_filter, + can_filter_all); +} + +bool TableReader::_is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { + DORIS_CHECK(conjunct != nullptr); + DORIS_CHECK(conjunct->root() != nullptr); + const auto root = conjunct->root(); + const auto impl = root->get_impl(); + const auto predicate = impl != nullptr ? impl : root; + // Split pruning evaluates a predicate once before any file rows are read. Reordering + // non-deterministic or error-preserving expressions can change their row-level semantics, + // even when every referenced slot is a partition column or maps to a constant entry. + return predicate->is_safe_to_execute_on_selected_rows(); +} + +Status TableReader::_build_partition_prune_block(Block* block) const { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(!_projected_columns.empty()); + block->clear(); + for (const auto& column : _projected_columns) { + DORIS_CHECK(column.type != nullptr); + ColumnPtr value_column = column.type->create_column_const_with_default_value(1); + if (column.is_partition_key) { + const auto* partition_value = find_partition_value(column, _partition_values); + if (partition_value != nullptr) { + value_column = column.type->create_column_const(1, *partition_value); + } + } + block->insert({std::move(value_column), column.type, column.name}); + } + return Status::OK(); +} + +Status TableReader::_parse_delete_predicates(const SplitReadOptions& options) { + DeleteFileDesc desc {.fs_name = options.current_range.fs_name}; + bool has_delete_file = false; + RETURN_IF_ERROR(_parse_deletion_vector_file(options.current_range.table_format_params, &desc, + &has_delete_file)); + if (has_delete_file) { + DORIS_CHECK(options.cache != nullptr); + Status create_status = Status::OK(); + + bool decoded_cache_hit = false; + _deletion_vector = options.cache->get( + desc.key, + [&]() -> DeletionVector* { + auto deletion_vector = std::make_unique(); + + DeletionVectorReader dv_reader(_runtime_state, _scanner_profile, *_scan_params, + desc, _io_ctx.get()); + create_status = dv_reader.open(); + if (!create_status.ok()) [[unlikely]] { + return nullptr; + } + + size_t bytes_read = desc.size; + std::vector buffer(bytes_read); + DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.io_error", { + create_status = + Status::IOError("injected format v2 deletion vector read failure"); + return nullptr; + }); + DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.should_stop", { + create_status = Status::EndOfFile("stop read."); + return nullptr; + }); + create_status = + dv_reader.read_at(desc.start_offset, {buffer.data(), bytes_read}); + const auto& file_cache_stats = dv_reader.file_cache_statistics(); + COUNTER_UPDATE(_profile.dv_file_cache_hit_count, + file_cache_stats.num_local_io_total); + COUNTER_UPDATE(_profile.dv_file_cache_miss_count, + file_cache_stats.num_remote_io_total); + COUNTER_UPDATE(_profile.dv_file_cache_peer_read_count, + file_cache_stats.num_peer_io_total); + if (!create_status.ok()) [[unlikely]] { + return nullptr; + } + + const char* buf = buffer.data(); + SCOPED_TIMER(_profile.parse_delete_file_time); + create_status = parse_deletion_vector(buf, bytes_read, desc.format, + deletion_vector.get()); + if (!create_status.ok()) [[unlikely]] { + return nullptr; + } + COUNTER_UPDATE(_profile.num_delete_rows, deletion_vector->cardinality()); + return deletion_vector.release(); + }, + &decoded_cache_hit); + RETURN_IF_ERROR(create_status); + COUNTER_UPDATE(decoded_cache_hit ? _profile.decoded_dv_cache_hit_count + : _profile.decoded_dv_cache_miss_count, + 1); + } + + return Status::OK(); +} +} // namespace doris::format diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h new file mode 100644 index 00000000000000..75217a5cc83924 --- /dev/null +++ b/be/src/format_v2/table_reader.h @@ -0,0 +1,1676 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/exception.h" +#include "common/logging.h" +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_const.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/field.h" +#include "exec/common/stringop_substring.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vexpr_fwd.h" +#include "exprs/vslot_ref.h" +#include "format_v2/column_data.h" +#include "format_v2/column_mapper.h" +#include "format_v2/deletion_vector.h" +#include "format_v2/expr/cast.h" +#include "format_v2/expr/delete_predicate.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/schema_projection.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/io_common.h" +#include "runtime/descriptors.h" +#include "storage/segment/condition_cache.h" + +namespace doris { +class Block; +class RuntimeState; +} // namespace doris + +namespace doris::format { +struct DeleteFileDesc; + +using DeleteRows = std::vector; + +// Row-level predicates on table/global schema. They are rewritten to file-local expressions when +// possible, and remain the source of row-level filtering after localization. +struct TableFilter { + VExprContextSPtr conjunct; + std::vector global_indices; +}; + +struct ScanTask { + virtual ~ScanTask() = default; + + std::unique_ptr data_file; +}; + +struct ProjectedColumnBuildContext { + const TFileScanRangeParams* scan_params = nullptr; + const TFileRangeDesc* range = nullptr; + RuntimeState* runtime_state = nullptr; + std::optional schema_column = std::nullopt; + size_t next_file_column_idx = 0; +}; + +struct ReadProfile { + RuntimeProfile::Counter* num_delete_files = nullptr; + RuntimeProfile::Counter* num_delete_rows = nullptr; + RuntimeProfile::Counter* parse_delete_file_time = nullptr; + RuntimeProfile::Counter* decoded_dv_cache_hit_count = nullptr; + RuntimeProfile::Counter* decoded_dv_cache_miss_count = nullptr; + RuntimeProfile::Counter* dv_file_cache_hit_count = nullptr; + RuntimeProfile::Counter* dv_file_cache_miss_count = nullptr; + RuntimeProfile::Counter* dv_file_cache_peer_read_count = nullptr; + RuntimeProfile::Counter* exec_timer = nullptr; + RuntimeProfile::Counter* prepare_split_timer = nullptr; + RuntimeProfile::Counter* finalize_timer = nullptr; + RuntimeProfile::Counter* create_reader_timer = nullptr; + RuntimeProfile::Counter* pushdown_agg_timer = nullptr; + RuntimeProfile::Counter* open_reader_timer = nullptr; + RuntimeProfile::Counter* runtime_filter_partition_prune_timer = nullptr; + RuntimeProfile::Counter* runtime_filter_partition_pruned_range_counter = nullptr; +}; + +struct TableReadOptions { + // Columns need to be read from file and output by table reader. They are all in table/global + // schema semantics. + const std::vector projected_columns; + // All complex conjuncts from scan operator + const VExprContextSPtrs conjuncts; + // File format of the underlying data files, needed for reader initialization and reader-level + // filter pushdown. + const FileFormat format; + TFileScanRangeParams* scan_params; + std::shared_ptr io_ctx; + RuntimeState* runtime_state; + RuntimeProfile* scanner_profile; + // File formats without complete self-describing metadata, such as CSV, Text, and JSON, need + // the FE-planned physical file slots to build their file-local schema and deserialize values. + const std::vector* file_slot_descs = nullptr; + // Push-down aggregate type. + const TPushAggOp::type push_down_agg_type = TPushAggOp::type::NONE; + // Digest of stable pushed-down predicates. A zero digest disables condition cache. + uint64_t condition_cache_digest = 0; +}; + +struct SplitReadOptions { + // Split-level information for reader initialization, which may include file path, partition values, delete file info, etc. The content is table format specific and opaque to table reader base class; it's the responsibility of the concrete table reader implementation to parse necessary information for reader initialization and filter pushdown. + std::map partition_values; + // Latest scanner conjuncts rewritten to table/global column indices. Runtime filters may + // arrive after TableReader::init(), so scanner-driven splits replace the initial snapshot. + // nullopt preserves the initial snapshot for standalone TableReader callers. + std::optional conjuncts; + // Independent clones used for partition pruning because evaluation prepares and opens them + // against a synthetic partition block before the file reader opens its row-level conjuncts. + VExprContextSPtrs partition_prune_conjuncts; + // Table-level COUNT may emit one metadata-derived batch and resume on a later scheduler turn. + // It is safe only after every runtime filter assigned to the scanner has arrived; otherwise a + // filter could arrive after synthetic rows have already been returned and those rows cannot be + // retracted. Standalone TableReader callers have no scanner runtime-filter lifecycle. + bool all_runtime_filters_applied = true; + ShardedKVCache* cache = nullptr; + TFileRangeDesc current_range; + FileFormat current_split_format = FileFormat::PARQUET; + std::optional global_rowid_context; +}; + +// Base class for table-level readers. +// This layer owns common table-level orchestration, such as split iteration, dynamic partition +// pruning, delete handling and conversion from file-local blocks to table-schema blocks. Concrete +// table-format readers only need to provide format-specific hooks for opening readers and parsing +// split metadata. +class TableReader { +public: + virtual ~TableReader() = default; + + // Initialize common runtime options for the table reader. Subclasses may call this from their + // own init(options); table-format schema and split metadata are provided later per split. + virtual Status init(TableReadOptions&& options); + + // FileScannerV2 adjusts this before each get_block() using an adaptive bytes-per-row estimate. + // Store it here as well as forwarding to the current reader so newly opened split readers start + // with the latest predicted batch size. + virtual void set_batch_size(size_t batch_size) { + _batch_size = std::max(1, batch_size); + if (_data_reader.reader != nullptr) { + _data_reader.reader->set_batch_size(_batch_size); + } + } + +#ifdef BE_TEST + size_t TEST_batch_size() const { return _batch_size; } +#endif + + // Prepare for reading a new split/task. + // 1. Pass a new split/task to reader, which will be used in subsequent open_reader() to initialize the underlying file reader. + // 2. Parse delete predicates from split/task information, which will be used for later dynamic filtering and delete handling. + virtual Status prepare_split(const SplitReadOptions& options); + + virtual bool current_split_pruned() const { return _current_split_pruned; } + + // Discard the active split after the caller decides an error is ignorable, for example a + // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start + // with no concrete reader or split-local state left from the failed split. + virtual Status abort_split() { + if (_data_reader.reader != nullptr) { + RETURN_IF_ERROR(close_current_reader()); + } else { + _current_task.reset(); + _current_file_description.reset(); + } + _delete_rows = nullptr; + _remaining_table_level_count = -1; + _current_split_pruned = false; + return Status::OK(); + } + + // Public entry point for reading a table-schema block. The base class opens the current reader, + // advances across EOF, and closes exhausted readers. Subclasses provide protected hooks for + // table-format-specific behavior. + virtual Status get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.exec_timer); + DORIS_CHECK(block->columns() == _projected_columns.size()); + block->clear_column_data(_projected_columns.size()); + + while (true) { + if (*eos) { + return Status::OK(); + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + *eos = true; + return Status::OK(); + } + if (!_data_reader.reader) { + if (_is_table_level_count_active()) { + RETURN_IF_ERROR(_read_table_level_count(block, eos)); + return Status::OK(); + } + RETURN_IF_ERROR(create_next_reader(eos)); + if (!_data_reader.reader) { + DCHECK(*eos); + return Status::OK(); + } + } + + // Materialize a reduced row set for upper aggregate operators when aggregate + // pushdown can be applied. This is not the final aggregate result: COUNT emits + // `count` default rows for the upper COUNT(*), and MIN/MAX emits two rows containing + // file-level min/max values for the upper MIN/MAX. + if (!_aggregate_pushdown_tried) { + SCOPED_TIMER(_profile.pushdown_agg_timer); + bool pushed_down = false; + const auto status = _try_materialize_aggregate_pushdown_rows(block, &pushed_down); + if (!status.ok()) { + if (_io_ctx != nullptr && _io_ctx->should_stop && + status.is()) { + *eos = true; + return Status::OK(); + } + return status; + } + if (pushed_down) { + return Status::OK(); + } + } + + bool current_eof = false; + _data_reader.block_template.clear_column_data( + cast_set(_data_reader.file_block_layout.size())); + size_t current_rows = 0; + RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, + ¤t_rows, ¤t_eof)); + const bool stopped_during_read = _io_ctx != nullptr && _io_ctx->should_stop; + if (current_rows == 0) { + if (current_eof) { + _current_reader_reached_eof = !stopped_during_read; + RETURN_IF_ERROR(close_current_reader()); + } + continue; + } + DCHECK_EQ(_data_reader.block_template.columns(), _data_reader.file_block_layout.size()) + << _data_reader.block_template.dump_structure(); +#ifndef NDEBUG + RETURN_IF_ERROR(_check_file_block_columns("after file reader get_block", current_rows)); +#endif + DORIS_CHECK(block->columns() == _data_reader.column_mapper->mappings().size()); + RETURN_IF_ERROR(finalize_chunk(block, current_rows)); +#ifndef NDEBUG + RETURN_IF_ERROR( + _check_table_block_columns("after finalize_chunk", block, current_rows)); +#endif + if (current_eof) { + _current_reader_reached_eof = !stopped_during_read; + RETURN_IF_ERROR(close_current_reader()); + } + return Status::OK(); + } + } + + // Close the table reader and the currently active file reader. Subclasses that hold additional + // table-format resources should override this and call TableReader::close() first. + virtual Status close() { + if (_data_reader.reader) { + RETURN_IF_ERROR(close_current_reader()); + } + _current_task.reset(); + _current_file_description.reset(); + _remaining_table_level_count = -1; + return Status::OK(); + } + + int64_t condition_cache_hit_count() const { return _condition_cache_hit_count; } + + virtual std::string debug_string() const; + + virtual Status annotate_projected_column(const TFileScanSlotInfo& slot_info, + ProjectedColumnBuildContext* context, + ColumnDefinition* column) const; + + virtual Status validate_projected_columns(const ProjectedColumnBuildContext& context) const { + (void)context; + return Status::OK(); + } + +protected: + std::optional _find_current_table_column_by_field_id(int32_t field_id, + DataTypePtr type) const; + + // Parse deletion vector information from table format specific file description. + virtual Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, + DeleteFileDesc* desc, bool* has_delete_file) { + *has_delete_file = false; + return Status::OK(); + } + + // Advance to the next reader. This closes the current reader first and then opens the next + // concrete reader. Subclasses should not duplicate this loop. + Status create_next_reader(bool* eos); + virtual Status create_file_reader(std::unique_ptr* reader); + virtual TableColumnMappingMode mapping_mode() const { return TableColumnMappingMode::BY_NAME; } + virtual Status annotate_file_schema(std::vector* file_schema) { + DORIS_CHECK(file_schema != nullptr); + return Status::OK(); + } + + // Open the concrete reader for the current split/task and build the file-local scan request. + virtual Status open_reader() { + SCOPED_TIMER(_profile.open_reader_timer); + // 1. Get file schema and create column mapping. + std::vector file_schema; + RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + // For Paimon/Hudi, FE can provide field ids through `history_schema_info`. Annotate the + // file schema before column mapping when the table format maps columns by field id. + RETURN_IF_ERROR(annotate_file_schema(&file_schema)); + _data_reader.file_schema = file_schema; + _mapper_options.mode = mapping_mode(); + + _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + DORIS_CHECK(_data_reader.column_mapper != nullptr); + RETURN_IF_ERROR(_data_reader.column_mapper->create_mapping(_projected_columns, + _partition_values, file_schema)); + DORIS_CHECK(_data_reader.column_mapper->mappings().size() == _projected_columns.size()); + + // 2. Build table filters based on conjuncts and column predicates. + RETURN_IF_ERROR(_build_table_filters_from_conjuncts()); + + // 3. Create file scan request based on column mapping and table filters, then open file + // reader with the request. File scan request carries row-level expression filters and + // file-level pruning hints. Only expression filters decide returned rows. + auto file_request = std::make_shared(); + RETURN_IF_ERROR(_data_reader.column_mapper->create_scan_request( + _table_filters, _projected_columns, file_request.get(), _runtime_state)); + bool constant_filter_pruned_split = false; + RETURN_IF_ERROR(_evaluate_constant_filters(&constant_filter_pruned_split)); + if (constant_filter_pruned_split) { + RETURN_IF_ERROR(close_current_reader()); + return Status::OK(); + } + RETURN_IF_ERROR(customize_file_scan_request(file_request.get())); + RETURN_IF_ERROR(_open_local_filter_exprs(*file_request)); + _data_reader.file_block_layout.clear(); + _data_reader.block_template.clear(); + _data_reader.file_block_layout.resize(file_request->local_positions.size()); + + // 4. Build file block layout from file schema and column mapping. The layout describes + // the block returned by file reader before table-column materialization. + for (const auto& [file_column_id, block_position] : file_request->local_positions) { + DORIS_CHECK(block_position.value() < _data_reader.file_block_layout.size()); + const auto* field = _find_column_definition(_data_reader.file_schema, file_column_id); + DORIS_CHECK(field != nullptr); + + ColumnDefinition projected_field; + { + auto it = std::find_if( + file_request->non_predicate_columns.begin(), + file_request->non_predicate_columns.end(), + [&](const LocalColumnIndex& p) { return p.column_id() == file_column_id; }); + if (it != file_request->non_predicate_columns.end()) { + RETURN_IF_ERROR(project_column_definition(*field, *it, &projected_field)); + } + } + { + auto it = std::find_if( + file_request->predicate_columns.begin(), + file_request->predicate_columns.end(), + [&](const LocalColumnIndex& p) { return p.column_id() == file_column_id; }); + if (it != file_request->predicate_columns.end()) { + RETURN_IF_ERROR(project_column_definition(*field, *it, &projected_field)); + } + } + _data_reader.file_block_layout[block_position.value()] = { + .file_column_id = file_column_id, + .name = projected_field.name, + .type = projected_field.type, + }; + DORIS_CHECK(_data_reader.file_block_layout[block_position.value()].type != nullptr); + } + + // 5. Prepare block template from file block layout. The block template stores the block + // returned by file reader before table-column materialization. + _data_reader.block_template.reserve(_data_reader.file_block_layout.size()); + for (const auto& column : _data_reader.file_block_layout) { + _data_reader.block_template.insert( + {column.type->create_column(), column.type, column.name}); + } + if (VLOG_DEBUG_IS_ON) { + VLOG_DEBUG << "TableReader debug: " << debug_string(); + } + RETURN_IF_ERROR(_open_mapping_exprs()); + RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + RETURN_IF_ERROR(_init_reader_condition_cache(*file_request)); + return Status::OK(); + } + + Status _build_table_filters_from_conjuncts(); + Status _evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, + bool* can_filter_all); + static bool _is_safe_to_pre_execute(const VExprContextSPtr& conjunct); + Status _build_partition_prune_block(Block* block) const; + Status _open_local_filter_exprs(const FileScanRequest& file_request); + Status _init_reader_condition_cache(const FileScanRequest& file_request); + void _finalize_reader_condition_cache(); + bool _should_enable_condition_cache(const FileScanRequest& file_request) const; + + Status _evaluate_constant_filters(bool* can_filter_all) { + DORIS_CHECK(can_filter_all != nullptr); + DORIS_CHECK_LE(_constant_pruning_safe_filter_count, _table_filters.size()); + *can_filter_all = false; + // The bound was derived from the original `_conjuncts` order, which includes slotless + // expressions omitted from `_table_filters`. Iterating only this prefix therefore cannot + // skip an unsafe row-level predicate and pre-execute a later constant predicate. + for (size_t i = 0; i < _constant_pruning_safe_filter_count; ++i) { + const auto& table_filter = _table_filters[i]; + if (table_filter.conjunct == nullptr) { + continue; + } + DORIS_CHECK(_is_safe_to_pre_execute(table_filter.conjunct)); + // RuntimeFilterExpr does not implement execute_column_impl(); it is evaluated by the + // row-level filter path through execute_filter(). Constant split pruning uses + // VExprContext::execute() on a one-row synthetic block, so runtime filters must not be + // pre-executed here even when their referenced slot maps to a constant value. + if (table_filter.conjunct->root()->is_rf_wrapper() || + !_table_filter_has_only_constant_entries(table_filter)) { + continue; + } + Block eval_block; + RETURN_IF_ERROR(_build_constant_filter_block(table_filter, &eval_block)); + RowDescriptor row_desc; + RETURN_IF_ERROR(table_filter.conjunct->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(table_filter.conjunct->open(_runtime_state)); + int result_column_id = -1; + RETURN_IF_ERROR(table_filter.conjunct->execute(&eval_block, &result_column_id)); + DORIS_CHECK(result_column_id >= 0); + if (_filter_result_filters_all(eval_block.get_by_position(result_column_id).column)) { + *can_filter_all = true; + return Status::OK(); + } + } + return Status::OK(); + } + + bool _table_filter_has_only_constant_entries(const TableFilter& table_filter) const { + const auto& filter_entries = _data_reader.column_mapper->filter_entries(); + for (const auto global_index : table_filter.global_indices) { + const auto entry_it = filter_entries.find(global_index); + if (entry_it == filter_entries.end() || !entry_it->second.is_constant()) { + return false; + } + } + return !table_filter.global_indices.empty(); + } + + Status _build_constant_filter_block(const TableFilter& table_filter, Block* eval_block) { + DORIS_CHECK(eval_block != nullptr); + eval_block->clear(); + const auto& mappings = _data_reader.column_mapper->mappings(); + const auto& filter_entries = _data_reader.column_mapper->filter_entries(); + DORIS_CHECK(mappings.size() == _projected_columns.size()); + for (size_t column_idx = 0; column_idx < mappings.size(); ++column_idx) { + const auto global_index = GlobalIndex(column_idx); + const auto& mapping = mappings[column_idx]; + const auto entry_it = filter_entries.find(global_index); + const bool referenced_by_filter = + std::find(table_filter.global_indices.begin(), + table_filter.global_indices.end(), + global_index) != table_filter.global_indices.end(); + if (referenced_by_filter && entry_it != filter_entries.end() && + entry_it->second.is_constant()) { + ColumnPtr constant_column; + RETURN_IF_ERROR(_materialize_constant_filter_column( + entry_it->second.constant_index(), &constant_column)); + eval_block->insert({std::move(constant_column), mapping.table_type, + mapping.table_column_name}); + } else { + eval_block->insert({mapping.table_type->create_column_const_with_default_value(1), + mapping.table_type, mapping.table_column_name}); + } + } + return Status::OK(); + } + + Status _materialize_constant_filter_column(ConstantIndex constant_index, ColumnPtr* column) { + DORIS_CHECK(column != nullptr); + const auto& constant_entry = _data_reader.column_mapper->constant_map().get(constant_index); + DORIS_CHECK(constant_entry.expr != nullptr); + DORIS_CHECK(constant_entry.type != nullptr); + RowDescriptor row_desc; + RETURN_IF_ERROR(constant_entry.expr->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(constant_entry.expr->open(_runtime_state)); + Block eval_block; + eval_block.insert({constant_entry.type->create_column_const_with_default_value(1), + constant_entry.type, "__table_reader_constant_filter"}); + int result_column_id = -1; + RETURN_IF_ERROR(constant_entry.expr->execute(&eval_block, &result_column_id)); + DORIS_CHECK(result_column_id >= 0); + *column = eval_block.get_by_position(result_column_id).column; + DORIS_CHECK((*column)->size() == 1); + return Status::OK(); + } + + static bool _filter_result_filters_all(const ColumnPtr& filter_column) { + DORIS_CHECK(filter_column.get() != nullptr); + DORIS_CHECK(filter_column->size() == 1); + return !filter_column->get_bool(0); + } + + virtual Status customize_file_scan_request(FileScanRequest* file_request) { + return _append_delete_predicate(file_request); + } + + bool _is_table_level_count_active() const { return _remaining_table_level_count >= 0; } + + Status _materialize_count_rows(size_t rows, Block* block) const { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(block->columns() > 0 || rows == 0); + for (size_t column_idx = 0; column_idx < block->columns(); ++column_idx) { + auto column = block->get_by_position(column_idx).type->create_column(); + column->resize(rows); + block->replace_by_position(column_idx, std::move(column)); + } + return Status::OK(); + } + + Status _read_table_level_count(Block* block, bool* eos) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(eos != nullptr); + DORIS_CHECK(_push_down_agg_type == TPushAggOp::type::COUNT); + DORIS_CHECK(_remaining_table_level_count >= 0); + if (_remaining_table_level_count == 0) { + _remaining_table_level_count = -1; + _current_task.reset(); + *eos = true; + return Status::OK(); + } + + const int64_t batch_size = _runtime_state == nullptr + ? _remaining_table_level_count + : static_cast(_runtime_state->batch_size()); + const auto rows = std::min(_remaining_table_level_count, batch_size); + RETURN_IF_ERROR(_materialize_count_rows(cast_set(rows), block)); + _remaining_table_level_count -= rows; + *eos = false; + return Status::OK(); + } + + void _append_file_scan_column(FileScanRequest* request, LocalColumnId column_id, + std::vector* scan_columns) { + DORIS_CHECK(request != nullptr); + DORIS_CHECK(scan_columns != nullptr); + FileScanRequestBuilder builder(request); + Status status; + if (scan_columns == &request->predicate_columns) { + status = builder.add_predicate_column(column_id); + } else { + DORIS_CHECK(scan_columns == &request->non_predicate_columns); + status = builder.add_non_predicate_column(column_id); + } + DORIS_CHECK(status.ok()) << status.to_string(); + if (column_id == LocalColumnId(ROW_POSITION_COLUMN_ID) && + _find_column_definition(_data_reader.file_schema, column_id) == nullptr) { + _data_reader.file_schema.push_back(row_position_column_definition()); + } + } + + // Append DeletePredicate to file scan request if there are deletes. The predicate will be evaluated in file reader level and filter out deleted rows before returning data to table reader. + Status _append_delete_predicate(FileScanRequest* request) { + DORIS_CHECK(request != nullptr); + if ((_delete_rows == nullptr || _delete_rows->empty()) && + (_deletion_vector == nullptr || _deletion_vector->isEmpty())) { + return Status::OK(); + } + const auto row_position_column_id = LocalColumnId(ROW_POSITION_COLUMN_ID); + _append_file_scan_column(request, row_position_column_id, &request->predicate_columns); + + const auto block_position = request->local_positions.at(row_position_column_id); + auto append_predicate = [&](auto& deleted_rows) { + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->add_child(VSlotRef::create_shared( + cast_set(block_position.value()), cast_set(block_position.value()), + -1, std::make_shared(), ROW_POSITION_COLUMN_NAME)); + request->delete_conjuncts.push_back( + VExprContext::create_shared(std::move(delete_predicate))); + }; + if (_delete_rows != nullptr && !_delete_rows->empty()) { + append_predicate(*_delete_rows); + } + if (_deletion_vector != nullptr && !_deletion_vector->isEmpty()) { + append_predicate(*_deletion_vector); + } + return Status::OK(); + } + + // Close the current concrete reader. This hook is called by both create_next_reader() and + // close(), so it should remain idempotent. + virtual Status close_current_reader() { + _finalize_reader_condition_cache(); + RETURN_IF_ERROR(_data_reader.reader->close()); + _data_reader.reader.reset(); + if (_data_reader.column_mapper != nullptr) { + _data_reader.column_mapper->clear(); + _data_reader.column_mapper.reset(); + } + _table_filters.clear(); + _constant_pruning_safe_filter_count = 0; + _data_reader.file_schema.clear(); + _data_reader.file_block_layout.clear(); + _data_reader.block_template.clear(); + _current_task.reset(); + _current_file_description.reset(); + _current_reader_reached_eof = false; + return Status::OK(); + } + + void _record_scan_rows(size_t rows) { + if (_io_ctx != nullptr && _io_ctx->file_reader_stats != nullptr) { + _io_ctx->file_reader_stats->read_rows += rows; + } + } + + // Finalize file-local block to table/global schema block. + Status finalize_chunk(Block* block, const size_t rows) { + SCOPED_TIMER(_profile.finalize_timer); + size_t idx = 0; + for (const auto& mapping : _data_reader.column_mapper->mappings()) { + ColumnPtr column; + RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, rows, + &column)); + block->replace_by_position(idx, IColumn::mutate(std::move(column))); + idx++; + } + RETURN_IF_ERROR(materialize_virtual_columns(block)); + // Enforce CHAR/VARCHAR length declared by the table schema after all file-to-table + // materialization has finished. + RETURN_IF_ERROR(_truncate_char_or_varchar_columns(block)); + return Status::OK(); + } + + // Materialize virtual columns in the table block, such as Iceberg _row_id and + // _last_updated_sequence_number. This runs after normal column materialization so finalize + // expressions can reference those virtual columns. + virtual Status materialize_virtual_columns(Block* table_block) { return Status::OK(); } + +#ifndef NDEBUG + Status _check_file_block_columns(std::string_view stage, size_t rows) { + DORIS_CHECK(_data_reader.block_template.columns() == _data_reader.file_block_layout.size()); + for (size_t idx = 0; idx < _data_reader.block_template.columns(); ++idx) { + const auto& file_block_column = _data_reader.file_block_layout[idx]; + const auto& column_with_type = _data_reader.block_template.get_by_position(idx); + const auto* column = column_with_type.column.get(); + try { + if (column == nullptr) { + auto st = Status::InternalError( + "Invalid file block column {} at {}: file_column_id={}, name='{}', " + "type={}, column=null, expected_rows={}, reader={}", + idx, stage, file_block_column.file_column_id.value(), + file_block_column.name, + file_block_column.type == nullptr ? "null" + : file_block_column.type->get_name(), + rows, debug_string()); + LOG(WARNING) << st; + return st; + } + column->sanity_check(); + auto st = column_with_type.check_type_and_column_match(); + if (!st.ok()) { + auto contextual_status = Status::InternalError( + "Invalid file block column {} at {}: file_column_id={}, name='{}', " + "type={}, column={}, column_size={}, expected_rows={}, error={}, " + "reader={}", + idx, stage, file_block_column.file_column_id.value(), + file_block_column.name, + file_block_column.type == nullptr ? "null" + : file_block_column.type->get_name(), + column->get_name(), column->size(), rows, st.to_string(), + debug_string()); + LOG(WARNING) << contextual_status; + return contextual_status; + } + } catch (const Exception& e) { + auto st = Status::InternalError( + "Invalid file block column {} at {}: file_column_id={}, name='{}', " + "type={}, column={}, column_size={}, expected_rows={}, error={}, " + "reader={}", + idx, stage, file_block_column.file_column_id.value(), + file_block_column.name, + file_block_column.type == nullptr ? "null" + : file_block_column.type->get_name(), + column == nullptr ? "null" : column->get_name(), + column == nullptr ? 0 : column->size(), rows, e.to_string(), + debug_string()); + LOG(WARNING) << st; + return st; + } catch (const std::exception& e) { + auto st = Status::InternalError( + "Invalid file block column {} at {}: file_column_id={}, name='{}', " + "type={}, column={}, column_size={}, expected_rows={}, error={}, " + "reader={}", + idx, stage, file_block_column.file_column_id.value(), + file_block_column.name, + file_block_column.type == nullptr ? "null" + : file_block_column.type->get_name(), + column == nullptr ? "null" : column->get_name(), + column == nullptr ? 0 : column->size(), rows, e.what(), debug_string()); + LOG(WARNING) << st; + return st; + } + } + return Status::OK(); + } + + Status _check_table_block_columns(std::string_view stage, const Block* block, size_t rows) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(block->columns() == _data_reader.column_mapper->mappings().size()); + for (size_t idx = 0; idx < block->columns(); ++idx) { + const auto& mapping = _data_reader.column_mapper->mappings()[idx]; + const auto& column_with_type = block->get_by_position(idx); + const auto* column = column_with_type.column.get(); + try { + if (column == nullptr) { + auto st = Status::InternalError( + "Invalid table block column {} at {}: table_column='{}', " + "global_index={}, type={}, column=null, expected_rows={}, mapping={}", + idx, stage, mapping.table_column_name, mapping.global_index.value(), + mapping.table_type == nullptr ? "null" : mapping.table_type->get_name(), + rows, mapping.debug_string()); + LOG(WARNING) << st; + return st; + } + column->sanity_check(); + auto st = column_with_type.check_type_and_column_match(); + if (!st.ok()) { + auto contextual_status = Status::InternalError( + "Invalid table block column {} at {}: table_column='{}', " + "global_index={}, type={}, column={}, column_size={}, " + "expected_rows={}, error={}, mapping={}", + idx, stage, mapping.table_column_name, mapping.global_index.value(), + mapping.table_type == nullptr ? "null" : mapping.table_type->get_name(), + column->get_name(), column->size(), rows, st.to_string(), + mapping.debug_string()); + LOG(WARNING) << contextual_status; + return contextual_status; + } + } catch (const Exception& e) { + auto st = Status::InternalError( + "Invalid table block column {} at {}: table_column='{}', global_index={}, " + "type={}, column={}, column_size={}, expected_rows={}, error={}, " + "mapping={}", + idx, stage, mapping.table_column_name, mapping.global_index.value(), + mapping.table_type == nullptr ? "null" : mapping.table_type->get_name(), + column == nullptr ? "null" : column->get_name(), + column == nullptr ? 0 : column->size(), rows, e.to_string(), + mapping.debug_string()); + LOG(WARNING) << st; + return st; + } catch (const std::exception& e) { + auto st = Status::InternalError( + "Invalid table block column {} at {}: table_column='{}', global_index={}, " + "type={}, column={}, column_size={}, expected_rows={}, error={}, " + "mapping={}", + idx, stage, mapping.table_column_name, mapping.global_index.value(), + mapping.table_type == nullptr ? "null" : mapping.table_type->get_name(), + column == nullptr ? "null" : column->get_name(), + column == nullptr ? 0 : column->size(), rows, e.what(), + mapping.debug_string()); + LOG(WARNING) << st; + return st; + } + } + return Status::OK(); + } +#endif + + Status _truncate_char_or_varchar_columns(Block* block) { + DORIS_CHECK(block != nullptr); + if (_runtime_state == nullptr || + !_runtime_state->query_options().truncate_char_or_varchar_columns) { + return Status::OK(); + } + DORIS_CHECK(block->columns() == _data_reader.column_mapper->mappings().size()); + for (size_t idx = 0; idx < _data_reader.column_mapper->mappings().size(); ++idx) { + const auto& mapping = _data_reader.column_mapper->mappings()[idx]; + if (!_should_truncate_char_or_varchar_column(mapping)) { + continue; + } + const auto target_len = + assert_cast(remove_nullable(mapping.table_type).get()) + ->len(); + _truncate_char_or_varchar_column(block, idx, target_len); + } + return Status::OK(); + } + + // Return true when the table schema has a bounded CHAR/VARCHAR length that is stricter than + // the file-side type. Examples: + // - table VARCHAR(10), file VARCHAR(20): truncate to 10; + // - table VARCHAR(10), file STRING: truncate to 10 because STRING has no declared bound; + // - table STRING, any file type: no truncation because the target has no bound. + static bool _should_truncate_char_or_varchar_column(const ColumnMapping& mapping) { + if (mapping.table_type == nullptr) { + return false; + } + const auto table_type = remove_nullable(mapping.table_type); + const auto primitive_type = table_type->get_primitive_type(); + if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { + return false; + } + const auto target_len = assert_cast(table_type.get())->len(); + if (target_len <= 0) { + return false; + } + if (mapping.file_type == nullptr) { + return true; + } + const auto file_type = remove_nullable(mapping.file_type); + DORIS_CHECK(file_type != nullptr); + int file_len = -1; + if (file_type->get_primitive_type() == TYPE_VARCHAR || + file_type->get_primitive_type() == TYPE_CHAR || + file_type->get_primitive_type() == TYPE_STRING) { + file_len = assert_cast(file_type.get())->len(); + } + + return file_len < 0 || target_len < file_len; + } + + // Truncate a materialized CHAR/VARCHAR column in place by reusing the vectorized substring + // implementation: substring(column, 1, len). Nullable columns are unwrapped before substring + // execution and wrapped back with the original null map afterward, because substring operates + // on the nested string payload only. + static void _truncate_char_or_varchar_column(Block* block, size_t idx, int len) { + DORIS_CHECK(block != nullptr); + auto int_type = std::make_shared(); + const auto num_columns_without_result = cast_set(block->columns()); + auto& target = block->get_by_position(idx); + const bool is_nullable = target.type->is_nullable(); + ColumnPtr input_column = target.column; + ColumnPtr null_map_column; + if (is_nullable) { + const auto* nullable_column = assert_cast(target.column.get()); + input_column = nullable_column->get_nested_column_ptr(); + null_map_column = nullable_column->get_null_map_column_ptr(); + } + block->replace_by_position(idx, std::move(input_column)); + block->insert({int_type->create_column_const(block->rows(), to_field(1)), + int_type, "const 1"}); + block->insert({int_type->create_column_const(block->rows(), to_field(len)), + int_type, "const len"}); + block->insert({nullptr, std::make_shared(), "result"}); + + ColumnNumbers temp_arguments(3); + temp_arguments[0] = cast_set(idx); + temp_arguments[1] = num_columns_without_result; + temp_arguments[2] = num_columns_without_result + 1; + const uint32_t result_column_id = num_columns_without_result + 2; + SubstringUtil::substring_execute(*block, temp_arguments, result_column_id, block->rows()); + + ColumnPtr result_column = block->get_by_position(result_column_id).column; + if (is_nullable) { + result_column = ColumnNullable::create(std::move(result_column), null_map_column); + } + block->replace_by_position(idx, std::move(result_column)); + block->erase_tail(num_columns_without_result); + } + + Status _try_materialize_aggregate_pushdown_rows(Block* block, bool* pushed_down) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(pushed_down != nullptr); + *pushed_down = false; + block->clear_column_data(_projected_columns.size()); + _aggregate_pushdown_tried = true; + if (!_supports_aggregate_pushdown(_push_down_agg_type)) { + return Status::OK(); + } + + FileAggregateRequest file_request; + RETURN_IF_ERROR(_build_file_aggregate_request(_push_down_agg_type, &file_request)); + FileAggregateResult file_result; + const auto status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + if (status.is()) { + return Status::OK(); + } + RETURN_IF_ERROR(status); + RETURN_IF_ERROR( + _materialize_aggregate_pushdown_rows(_push_down_agg_type, file_result, block)); + *pushed_down = true; + RETURN_IF_ERROR(close_current_reader()); + return Status::OK(); + } + + virtual bool _supports_aggregate_pushdown(TPushAggOp::type agg_type) const { + // Only COUNT and MIN/MAX can be push down. + if (agg_type != TPushAggOp::type::COUNT && agg_type != TPushAggOp::type::MINMAX) { + return false; + } + // Aggregate pushdown returns reduced synthetic rows and may close the physical reader + // before the next scheduler turn. If a runtime filter is still pending, those rows could + // escape before the filter arrives and cannot later be reconstructed from real file rows. + // This is the same irreversibility constraint as table-level metadata COUNT, and applies + // to COUNT and MIN/MAX for Parquet/ORC as well as COUNT for text readers. + if (!_all_runtime_filters_applied_for_split) { + return false; + } + // Scanner owns the original conjunct list and evaluates it after TableReader finalizes + // rows. Even a slotless conjunct that cannot become a TableFilter must see every source + // row before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. + if (!_conjuncts.empty()) { + return false; + } + // Only support aggregate pushdown when there is no delete or filter, so + // the reduced rows consumed by the upper aggregate remain semantically equivalent to a + // normal scan. + if ((_delete_rows != nullptr && !_delete_rows->empty()) || + (_deletion_vector != nullptr && !_deletion_vector->isEmpty())) { + return false; + } + if (!_table_filters.empty()) { + return false; + } + if (agg_type == TPushAggOp::type::COUNT) { + return true; + } + // For MIN/MAX, only support direct file-to-table column mappings. The two emitted rows + // must be enough for the upper MIN/MAX aggregate without evaluating default expressions or + // virtual columns. + for (const auto& mapping : _data_reader.column_mapper->mappings()) { + if (!mapping.file_local_id.has_value() || + mapping.virtual_column_type != TableVirtualColumnType::INVALID || + mapping.default_expr != nullptr || mapping.file_type == nullptr || + mapping.table_type == nullptr) { + return false; + } + if (!_can_push_down_minmax_for_mapping(mapping)) { + return false; + } + } + return true; + } + + static ColumnPtr _detach_column(ColumnPtr column) { + DORIS_CHECK(column.get() != nullptr); + return IColumn::mutate(std::move(column)); + } + + static Status _align_column_nullability(ColumnPtr* column, const DataTypePtr& table_type) { + DORIS_CHECK(column != nullptr); + DORIS_CHECK(column->get() != nullptr); + DORIS_CHECK(table_type != nullptr); + // Must return non-const column + *column = (*column)->convert_to_full_column_if_const(); + if (table_type->is_nullable()) { + const auto& nested_type = + assert_cast(*table_type).get_nested_type(); + if (!(*column)->is_nullable()) { + RETURN_IF_ERROR(_align_column_nullability(column, nested_type)); + *column = make_nullable(*column); + return Status::OK(); + } + const auto& nullable_column = assert_cast(**column); + ColumnPtr nested_column = nullable_column.get_nested_column_ptr(); + RETURN_IF_ERROR(_align_column_nullability(&nested_column, nested_type)); + *column = ColumnNullable::create(nested_column, + nullable_column.get_null_map_column_ptr()); + return Status::OK(); + } + if ((*column)->is_nullable()) { + const auto& nullable_column = assert_cast(**column); + if (nullable_column.has_null()) { + return Status::InternalError( + "Default expression produced NULL for non-nullable table column"); + } + ColumnPtr nested_column = nullable_column.get_nested_column_ptr(); + RETURN_IF_ERROR(_align_column_nullability(&nested_column, table_type)); + *column = nested_column; + return Status::OK(); + } + if (const auto* array_type = typeid_cast(table_type.get())) { + const auto& array_column = assert_cast(**column); + ColumnPtr nested_column = array_column.get_data_ptr(); + RETURN_IF_ERROR( + _align_column_nullability(&nested_column, array_type->get_nested_type())); + *column = ColumnArray::create(nested_column, array_column.get_offsets_ptr()); + return Status::OK(); + } + if (const auto* map_type = typeid_cast(table_type.get())) { + const auto& map_column = assert_cast(**column); + ColumnPtr key_column = map_column.get_keys_ptr(); + ColumnPtr value_column = map_column.get_values_ptr(); + RETURN_IF_ERROR(_align_column_nullability(&key_column, map_type->get_key_type())); + RETURN_IF_ERROR(_align_column_nullability(&value_column, map_type->get_value_type())); + *column = ColumnMap::create(key_column, value_column, map_column.get_offsets_ptr()); + return Status::OK(); + } + if (const auto* struct_type = typeid_cast(table_type.get())) { + const auto& struct_column = assert_cast(**column); + Columns columns = struct_column.get_columns_copy(); + DORIS_CHECK(columns.size() == struct_type->get_elements().size()); + for (size_t i = 0; i < columns.size(); ++i) { + RETURN_IF_ERROR( + _align_column_nullability(&columns[i], struct_type->get_element(i))); + } + *column = ColumnStruct::create(columns); + return Status::OK(); + } + return Status::OK(); + } + + static Status _execute_default_expr_without_root_type_check( + const VExprContextSPtr& default_expr, const Block* block, + ColumnWithTypeAndName* result_data) { + DORIS_CHECK(default_expr != nullptr); + DORIS_CHECK(block != nullptr); + DORIS_CHECK(result_data != nullptr); + ColumnPtr result_column; + Status st; + RETURN_IF_CATCH_EXCEPTION({ + st = default_expr->root()->execute_column_impl(default_expr.get(), block, nullptr, + block->rows(), result_column); + }); + RETURN_IF_ERROR(st); + DORIS_CHECK(result_column.get() != nullptr); + if (result_column->size() != block->rows()) { + return Status::InternalError( + "Default expr {} return column size {} not equal to expected size {}", + default_expr->expr_name(), result_column->size(), block->rows()); + } + result_data->column = result_column; + result_data->type = default_expr->execute_type(block); + result_data->name = default_expr->expr_name(); + return Status::OK(); + } + + Status _cast_column_to_type(ColumnPtr* column, const DataTypePtr& file_type, + const DataTypePtr& table_type, + const std::string& column_name) const { + DORIS_CHECK(column != nullptr); + DORIS_CHECK(column->get() != nullptr); + DORIS_CHECK(file_type != nullptr); + DORIS_CHECK(table_type != nullptr); + if (file_type->equals(*table_type)) { + return Status::OK(); + } + + DataTypePtr input_type = file_type; + // Cast wrappers unwrap nullable inputs according to the declared input type, so keep the + // root nullability of the declared type aligned with the actual column shape. + if ((*column)->is_nullable() && !input_type->is_nullable()) { + input_type = make_nullable(input_type); + } else if (!(*column)->is_nullable() && input_type->is_nullable()) { + input_type = remove_nullable(input_type); + } + Block cast_block; + cast_block.insert({*column, input_type, column_name}); + auto slot_ref = VSlotRef::create_shared(0, 0, -1, input_type, column_name); + auto cast_expr = Cast::create_shared(table_type); + cast_expr->add_child(std::move(slot_ref)); + auto cast_ctx = VExprContext::create_shared(std::move(cast_expr)); + RowDescriptor row_desc; + RETURN_IF_ERROR(cast_ctx->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(cast_ctx->open(_runtime_state)); + ColumnPtr cast_column; + RETURN_IF_ERROR(cast_ctx->execute(&cast_block, cast_column)); + *column = std::move(cast_column); + return Status::OK(); + } + + Status _materialize_present_child_mapping_column(const ColumnMapping& mapping, + const ColumnPtr& file_column, + const size_t rows, ColumnPtr* column) { + DORIS_CHECK(column != nullptr); + DORIS_CHECK(mapping.file_type != nullptr); + DORIS_CHECK(mapping.table_type != nullptr); + *column = file_column; + if (!mapping.is_trivial) { + if (!mapping.child_mappings.empty()) { + RETURN_IF_ERROR( + _materialize_complex_mapping_column(mapping, *column, rows, column)); + } else { + RETURN_IF_ERROR(_cast_column_to_type(column, mapping.file_type, mapping.table_type, + mapping.file_column_name)); + } + } + RETURN_IF_ERROR(_align_column_nullability(column, mapping.table_type)); + return Status::OK(); + } + + Status _materialize_mapping_column(const ColumnMapping& mapping, Block* current_block, + const size_t rows, ColumnPtr* column) { + if (!mapping.is_trivial && mapping.file_local_id.has_value() && + !mapping.child_mappings.empty()) { + DCHECK(mapping.projection != nullptr); + int res_id; + auto st = mapping.projection->execute(current_block, &res_id); + if (!st.ok()) { + return Status::InternalError( + "Failed to execute complex mapping projection for table column '{}' " + "(global_index={}, file_local_id={}, rows={}): {}, mapping={}", + mapping.table_column_name, mapping.global_index.value(), + *mapping.file_local_id, rows, st.to_string(), mapping.debug_string()); + } + ColumnPtr result_column = current_block->get_by_position(res_id).column; + RETURN_IF_ERROR( + _materialize_complex_mapping_column(mapping, result_column, rows, column)); + return Status::OK(); + } + if (mapping.projection != nullptr) { + int res_id; + auto st = mapping.projection->execute(current_block, &res_id); + if (!st.ok()) { + std::string file_local_id = "null"; + if (mapping.file_local_id.has_value()) { + file_local_id = std::to_string(*mapping.file_local_id); + } + return Status::InternalError( + "Failed to execute mapping projection for table column '{}' " + "(global_index={}, file_local_id={}, rows={}): {}, mapping={}", + mapping.table_column_name, mapping.global_index.value(), file_local_id, + rows, st.to_string(), mapping.debug_string()); + } + ColumnPtr result_column = current_block->get_by_position(res_id).column; + *column = _detach_column(std::move(result_column)); + return Status::OK(); + } + if (mapping.default_expr != nullptr) { + if (current_block->rows() == rows) { + ColumnWithTypeAndName result; + RETURN_IF_ERROR(_execute_default_expr_without_root_type_check( + mapping.default_expr, current_block, &result)); + ColumnPtr result_column = result.column; + RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); + *column = _detach_column(std::move(result_column)); + } else { + DORIS_CHECK(mapping.constant_index.has_value()); + Block eval_block; + eval_block.insert({mapping.table_type->create_column_const_with_default_value(rows), + mapping.table_type, "__table_reader_const_rows"}); + ColumnWithTypeAndName result; + RETURN_IF_ERROR(_execute_default_expr_without_root_type_check( + mapping.default_expr, &eval_block, &result)); + ColumnPtr result_column = result.column; + RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); + *column = _detach_column(std::move(result_column)); + } + return Status::OK(); + } + ColumnPtr result_column = mapping.table_type->create_column_const_with_default_value(rows); + *column = _detach_column(std::move(result_column)); + return Status::OK(); + } + + Status _materialize_complex_mapping_column(const ColumnMapping& mapping, + const ColumnPtr& file_column, const size_t rows, + ColumnPtr* column) { + DORIS_CHECK(mapping.table_type != nullptr); + DORIS_CHECK(file_column.get() != nullptr); + const auto table_type = remove_nullable(mapping.table_type); + switch (table_type->get_primitive_type()) { + case TYPE_STRUCT: + RETURN_IF_ERROR(_materialize_struct_mapping_column(mapping, file_column, rows, column)); + break; + case TYPE_ARRAY: + RETURN_IF_ERROR(_materialize_array_mapping_column(mapping, file_column, rows, column)); + break; + case TYPE_MAP: + RETURN_IF_ERROR(_materialize_map_mapping_column(mapping, file_column, rows, column)); + break; + default: + *column = _detach_column(file_column); + break; + } + return Status::OK(); + } + + static std::vector _present_child_mappings_in_file_order( + const std::vector& child_mappings) { + std::vector result; + result.reserve(child_mappings.size()); + for (const auto& child_mapping : child_mappings) { + if (child_mapping.file_local_id.has_value()) { + result.push_back(&child_mapping); + } + } + std::ranges::sort(result, [](const ColumnMapping* lhs, const ColumnMapping* rhs) { + DORIS_CHECK(lhs->file_local_id.has_value()); + DORIS_CHECK(rhs->file_local_id.has_value()); + return *lhs->file_local_id < *rhs->file_local_id; + }); + return result; + } + + static size_t _file_child_ordinal_for_mapping( + const ColumnMapping& mapping, const ColumnMapping& child_mapping, + const std::vector& file_ordered_children) { + DORIS_CHECK(child_mapping.file_local_id.has_value()); + if (!mapping.projected_file_children.empty()) { + const auto child_it = std::ranges::find_if( + mapping.projected_file_children, [&](const ColumnDefinition& file_child) { + return file_child.file_local_id() == *child_mapping.file_local_id; + }); + DORIS_CHECK(child_it != mapping.projected_file_children.end()); + return static_cast( + std::distance(mapping.projected_file_children.begin(), child_it)); + } + const auto child_it = std::ranges::find(file_ordered_children, &child_mapping); + DORIS_CHECK(child_it != file_ordered_children.end()); + return static_cast(std::distance(file_ordered_children.begin(), child_it)); + } + + static std::vector _child_mappings_in_table_type_order( + const ColumnMapping& mapping, const DataTypeStruct& table_type) { + std::vector result; + result.reserve(mapping.child_mappings.size()); + for (size_t child_idx = 0; child_idx < table_type.get_elements().size(); ++child_idx) { + const auto& child_name = table_type.get_element_name(child_idx); + const auto child_it = std::ranges::find_if( + mapping.child_mappings, [&](const ColumnMapping& child_mapping) { + return child_mapping.table_column_name == child_name; + }); + DORIS_CHECK(child_it != mapping.child_mappings.end()) + << mapping.debug_string() << ", table_child_name=" << child_name; + result.push_back(&*child_it); + } + return result; + } + + static const IColumn* _nested_column_if_nullable(const ColumnPtr& column, + const NullMap** null_map) { + DORIS_CHECK(column.get() != nullptr); + if (const auto* nullable_column = check_and_get_column(*column)) { + if (null_map != nullptr) { + *null_map = &nullable_column->get_null_map_data(); + } + return &nullable_column->get_nested_column(); + } + return column.get(); + } + + Status _materialize_struct_mapping_column(const ColumnMapping& mapping, + const ColumnPtr& file_column, const size_t rows, + ColumnPtr* column) { + DORIS_CHECK(mapping.table_type != nullptr); + const auto* table_type = + assert_cast(remove_nullable(mapping.table_type).get()); + const auto full_file_column = file_column->convert_to_full_column_if_const(); + const NullMap* parent_null_map = nullptr; + const auto* nested_file_column = + _nested_column_if_nullable(full_file_column, &parent_null_map); + const auto* file_struct = assert_cast(nested_file_column); + DORIS_CHECK(table_type->get_elements().size() == mapping.child_mappings.size()); + + Columns child_columns; + child_columns.reserve(mapping.child_mappings.size()); + const auto file_ordered_children = + _present_child_mappings_in_file_order(mapping.child_mappings); + const auto table_ordered_children = + _child_mappings_in_table_type_order(mapping, *table_type); + for (const auto* child_mapping : table_ordered_children) { + DORIS_CHECK(child_mapping != nullptr); + if (!child_mapping->file_local_id.has_value()) { + child_columns.push_back( + child_mapping->table_type->create_column_const_with_default_value(rows) + ->convert_to_full_column_if_const()); + continue; + } + const auto file_child_idx = + _file_child_ordinal_for_mapping(mapping, *child_mapping, file_ordered_children); + DORIS_CHECK(file_child_idx < file_struct->get_columns().size()); + ColumnPtr child_column = file_struct->get_column_ptr(file_child_idx); + RETURN_IF_ERROR(_materialize_present_child_mapping_column(*child_mapping, child_column, + rows, &child_column)); + child_columns.push_back(std::move(child_column)); + } + MutableColumns mutable_child_columns; + mutable_child_columns.reserve(child_columns.size()); + for (auto& child_column : child_columns) { + mutable_child_columns.push_back(IColumn::mutate(std::move(child_column))); + } + auto result = ColumnStruct::create(std::move(mutable_child_columns)); + if (mapping.table_type->is_nullable()) { + auto null_map = ColumnUInt8::create(); + auto& null_map_data = null_map->get_data(); + null_map_data.resize(rows); + if (parent_null_map != nullptr) { + DORIS_CHECK(parent_null_map->size() == rows); + null_map_data.assign(parent_null_map->begin(), parent_null_map->end()); + } else { + std::fill(null_map_data.begin(), null_map_data.end(), 0); + } + *column = ColumnNullable::create(std::move(result), std::move(null_map)); + } else { + *column = std::move(result); + } + return Status::OK(); + } + + Status _materialize_array_mapping_column(const ColumnMapping& mapping, + const ColumnPtr& file_column, const size_t rows, + ColumnPtr* column) { + DORIS_CHECK(mapping.child_mappings.size() == 1); + const auto full_file_column = file_column->convert_to_full_column_if_const(); + const NullMap* parent_null_map = nullptr; + const auto* nested_file_column = + _nested_column_if_nullable(full_file_column, &parent_null_map); + const auto* file_array = assert_cast(nested_file_column); + ColumnPtr nested_column = file_array->get_data_ptr(); + const auto& element_mapping = mapping.child_mappings[0]; + RETURN_IF_ERROR(_materialize_present_child_mapping_column( + element_mapping, nested_column, nested_column->size(), &nested_column)); + auto offsets_column = file_array->get_offsets_ptr()->convert_to_full_column_if_const(); + auto result = ColumnArray::create(IColumn::mutate(std::move(nested_column)), + IColumn::mutate(std::move(offsets_column))); + if (mapping.table_type->is_nullable()) { + auto null_map = ColumnUInt8::create(); + auto& null_map_data = null_map->get_data(); + null_map_data.resize(rows); + if (parent_null_map != nullptr) { + DORIS_CHECK(parent_null_map->size() == rows); + null_map_data.assign(parent_null_map->begin(), parent_null_map->end()); + } else { + std::fill(null_map_data.begin(), null_map_data.end(), 0); + } + *column = ColumnNullable::create(std::move(result), std::move(null_map)); + } else { + *column = std::move(result); + } + return Status::OK(); + } + + Status _materialize_map_mapping_column(const ColumnMapping& mapping, + const ColumnPtr& file_column, const size_t rows, + ColumnPtr* column) { + const auto full_file_column = file_column->convert_to_full_column_if_const(); + const NullMap* parent_null_map = nullptr; + const auto* nested_file_column = + _nested_column_if_nullable(full_file_column, &parent_null_map); + const auto* file_map = assert_cast(nested_file_column); + ColumnPtr key_column = file_map->get_keys_ptr(); + ColumnPtr value_column = file_map->get_values_ptr(); + + const ColumnMapping* key_mapping = nullptr; + const ColumnMapping* value_mapping = nullptr; + for (const auto& child_mapping : mapping.child_mappings) { + if (!child_mapping.file_local_id.has_value()) { + continue; + } + if (*child_mapping.file_local_id == 0) { + key_mapping = &child_mapping; + } else if (*child_mapping.file_local_id == 1) { + value_mapping = &child_mapping; + } + } + + if (key_mapping != nullptr) { + RETURN_IF_ERROR(_materialize_present_child_mapping_column( + *key_mapping, key_column, key_column->size(), &key_column)); + } + if (value_mapping != nullptr) { + RETURN_IF_ERROR(_materialize_present_child_mapping_column( + *value_mapping, value_column, value_column->size(), &value_column)); + } + auto offsets_column = file_map->get_offsets_ptr()->convert_to_full_column_if_const(); + auto result = ColumnMap::create(IColumn::mutate(std::move(key_column)), + IColumn::mutate(std::move(value_column)), + IColumn::mutate(std::move(offsets_column))); + if (mapping.table_type->is_nullable()) { + auto null_map = ColumnUInt8::create(); + auto& null_map_data = null_map->get_data(); + null_map_data.resize(rows); + if (parent_null_map != nullptr) { + DORIS_CHECK(parent_null_map->size() == rows); + null_map_data.assign(parent_null_map->begin(), parent_null_map->end()); + } else { + std::fill(null_map_data.begin(), null_map_data.end(), 0); + } + *column = ColumnNullable::create(std::move(result), std::move(null_map)); + } else { + *column = std::move(result); + } + return Status::OK(); + } + + Status _open_mapping_exprs() { + RowDescriptor row_desc; + for (const auto& mapping : _data_reader.column_mapper->mappings()) { + if (mapping.projection != nullptr) { + RETURN_IF_ERROR(mapping.projection->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(mapping.projection->open(_runtime_state)); + } + if (mapping.default_expr != nullptr) { + RETURN_IF_ERROR(mapping.default_expr->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(mapping.default_expr->open(_runtime_state)); + } + } + return Status::OK(); + } + + Status _build_file_aggregate_request(TPushAggOp::type agg_type, + FileAggregateRequest* request) const { + DORIS_CHECK(request != nullptr); + DORIS_CHECK(_supports_aggregate_pushdown(agg_type)); + request->agg_type = agg_type; + request->columns.clear(); + if (agg_type == TPushAggOp::type::COUNT) { + // COUNT pushdown historically meant COUNT(*) and therefore carried no columns. For + // complex COUNT(col), materializing the full MAP/LIST/STRUCT value only to test the + // top-level NULL bit can be extremely expensive. When the scan projects exactly one + // directly-mapped complex column, pass that file column to the reader so formats such + // as Parquet can count the column shape from metadata/levels without decoding payload + // values like MAP value strings. Other COUNT cases stay on the existing row-count path + // to avoid changing count(*) semantics. + if (_data_reader.column_mapper->mappings().size() == 1) { + const auto& mapping = _data_reader.column_mapper->mappings()[0]; + if (mapping.file_local_id.has_value() && mapping.file_type != nullptr && + is_complex_type(remove_nullable(mapping.file_type)->get_primitive_type()) && + mapping.virtual_column_type == TableVirtualColumnType::INVALID && + mapping.default_expr == nullptr) { + FileAggregateRequest::Column column; + column.projection = + LocalColumnIndex::top_level(LocalColumnId(*mapping.file_local_id)); + request->columns.push_back(std::move(column)); + } + } + return Status::OK(); + } + request->columns.reserve(_data_reader.column_mapper->mappings().size()); + for (const auto& mapping : _data_reader.column_mapper->mappings()) { + DORIS_CHECK(mapping.file_local_id.has_value()); + FileAggregateRequest::Column column; + column.projection = LocalColumnIndex::top_level(LocalColumnId(*mapping.file_local_id)); + if (!mapping.child_mappings.empty()) { + RETURN_IF_ERROR(build_aggregate_projection(mapping, &column.projection)); + } + request->columns.push_back(std::move(column)); + } + return Status::OK(); + } + + Status _materialize_aggregate_pushdown_rows(TPushAggOp::type agg_type, + const FileAggregateResult& file_result, + Block* block) { + if (agg_type == TPushAggOp::type::COUNT) { + // COUNT pushdown is not a final count value. It emits `count` default rows so the + // upper COUNT(*) aggregate can count them and produce the final result, including + // zero rows when count is 0. + DORIS_CHECK(file_result.count >= 0); + return _materialize_count_rows(cast_set(file_result.count), block); + } + // MIN/MAX pushdown emits two rows, min first and max second, for each projected column. + // The upper MIN/MAX aggregate consumes those two rows to produce the final aggregate value. + DORIS_CHECK(file_result.columns.size() == _data_reader.column_mapper->mappings().size()); + DORIS_CHECK(block->columns() == _data_reader.column_mapper->mappings().size()); + Block file_block; + file_block.reserve(_data_reader.file_block_layout.size()); + for (const auto& column : _data_reader.file_block_layout) { + file_block.insert({column.type->create_column(), column.type, column.name}); + } + for (size_t column_idx = 0; column_idx < file_result.columns.size(); ++column_idx) { + const auto& result_column = file_result.columns[column_idx]; + if (!result_column.has_min || !result_column.has_max) { + return Status::NotSupported("Missing min/max aggregate result for column {}", + _projected_columns[column_idx].name); + } + bool found_file_column = false; + for (size_t block_position = 0; block_position < _data_reader.file_block_layout.size(); + ++block_position) { + if (_data_reader.file_block_layout[block_position].file_column_id == + file_result.columns[column_idx].projection.column_id()) { + found_file_column = true; + auto column = file_block.get_by_position(block_position) + .type->create_column() + ->assert_mutable(); + RETURN_IF_ERROR(_insert_aggregate_projection_value( + file_result.columns[column_idx].projection, result_column.min_value, + column.get())); + RETURN_IF_ERROR(_insert_aggregate_projection_value( + file_result.columns[column_idx].projection, result_column.max_value, + column.get())); + file_block.replace_by_position(block_position, std::move(column)); + break; + } + } + DORIS_CHECK(found_file_column); + } + for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size(); + ++column_idx) { + ColumnPtr table_column; + RETURN_IF_ERROR( + _materialize_mapping_column(_data_reader.column_mapper->mappings()[column_idx], + &file_block, 2, &table_column)); + block->replace_by_position(column_idx, std::move(table_column)); + } + return Status::OK(); + } + + struct FileBlockColumn { + LocalColumnId file_column_id = LocalColumnId::invalid(); + std::string name; + DataTypePtr type; + }; + + struct DataReader { + std::unique_ptr reader; + std::unique_ptr column_mapper; + // Schema of the data file, also including virtual column (row position). + std::vector file_schema; + // Layout of the block returned by file reader, determined by column mapping and file + // schema. It is used for file reader to materialize columns into correct type and position. + std::vector file_block_layout; + Block block_template; + }; + DataReader _data_reader; + std::vector _projected_columns; + std::unique_ptr _current_task; + std::optional _current_file_description; + // Range-level compression has higher priority than scan-param compression. TVF/load can keep + // the logical format as CSV/TEXT while carrying the concrete compression such as GZ or LZO on + // each TFileRangeDesc, matching the old FileScanner reader contract. + TFileCompressType::type _current_range_compress_type = TFileCompressType::UNKNOWN; + std::optional _current_range_load_id; + TFileRangeDesc _current_file_range_desc; + std::shared_ptr _system_properties; + // partition key -> value + std::map _partition_values; + // Predicates built from scan conjuncts before file-level localization. + std::vector _table_filters; + // Number of localized filters before the first unsafe conjunct in the original row-level + // order. This differs from scanning `_table_filters` for safety because slotless predicates are + // intentionally absent from that vector but must still act as ordering barriers. + size_t _constant_pruning_safe_filter_count = 0; + VExprContextSPtrs _conjuncts; + ReadProfile _profile; + // Parsed from row-position based delete files, including position delete and deletion vector. + DeleteRows* _delete_rows = nullptr; + DeletionVector* _deletion_vector = nullptr; + TFileScanRangeParams* _scan_params; + std::shared_ptr _io_ctx; + RuntimeState* _runtime_state; + RuntimeProfile* _scanner_profile; + const std::vector* _file_slot_descs = nullptr; + FileFormat _format; + TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; + size_t _batch_size = 0; + uint64_t _condition_cache_digest = 0; + segment_v2::ConditionCache::ExternalCacheKey _condition_cache_key; + std::shared_ptr> _condition_cache; + std::shared_ptr _condition_cache_ctx; + int64_t _condition_cache_hit_count = 0; + bool _current_reader_reached_eof = false; + int64_t _remaining_table_level_count = -1; + // Snapshot supplied by FileScannerV2 for the active split. It gates every shortcut that emits + // irreversible aggregate rows, not only the table-level row-count shortcut in prepare_split(). + bool _all_runtime_filters_applied_for_split = true; + std::optional _global_rowid_context; + bool _aggregate_pushdown_tried = false; + bool _current_split_pruned = false; + TableColumnMapperOptions _mapper_options; + +private: + static const ColumnDefinition* _find_column_definition( + const std::vector& schema, LocalColumnId column_id) { + for (const auto& field : schema) { + if (field.file_local_id() == column_id.value()) { + return &field; + } + } + return nullptr; + } + + static bool _can_push_down_minmax_for_mapping(const ColumnMapping& mapping) { + if (mapping.child_mappings.empty()) { + // Direct mappings use a slot-ref projection to materialize the file column. The + // projection does not transform ordering; casts and other conversions are already + // represented by a non-trivial mapping and must fall back to row scanning. + return mapping.is_trivial; + } + const auto primitive_type = remove_nullable(mapping.file_type)->get_primitive_type(); + if (primitive_type != TYPE_STRUCT) { + return false; + } + size_t mapped_children = 0; + const ColumnMapping* mapped_child = nullptr; + for (const auto& child_mapping : mapping.child_mappings) { + if (!child_mapping.file_local_id.has_value()) { + continue; + } + ++mapped_children; + mapped_child = &child_mapping; + } + return mapped_children == 1 && mapped_child != nullptr && + _can_push_down_minmax_for_mapping(*mapped_child); + } + + static Status build_aggregate_projection(const ColumnMapping& mapping, + LocalColumnIndex* projection) { + DORIS_CHECK(projection != nullptr); + DORIS_CHECK(mapping.file_local_id.has_value()); + *projection = LocalColumnIndex::local(*mapping.file_local_id); + projection->children.clear(); + projection->project_all_children = true; + if (mapping.child_mappings.empty()) { + return Status::OK(); + } + projection->project_all_children = false; + for (const auto& child_mapping : mapping.child_mappings) { + if (!child_mapping.file_local_id.has_value()) { + continue; + } + LocalColumnIndex child_projection; + RETURN_IF_ERROR(build_aggregate_projection(child_mapping, &child_projection)); + projection->children.push_back(std::move(child_projection)); + } + DORIS_CHECK(projection->children.size() == 1); + return Status::OK(); + } + + static Status _insert_aggregate_projection_value(const LocalColumnIndex& projection, + const Field& value, IColumn* column) { + DORIS_CHECK(column != nullptr); + if (auto* nullable_column = check_and_get_column(*column)) { + RETURN_IF_ERROR(_insert_aggregate_projection_value( + projection, value, &nullable_column->get_nested_column())); + nullable_column->get_null_map_data().push_back(0); + return Status::OK(); + } + if (projection.project_all_children || projection.children.empty()) { + column->insert(value); + return Status::OK(); + } + auto* struct_column = assert_cast(column); + DORIS_CHECK(projection.children.size() == 1); + const auto& child_projection = projection.children[0]; + DORIS_CHECK(struct_column->get_columns().size() == 1); + RETURN_IF_ERROR(_insert_aggregate_projection_value(child_projection, value, + &struct_column->get_column(0))); + return Status::OK(); + } + + // Parse a DV into its compressed bitmap. Position delete files continue to use _delete_rows. + Status _parse_delete_predicates(const SplitReadOptions& options); +}; + +} // namespace doris::format diff --git a/be/src/format_v2/timestamp_statistics.h b/be/src/format_v2/timestamp_statistics.h new file mode 100644 index 00000000000000..6b13f44ab16220 --- /dev/null +++ b/be/src/format_v2/timestamp_statistics.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/check.h" + +namespace doris::format { + +inline int64_t floor_epoch_seconds(int64_t value, int64_t units_per_second) { + DORIS_CHECK(units_per_second > 0); + auto seconds = value / units_per_second; + if (value < 0 && value % units_per_second != 0) { + --seconds; + } + return seconds; +} + +// UTC instants are ordered, but converting them to civil DATETIMEV2 values is not monotonic when +// a timezone moves its clock backward. Metadata min/max converted across such a transition cannot +// safely form a ZoneMap: the true civil minimum or maximum may occur inside the UTC interval. +inline bool utc_timestamp_range_is_monotonic(int64_t min_seconds, int64_t max_seconds, + const cctz::time_zone& timezone) { + // These endpoints come from external file metadata. An inverted range is corrupt rather than + // a valid precondition violation, so report it as unusable statistics and let every caller + // take its conservative scan/fallback path instead of terminating the BE. + if (min_seconds > max_seconds) { + return false; + } + auto current = cctz::time_point(cctz::seconds(min_seconds)); + const auto range_end = cctz::time_point(cctz::seconds(max_seconds)); + cctz::time_zone::civil_transition transition; + while (timezone.next_transition(current, &transition)) { + const auto transition_time = timezone.lookup(transition.to).trans; + if (transition_time > range_end) { + return true; + } + if (transition.to < transition.from) { + return false; + } + // Move past the transition that was just inspected. Some cctz implementations return the + // same transition again when queried at its exact instant, which would otherwise prevent + // us from seeing a later rollback in the requested range. + current = transition_time + cctz::seconds(1); + } + return true; +} + +} // namespace doris::format diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 31d7a7801afbde..43acf2e01fd3ee 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -58,25 +58,30 @@ namespace doris { constexpr std::string_view RANDOM_CACHE_BASE_PATH = "random"; -io::FileReaderOptions FileFactory::get_reader_options(RuntimeState* state, +io::FileReaderOptions FileFactory::get_reader_options(const TQueryOptions& option, const io::FileDescription& fd) { io::FileReaderOptions opts { .cache_base_path {}, .file_size = fd.file_size, .mtime = fd.mtime, }; - if (config::enable_file_cache && state != nullptr && - state->query_options().__isset.enable_file_cache && - state->query_options().enable_file_cache && fd.file_cache_admission) { + if (config::enable_file_cache && option.__isset.enable_file_cache && option.enable_file_cache && + fd.file_cache_admission) { opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; } - if (state != nullptr && state->query_options().__isset.file_cache_base_path && - state->query_options().file_cache_base_path != RANDOM_CACHE_BASE_PATH) { - opts.cache_base_path = state->query_options().file_cache_base_path; + if (option.__isset.file_cache_base_path && + option.file_cache_base_path != RANDOM_CACHE_BASE_PATH) { + opts.cache_base_path = option.file_cache_base_path; } return opts; } +io::FileReaderOptions FileFactory::get_reader_options(RuntimeState* state, + const io::FileDescription& fd) { + return state == nullptr ? get_reader_options(TQueryOptions {}, fd) + : get_reader_options(state->query_options(), fd); +} + int32_t get_broker_index(const std::vector& brokers, const std::string& path) { if (brokers.empty()) { return -1; diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h index 7d662e4fdde469..8454ab6ed9ec10 100644 --- a/be/src/io/file_factory.h +++ b/be/src/io/file_factory.h @@ -16,6 +16,7 @@ // under the License. #pragma once +#include #include #include #include @@ -64,6 +65,8 @@ struct FileDescription { // -1 means unset. // If the file length is not set, the file length will be fetched from the file system. int64_t file_size = -1; + int64_t range_start_offset = 0; + int64_t range_size = -1; // modification time of this file. // 0 means unset. int64_t mtime = 0; @@ -83,6 +86,8 @@ class FileFactory { ENABLE_FACTORY_CREATOR(FileFactory); public: + static io::FileReaderOptions get_reader_options(const TQueryOptions& option, + const io::FileDescription& fd); static io::FileReaderOptions get_reader_options(RuntimeState* state, const io::FileDescription& fd); diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index 031387708ea6c5..6afb9d4a705d70 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -82,6 +82,49 @@ struct FileCacheStatistics { int64_t segment_footer_index_local_io_timer = 0; int64_t segment_footer_index_remote_io_timer = 0; int64_t segment_footer_index_peer_io_timer = 0; + + void merge_from(const FileCacheStatistics& other) { + num_local_io_total += other.num_local_io_total; + num_remote_io_total += other.num_remote_io_total; + num_peer_io_total += other.num_peer_io_total; + local_io_timer += other.local_io_timer; + bytes_read_from_local += other.bytes_read_from_local; + bytes_read_from_remote += other.bytes_read_from_remote; + bytes_read_from_peer += other.bytes_read_from_peer; + remote_io_timer += other.remote_io_timer; + peer_io_timer += other.peer_io_timer; + remote_wait_timer += other.remote_wait_timer; + write_cache_io_timer += other.write_cache_io_timer; + bytes_write_into_cache += other.bytes_write_into_cache; + num_skip_cache_io_total += other.num_skip_cache_io_total; + read_cache_file_directly_timer += other.read_cache_file_directly_timer; + cache_get_or_set_timer += other.cache_get_or_set_timer; + lock_wait_timer += other.lock_wait_timer; + get_timer += other.get_timer; + set_timer += other.set_timer; + inverted_index_num_local_io_total += other.inverted_index_num_local_io_total; + inverted_index_num_remote_io_total += other.inverted_index_num_remote_io_total; + inverted_index_num_peer_io_total += other.inverted_index_num_peer_io_total; + inverted_index_bytes_read_from_local += other.inverted_index_bytes_read_from_local; + inverted_index_bytes_read_from_remote += other.inverted_index_bytes_read_from_remote; + inverted_index_bytes_read_from_peer += other.inverted_index_bytes_read_from_peer; + inverted_index_local_io_timer += other.inverted_index_local_io_timer; + inverted_index_remote_io_timer += other.inverted_index_remote_io_timer; + inverted_index_peer_io_timer += other.inverted_index_peer_io_timer; + inverted_index_io_timer += other.inverted_index_io_timer; + segment_footer_index_num_local_io_total += other.segment_footer_index_num_local_io_total; + segment_footer_index_num_remote_io_total += other.segment_footer_index_num_remote_io_total; + segment_footer_index_num_peer_io_total += other.segment_footer_index_num_peer_io_total; + segment_footer_index_bytes_read_from_local += + other.segment_footer_index_bytes_read_from_local; + segment_footer_index_bytes_read_from_remote += + other.segment_footer_index_bytes_read_from_remote; + segment_footer_index_bytes_read_from_peer += + other.segment_footer_index_bytes_read_from_peer; + segment_footer_index_local_io_timer += other.segment_footer_index_local_io_timer; + segment_footer_index_remote_io_timer += other.segment_footer_index_remote_io_timer; + segment_footer_index_peer_io_timer += other.segment_footer_index_peer_io_timer; + } }; struct IOContext { @@ -104,6 +147,11 @@ struct IOContext { bool is_dryrun = false; // if `is_warmup` == true, this I/O request is from a warm up task bool is_warmup {false}; + int64_t condition_cache_filtered_rows = 0; + // Rows removed by file-local predicate conjuncts inside FileReader/TableReader. Scanner-level + // output filtering already records its own unselected rows; this counter carries the rows that + // were filtered before the block returned to Scanner. + int64_t predicate_filtered_rows = 0; }; } // namespace io diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 13e4b7b6355a6f..4fdce6c07c2d0d 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -106,6 +106,14 @@ class RuntimeState { // for job task only RuntimeState(); +#ifdef BE_TEST + // Compatibility constructor for format_v2 tests backported to branch-4.1. + RuntimeState(const TQueryOptions& query_options, const TQueryGlobals& query_globals) + : RuntimeState(query_globals) { + set_query_options(query_options); + } +#endif + // Empty d'tor to avoid issues with unique_ptr. MOCK_DEFINE(virtual) ~RuntimeState(); diff --git a/be/src/storage/rowset/rowset_meta.h b/be/src/storage/rowset/rowset_meta.h index fc830462e7e875..51b11494862434 100644 --- a/be/src/storage/rowset/rowset_meta.h +++ b/be/src/storage/rowset/rowset_meta.h @@ -177,21 +177,21 @@ class RowsetMeta : public MetadataAdder { _rowset_meta_pb.set_index_disk_size(index_disk_size); } - void zone_maps(std::vector* zone_maps) { - for (const ZoneMap& zone_map : _rowset_meta_pb.zone_maps()) { + void zone_maps(std::vector<::doris::ZoneMap>* zone_maps) { + for (const ::doris::ZoneMap& zone_map : _rowset_meta_pb.zone_maps()) { zone_maps->push_back(zone_map); } } - void set_zone_maps(const std::vector& zone_maps) { - for (const ZoneMap& zone_map : zone_maps) { - ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); + void set_zone_maps(const std::vector<::doris::ZoneMap>& zone_maps) { + for (const ::doris::ZoneMap& zone_map : zone_maps) { + ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); *new_zone_map = zone_map; } } - void add_zone_map(const ZoneMap& zone_map) { - ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); + void add_zone_map(const ::doris::ZoneMap& zone_map) { + ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); *new_zone_map = zone_map; } diff --git a/be/src/storage/segment/condition_cache.cpp b/be/src/storage/segment/condition_cache.cpp index ebfea806e46fc6..f0272584d78467 100644 --- a/be/src/storage/segment/condition_cache.cpp +++ b/be/src/storage/segment/condition_cache.cpp @@ -23,7 +23,8 @@ namespace doris::segment_v2 { -bool ConditionCache::lookup(const CacheKey& key, ConditionCacheHandle* handle) { +template +bool ConditionCache::lookup(const KeyType& key, ConditionCacheHandle* handle) { if (key.encode().empty()) { return false; } @@ -35,13 +36,17 @@ bool ConditionCache::lookup(const CacheKey& key, ConditionCacheHandle* handle) { return true; } -void ConditionCache::insert(const CacheKey& key, std::shared_ptr> result) { - if (key.encode().empty()) { +template +void ConditionCache::insert(const KeyType& key, std::shared_ptr> result, + int64_t base_granule) { + auto encoded_key = key.encode(); + if (encoded_key.empty()) { return; } std::unique_ptr cache_value_ptr = std::make_unique(); cache_value_ptr->filter_result = result; + cache_value_ptr->base_granule = base_granule; ConditionCacheHandle( this, @@ -49,4 +54,16 @@ void ConditionCache::insert(const CacheKey& key, std::shared_ptrcapacity(), result->capacity(), CachePriority::NORMAL)); } +// Explicit template instantiations +template bool ConditionCache::lookup(const CacheKey& key, + ConditionCacheHandle* handle); +template bool ConditionCache::lookup( + const ExternalCacheKey& key, ConditionCacheHandle* handle); +template void ConditionCache::insert( + const CacheKey& key, std::shared_ptr> filter_result, + int64_t base_granule); +template void ConditionCache::insert( + const ExternalCacheKey& key, std::shared_ptr> filter_result, + int64_t base_granule); + } // namespace doris::segment_v2 diff --git a/be/src/storage/segment/condition_cache.h b/be/src/storage/segment/condition_cache.h index 17b9f8470b8113..2f0670a4714fe4 100644 --- a/be/src/storage/segment/condition_cache.h +++ b/be/src/storage/segment/condition_cache.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "common/config.h" #include "common/status.h" @@ -38,7 +39,20 @@ #include "util/slice.h" #include "util/time.h" -namespace doris::segment_v2 { +namespace doris { + +// Context passed from scan/table-reader layers to physical readers for condition cache +// integration. On MISS, readers set filter_result[granule] to true when row-level predicates keep +// at least one row in that granule. On HIT, readers skip granules whose cached bit is false. +struct ConditionCacheContext { + bool is_hit = false; + std::shared_ptr> filter_result; // per-granule: true = has surviving rows + int64_t base_granule = 0; // global granule index of filter_result[0] + size_t num_granules = 0; // authoritative bitmap length; excludes allocation-only guard bits + static constexpr int GRANULE_SIZE = 2048; +}; + +namespace segment_v2 { class ConditionCacheHandle; @@ -67,6 +81,46 @@ class ConditionCache : public LRUCachePolicy { class CacheValue : public LRUCacheValueBase { public: std::shared_ptr> filter_result; + // The bitmap coordinate system is part of the cached result. A later scan may prune a + // different first row group, so it must not derive this origin from its current plan. + int64_t base_granule = 0; + }; + + // Cache key for external tables (Hive ORC/Parquet) + struct ExternalCacheKey { + static constexpr uint8_t BASE_GRANULE_AWARE_VERSION = 1; + + ExternalCacheKey() = default; + ExternalCacheKey(const std::string& path_, int64_t modification_time_, int64_t file_size_, + uint64_t digest_, int64_t start_offset_, int64_t size_, + uint8_t format_version_ = 0) + : path(path_), + modification_time(modification_time_), + file_size(file_size_), + digest(digest_), + start_offset(start_offset_), + size(size_), + format_version(format_version_) {} + std::string path; + int64_t modification_time = 0; + int64_t file_size = 0; + uint64_t digest = 0; + int64_t start_offset = 0; + int64_t size = 0; + uint8_t format_version = 0; + + [[nodiscard]] std::string encode() const { + std::string key = path; + char buf[41]; + memcpy(buf, &modification_time, 8); + memcpy(buf + 8, &file_size, 8); + memcpy(buf + 16, &digest, 8); + memcpy(buf + 24, &start_offset, 8); + memcpy(buf + 32, &size, 8); + buf[40] = static_cast(format_version); + key.append(buf, 41); + return key; + } }; // Create global instance of this class @@ -87,9 +141,12 @@ class ConditionCache : public LRUCachePolicy { /*element_count_capacity*/ 0, /*enable_prune*/ true, /*is_lru_k*/ false) {} - bool lookup(const CacheKey& key, ConditionCacheHandle* handle); + template + bool lookup(const KeyType& key, ConditionCacheHandle* handle); - void insert(const CacheKey& key, std::shared_ptr> filter_result); + template + void insert(const KeyType& key, std::shared_ptr> filter_result, + int64_t base_granule = 0); }; class ConditionCacheHandle { @@ -126,6 +183,11 @@ class ConditionCacheHandle { return ((ConditionCache::CacheValue*)_cache->value(_handle))->filter_result; } + int64_t get_base_granule() const { + DORIS_CHECK(_cache != nullptr); + return ((ConditionCache::CacheValue*)_cache->value(_handle))->base_granule; + } + private: LRUCachePolicy* _cache = nullptr; Cache::Handle* _handle = nullptr; @@ -134,4 +196,5 @@ class ConditionCacheHandle { DISALLOW_COPY_AND_ASSIGN(ConditionCacheHandle); }; -} // namespace doris::segment_v2 +} // namespace segment_v2 +} // namespace doris diff --git a/be/src/util/jni-util.h b/be/src/util/jni-util.h index 59a9cd503de022..16fbe1b587f802 100644 --- a/be/src/util/jni-util.h +++ b/be/src/util/jni-util.h @@ -606,6 +606,14 @@ class Object { bool uninitialized() const { return _obj == nullptr; } + void reset(JNIEnv* env) { + if (_obj == nullptr) { + return; + } + RefHelper::destroy(env, _obj); + _obj = nullptr; + } + template bool equal(JNIEnv* env, const Object& other) { DCHECK(!uninitialized()); diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index d7c11338d63853..d97ae36b53bc5b 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -21,7 +21,39 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/test") # where to put generated libraries set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test") -file(GLOB_RECURSE UT_FILES CONFIGURE_DEPENDS *.cpp) +file(GLOB_RECURSE UT_FILES CONFIGURE_DEPENDS + agent/*.cpp + ai/*.cpp + cloud/*.cpp + common/*.cpp + core/*.cpp + exec/*.cpp + exprs/*.cpp + format/*.cpp + format_v2/*.cpp + gutil/*.cpp + io/*.cpp + load/*.cpp + olap/*.cpp + runtime/*.cpp + service/*.cpp + storage/*.cpp + testutil/*.cpp + udf/*.cpp + tools/*.cpp + util/*.cpp + vec/*.cpp +) + +if (ENABLE_TDE) + file(GLOB_RECURSE TDE_UT_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${TDE_MODULE_DIR}/*.cpp") + list(APPEND UT_FILES ${TDE_UT_FILES}) +endif() + +if (ENABLE_TLS) + file(GLOB_RECURSE TLS_UT_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${TLS_MODULE_DIR}/*.cpp") + list(APPEND UT_FILES ${TLS_UT_FILES}) +endif() set(PHDR_CACHE_TEST_DSO_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/common/phdr_cache_test_dso.cpp) list(REMOVE_ITEM UT_FILES ${PHDR_CACHE_TEST_DSO_SOURCE}) diff --git a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp new file mode 100644 index 00000000000000..69cf458e2fdc5f --- /dev/null +++ b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp @@ -0,0 +1,1852 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/column/column_decimal.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_nothing.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "core/field.h" +#include "core/string_ref.h" +#include "core/value/timestamptz_value.h" +#include "util/timezone_utils.h" + +namespace doris { +namespace { + +struct ReadColumnResult { + Status status; + MutableColumnPtr column; +}; + +template +DecodedColumnView make_fixed_view(DecodedValueKind kind, const std::vector& values, + const std::vector* null_map = nullptr) { + DecodedColumnView view; + view.value_kind = kind; + view.row_count = null_map != nullptr ? static_cast(null_map->size()) + : static_cast(values.size()); + view.values = values.empty() ? nullptr : reinterpret_cast(values.data()); + view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); + return view; +} + +DecodedColumnView make_binary_view(DecodedValueKind kind, const std::vector& values, + int fixed_length = -1, + const std::vector* null_map = nullptr) { + DecodedColumnView view; + view.value_kind = kind; + view.row_count = null_map != nullptr ? static_cast(null_map->size()) + : static_cast(values.size()); + view.binary_values = values.empty() ? nullptr : &values; + view.fixed_length = fixed_length; + view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); + return view; +} + +DecodedColumnView make_bool_view(const std::vector& values, + const std::vector* null_map = nullptr) { + DecodedColumnView view; + view.value_kind = DecodedValueKind::BOOL; + view.row_count = null_map != nullptr ? static_cast(null_map->size()) + : static_cast(values.size()); + view.values = values.empty() ? nullptr : reinterpret_cast(values.data()); + view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); + return view; +} + +DecodedColumnView with_logical_integer(DecodedColumnView view, int bit_width, bool is_signed) { + view.logical_integer_bit_width = bit_width; + view.logical_integer_is_signed = is_signed; + return view; +} + +ReadColumnResult read_column(const DataTypePtr& type, const DecodedColumnView& view) { + auto column = type->create_column(); + auto status = type->get_serde()->read_column_from_decoded_values(*column, view); + return {std::move(status), std::move(column)}; +} + +void expect_not_supported(const Status& status) { + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::NOT_IMPLEMENTED_ERROR, status.code()) << status; +} + +void expect_corruption(const Status& status) { + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::CORRUPTION, status.code()) << status; +} + +void expect_data_quality_error(const Status& status) { + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::DATA_QUALITY_ERROR, status.code()) << status; +} + +void expect_column_strings(const IDataType& type, const IColumn& column, + const std::vector& expected) { + ASSERT_EQ(expected.size(), column.size()); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(expected[row], type.to_string(column, row)) << "row=" << row; + } +} + +void expect_binary_column(const IColumn& column, const std::vector& expected) { + const auto& string_column = assert_cast(column); + ASSERT_EQ(expected.size(), string_column.size()); + for (size_t row = 0; row < expected.size(); ++row) { + const auto value = string_column.get_data_at(row); + EXPECT_EQ(expected[row], std::string(value.data, value.size)) << "row=" << row; + } +} + +void expect_nullable_all_null(const IColumn& column, size_t expected_size) { + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(expected_size, nullable_column.size()); + ASSERT_EQ(expected_size, nullable_column.get_nested_column().size()); + for (size_t row = 0; row < expected_size; ++row) { + EXPECT_TRUE(nullable_column.is_null_at(row)) << "row=" << row; + } +} + +Field read_field(const DataTypePtr& type, const DecodedColumnView& view) { + Field field; + auto status = type->get_serde()->read_field_from_decoded_value(*type, &field, view); + EXPECT_TRUE(status.ok()) << status; + return field; +} + +Status read_field_status(const DataTypePtr& type, const DecodedColumnView& view) { + Field field; + return type->get_serde()->read_field_from_decoded_value(*type, &field, view); +} + +std::vector string_refs(const std::vector& values) { + std::vector refs; + refs.reserve(values.size()); + for (const auto& value : values) { + refs.emplace_back(value.data(), value.size()); + } + return refs; +} + +#pragma pack(1) +struct TestInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() + +static_assert(sizeof(TestInt96Timestamp) == 12); + +Decimal128V3 decimal128_v3(Int128 value) { + return Decimal128V3(value); +} + +Decimal256 decimal256_from_int64(int64_t value) { + return Decimal256(wide::Int256(value)); +} + +} // namespace + +// ---------------------------------------------------------------------- +// Base SerDe behavior +// ---------------------------------------------------------------------- +// These cases define the default contract for types that have not implemented decoded-value +// materialization. Batch reads must report NotSupported, and the single-field path must surface +// the same error because it is implemented by delegating to the batch reader. + +TEST(DataTypeSerDeDecodedValuesTest, BaseSerdeRejectsDecodedValues) { + auto type = std::make_shared(); + std::vector values = {1}; + auto view = make_fixed_view(DecodedValueKind::INT32, values); + + auto result = read_column(type, view); + + expect_not_supported(result.status); + EXPECT_EQ(0, result.column->size()); + EXPECT_NE(std::string::npos, result.status.to_string().find("Nothing")); +} + +TEST(DataTypeSerDeDecodedValuesTest, BaseFieldUsesBatchReaderAndPropagatesError) { + auto type = std::make_shared(); + std::vector values = {1}; + auto view = make_fixed_view(DecodedValueKind::INT32, values); + Field field = Field::create_field(123); + + auto status = type->get_serde()->read_field_from_decoded_value(*type, &field, view); + + expect_not_supported(status); + EXPECT_EQ(TYPE_INT, field.get_type()); + EXPECT_EQ(123, field.get()); +} + +// ---------------------------------------------------------------------- +// Number SerDe happy path +// ---------------------------------------------------------------------- +// The numeric matrix verifies physical kind dispatch and the exact static_cast behavior used by +// the reader. Narrow integer overflow is intentionally locked to current C++ conversion behavior; +// if product semantics change to reject overflow, these expectations should be updated with the +// implementation change. + +TEST(DataTypeSerDeDecodedValuesTest, ReadBooleanFromBool) { + auto type = std::make_shared(); + std::vector values = {true, false, true}; + auto view = make_bool_view(values); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(3, column.size()); + EXPECT_EQ(1, column.get_element(0)); + EXPECT_EQ(0, column.get_element(1)); + EXPECT_EQ(1, column.get_element(2)); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadSignedIntegersFromInt32) { + std::vector values = {0, 1, -1, 127, -128}; + auto view = make_fixed_view(DecodedValueKind::INT32, values); + + { + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(values.size(), column.size()); + EXPECT_EQ(0, column.get_element(0)); + EXPECT_EQ(1, column.get_element(1)); + EXPECT_EQ(-1, column.get_element(2)); + EXPECT_EQ(127, column.get_element(3)); + EXPECT_EQ(-128, column.get_element(4)); + } + { + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(values.size(), column.size()); + for (size_t row = 0; row < values.size(); ++row) { + EXPECT_EQ(static_cast(values[row]), column.get_element(row)); + } + } + { + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(values.size(), column.size()); + for (size_t row = 0; row < values.size(); ++row) { + EXPECT_EQ(values[row], column.get_element(row)); + } + } + { + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(values.size(), column.size()); + for (size_t row = 0; row < values.size(); ++row) { + EXPECT_EQ(static_cast(values[row]), column.get_element(row)); + } + } + { + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(values.size(), column.size()); + for (size_t row = 0; row < values.size(); ++row) { + EXPECT_EQ(static_cast<__int128_t>(values[row]), column.get_element(row)); + } + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadSignedIntegersFromInt64) { + std::vector values = {0, 1, -1, 127, -128}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + + auto tiny = read_column(std::make_shared(), view); + ASSERT_TRUE(tiny.status.ok()) << tiny.status; + const auto& tiny_column = assert_cast(*tiny.column); + EXPECT_EQ(127, tiny_column.get_element(3)); + EXPECT_EQ(-128, tiny_column.get_element(4)); + + auto small = read_column(std::make_shared(), view); + ASSERT_TRUE(small.status.ok()) << small.status; + const auto& small_column = assert_cast(*small.column); + EXPECT_EQ(127, small_column.get_element(3)); + EXPECT_EQ(-128, small_column.get_element(4)); + + auto integer = read_column(std::make_shared(), view); + ASSERT_TRUE(integer.status.ok()) << integer.status; + const auto& int_column = assert_cast(*integer.column); + EXPECT_EQ(127, int_column.get_element(3)); + EXPECT_EQ(-128, int_column.get_element(4)); + + auto bigint = read_column(std::make_shared(), view); + ASSERT_TRUE(bigint.status.ok()) << bigint.status; + const auto& bigint_column = assert_cast(*bigint.column); + ASSERT_EQ(values.size(), bigint_column.size()); + for (size_t row = 0; row < values.size(); ++row) { + EXPECT_EQ(values[row], bigint_column.get_element(row)); + } + + auto largeint = read_column(std::make_shared(), view); + ASSERT_TRUE(largeint.status.ok()) << largeint.status; + const auto& largeint_column = assert_cast(*largeint.column); + ASSERT_EQ(values.size(), largeint_column.size()); + for (size_t row = 0; row < values.size(); ++row) { + EXPECT_EQ(static_cast<__int128_t>(values[row]), largeint_column.get_element(row)); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadIntegersFromUnsignedSources) { + { + std::vector values = {0, 1, std::numeric_limits::max()}; + auto view = make_fixed_view(DecodedValueKind::UINT32, values); + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(0, column.get_element(0)); + EXPECT_EQ(1, column.get_element(1)); + EXPECT_EQ(static_cast(std::numeric_limits::max()), + column.get_element(2)); + } + { + std::vector values = {0, 1, std::numeric_limits::max()}; + auto view = make_fixed_view(DecodedValueKind::UINT64, values); + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(0, column.get_element(0)); + EXPECT_EQ(1, column.get_element(1)); + EXPECT_EQ(static_cast<__int128_t>(std::numeric_limits::max()), + column.get_element(2)); + } + { + std::vector values = {static_cast(std::numeric_limits::max())}; + auto view = make_fixed_view(DecodedValueKind::UINT64, values); + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(std::numeric_limits::max(), column.get_element(0)); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalValues) { + { + std::vector values = {0, 127, 255, 32767, 65535, -1}; + auto view = + with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(values.size(), column.size()); + EXPECT_EQ(0, column.get_element(0)); + EXPECT_EQ(127, column.get_element(1)); + EXPECT_EQ(255, column.get_element(2)); + EXPECT_EQ(255, column.get_element(3)); + EXPECT_EQ(255, column.get_element(4)); + EXPECT_EQ(255, column.get_element(5)); + } + { + std::vector values = {32767, 65535, -1}; + auto view = + with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 16, false); + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(values.size(), column.size()); + EXPECT_EQ(32767, column.get_element(0)); + EXPECT_EQ(65535, column.get_element(1)); + EXPECT_EQ(65535, column.get_element(2)); + } + { + std::vector values = {-1}; + auto view = + with_logical_integer(make_fixed_view(DecodedValueKind::UINT32, values), 32, false); + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(1, column.size()); + EXPECT_EQ(4294967295LL, column.get_element(0)); + } + { + std::vector values = {-1}; + auto view = + with_logical_integer(make_fixed_view(DecodedValueKind::UINT64, values), 64, false); + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(1, column.size()); + EXPECT_EQ(static_cast<__int128_t>(std::numeric_limits::max()), + column.get_element(0)); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadSignedLogicalIntegersCastsPhysicalValues) { + std::vector values = {127, 128, 255, -1}; + auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, true); + auto result = read_column(std::make_shared(), view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + ASSERT_EQ(values.size(), column.size()); + EXPECT_EQ(static_cast(127), column.get_element(0)); + EXPECT_EQ(static_cast(-128), column.get_element(1)); + EXPECT_EQ(static_cast(-1), column.get_element(2)); + EXPECT_EQ(static_cast(-1), column.get_element(3)); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadFloatAndDouble) { + { + auto type = std::make_shared(); + std::vector values = {0.0F, -0.0F, 1.5F, -2.25F}; + auto result = read_column(type, make_fixed_view(DecodedValueKind::FLOAT, values)); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_FLOAT_EQ(0.0F, column.get_element(0)); + EXPECT_TRUE(std::signbit(column.get_element(1))); + EXPECT_FLOAT_EQ(1.5F, column.get_element(2)); + EXPECT_FLOAT_EQ(-2.25F, column.get_element(3)); + } + { + auto type = std::make_shared(); + std::vector values = {0.0, -0.0, 1.5, -2.25}; + auto result = read_column(type, make_fixed_view(DecodedValueKind::DOUBLE, values)); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_DOUBLE_EQ(0.0, column.get_element(0)); + EXPECT_TRUE(std::signbit(column.get_element(1))); + EXPECT_DOUBLE_EQ(1.5, column.get_element(2)); + EXPECT_DOUBLE_EQ(-2.25, column.get_element(3)); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadFloatSpecialValues) { + { + std::vector values = {std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity()}; + auto result = read_column(std::make_shared(), + make_fixed_view(DecodedValueKind::FLOAT, values)); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_TRUE(std::isnan(column.get_element(0))); + EXPECT_TRUE(std::isinf(column.get_element(1))); + EXPECT_FALSE(std::signbit(column.get_element(1))); + EXPECT_TRUE(std::isinf(column.get_element(2))); + EXPECT_TRUE(std::signbit(column.get_element(2))); + } + { + std::vector values = {std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity()}; + auto result = read_column(std::make_shared(), + make_fixed_view(DecodedValueKind::DOUBLE, values)); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_TRUE(std::isnan(column.get_element(0))); + EXPECT_TRUE(std::isinf(column.get_element(1))); + EXPECT_FALSE(std::signbit(column.get_element(1))); + EXPECT_TRUE(std::isinf(column.get_element(2))); + EXPECT_TRUE(std::signbit(column.get_element(2))); + } +} + +// ---------------------------------------------------------------------- +// Number SerDe error paths +// ---------------------------------------------------------------------- +// These cases separate unsupported physical kinds from corrupt decoded buffers. Unsupported kinds +// must not append to the destination column; missing value buffers are allowed only for empty or +// all-null batches where no non-null row can dereference the buffer. + +TEST(DataTypeSerDeDecodedValuesTest, NumberRejectsMismatchedKind) { + struct Case { + DataTypePtr type; + DecodedValueKind kind; + }; + std::vector cases = { + {std::make_shared(), DecodedValueKind::INT32}, + {std::make_shared(), DecodedValueKind::BOOL}, + {std::make_shared(), DecodedValueKind::DOUBLE}, + {std::make_shared(), DecodedValueKind::FLOAT}, + {std::make_shared(), DecodedValueKind::BINARY}, + }; + + for (const auto& test_case : cases) { + std::vector values = {1}; + auto result = read_column(test_case.type, make_fixed_view(test_case.kind, values)); + expect_not_supported(result.status); + EXPECT_EQ(0, result.column->size()); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, NumberRejectsMissingValuesWhenNonNullExists) { + auto type = std::make_shared(); + { + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT32; + view.row_count = 3; + auto result = read_column(type, view); + expect_corruption(result.status); + } + { + std::vector null_map = {1, 0, 1}; + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT32; + view.row_count = 3; + view.null_map = null_map.data(); + auto result = read_column(type, view); + expect_corruption(result.status); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, NumberAllowsMissingValuesForAllNullOrEmpty) { + auto type = std::make_shared(std::make_shared()); + { + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT32; + view.row_count = 0; + auto result = read_column(type, view); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(0, result.column->size()); + } + { + std::vector null_map = {1, 1, 1}; + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT32; + view.row_count = 3; + view.null_map = null_map.data(); + auto result = read_column(type, view); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + const auto& nested_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(3, nullable_column.size()); + for (size_t row = 0; row < nullable_column.size(); ++row) { + EXPECT_TRUE(nullable_column.is_null_at(row)); + EXPECT_EQ(0, nested_column.get_element(row)); + } + } +} + +TEST(DataTypeSerDeDecodedValuesTest, NumberRejectsOutOfRangeValueInStrictMode) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {127, 128}; + std::vector null_map = {0, 0}; + auto view = make_fixed_view(DecodedValueKind::INT64, values, &null_map); + view.enable_strict_mode = true; + + auto result = read_column(type, view); + + expect_data_quality_error(result.status); + const auto& nullable_column = assert_cast(*result.column); + EXPECT_EQ(0, nullable_column.size()); + EXPECT_EQ(0, nullable_column.get_null_map_data().size()); + EXPECT_EQ(0, nullable_column.get_nested_column().size()); +} + +TEST(DataTypeSerDeDecodedValuesTest, NumberNullsOutOfRangeValueInNonStrictMode) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {127, 128, -129, -128}; + std::vector null_map = {0, 0, 0, 0}; + auto view = make_fixed_view(DecodedValueKind::INT64, values, &null_map); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + const auto& nested_column = assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(4, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_TRUE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_EQ(127, nested_column.get_element(0)); + EXPECT_EQ(0, nested_column.get_element(1)); + EXPECT_EQ(0, nested_column.get_element(2)); + EXPECT_EQ(-128, nested_column.get_element(3)); +} + +TEST(DataTypeSerDeDecodedValuesTest, NumberRejectsUnsignedOverflowInStrictMode) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {static_cast(std::numeric_limits::max()), + std::numeric_limits::max()}; + std::vector null_map = {0, 0}; + auto view = make_fixed_view(DecodedValueKind::UINT64, values, &null_map); + view.enable_strict_mode = true; + + auto result = read_column(type, view); + + expect_data_quality_error(result.status); +} + +TEST(DataTypeSerDeDecodedValuesTest, NumberNullsUnsignedOverflowInNonStrictMode) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {static_cast(std::numeric_limits::max()), + std::numeric_limits::max()}; + std::vector null_map = {0, 0}; + auto view = make_fixed_view(DecodedValueKind::UINT64, values, &null_map); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + const auto& nested_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(2, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_EQ(std::numeric_limits::max(), nested_column.get_element(0)); + EXPECT_EQ(0, nested_column.get_element(1)); +} + +// ---------------------------------------------------------------------- +// String / Binary SerDe +// ---------------------------------------------------------------------- +// String-like decoded reads must preserve exact byte sequences. The embedded-NUL case prevents +// accidental C-string truncation. Nullable string tests ensure null rows materialize default nested +// values while the outer null map remains authoritative. + +TEST(DataTypeSerDeDecodedValuesTest, ReadStringFromBinary) { + auto type = std::make_shared(); + std::vector storage = {"alpha", "", std::string("a\0b", 3), "utf8-\xe4\xb8\xad"}; + auto refs = string_refs(storage); + + auto result = read_column(type, make_binary_view(DecodedValueKind::BINARY, refs)); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_binary_column(*result.column, storage); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadStringFromFixedBinary) { + auto type = std::make_shared(); + std::vector storage = {std::string("\x00\x01\x02\x03", 4), + std::string("\x7f\x80\xfe\xff", 4)}; + auto refs = string_refs(storage); + + auto result = read_column(type, make_binary_view(DecodedValueKind::FIXED_BINARY, refs, 4)); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_binary_column(*result.column, storage); +} + +TEST(DataTypeSerDeDecodedValuesTest, StringNullMapMaterialization) { + auto type = std::make_shared(std::make_shared()); + std::vector storage = {"alpha", "", "omega"}; + auto refs = string_refs(storage); + std::vector null_map = {0, 1, 0}; + + auto result = + read_column(type, make_binary_view(DecodedValueKind::BINARY, refs, -1, &null_map)); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(3, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + expect_binary_column(nullable_column.get_nested_column(), {"alpha", "", "omega"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, StringRejectsMismatchedKind) { + auto type = std::make_shared(); + for (auto kind : {DecodedValueKind::INT32, DecodedValueKind::INT64, DecodedValueKind::DOUBLE}) { + std::vector values = {1}; + auto result = read_column(type, make_fixed_view(kind, values)); + expect_not_supported(result.status); + EXPECT_EQ(0, result.column->size()); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, StringRejectsMissingBinaryValuesWhenNonNullExists) { + auto type = std::make_shared(); + DecodedColumnView view; + view.value_kind = DecodedValueKind::BINARY; + view.row_count = 1; + + auto result = read_column(type, view); + + expect_corruption(result.status); +} + +TEST(DataTypeSerDeDecodedValuesTest, StringAllowsAllNullWithoutBinaryValues) { + auto type = std::make_shared(std::make_shared()); + std::vector null_map = {1, 1}; + DecodedColumnView view; + view.value_kind = DecodedValueKind::BINARY; + view.row_count = 2; + view.null_map = null_map.data(); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(2, nullable_column.size()); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + expect_binary_column(nullable_column.get_nested_column(), {"", ""}); +} + +// ---------------------------------------------------------------------- +// DateV2 SerDe +// ---------------------------------------------------------------------- +// DateV2 accepts Parquet DATE-style epoch days as INT32. Null rows insert default nested dates and +// missing buffers are rejected only when a non-null row requires a value. + +TEST(DataTypeSerDeDecodedValuesTest, ReadDateV2FromEpochDays) { + auto type = std::make_shared(); + std::vector values = {-1, 0, 1, 18628, 18321}; + + auto result = read_column(type, make_fixed_view(DecodedValueKind::INT32, values)); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, + {"1969-12-31", "1970-01-01", "1970-01-02", "2021-01-01", "2020-02-29"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, DateV2HandlesNulls) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {0, 1, 2}; + std::vector null_map = {0, 1, 0}; + + auto result = read_column(type, make_fixed_view(DecodedValueKind::INT32, values, &null_map)); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(3, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + expect_column_strings(*type, *result.column, {"1970-01-01", "NULL", "1970-01-03"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, DateV2RejectsInvalidKind) { + auto type = std::make_shared(); + for (auto kind : + {DecodedValueKind::INT64, DecodedValueKind::BINARY, DecodedValueKind::DOUBLE}) { + std::vector values = {0}; + auto result = read_column(type, make_fixed_view(kind, values)); + expect_not_supported(result.status); + EXPECT_EQ(0, result.column->size()); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, DateV2RejectsMissingValuesWhenNonNullExists) { + auto type = std::make_shared(); + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT32; + view.row_count = 1; + + auto result = read_column(type, view); + + expect_corruption(result.status); +} + +// ---------------------------------------------------------------------- +// DateTimeV2 SerDe +// ---------------------------------------------------------------------- +// Timestamp decoding covers INT64 micros/millis, UNKNOWN-as-micros compatibility, UTC-adjusted +// conversion with explicit/default timezones, INT96 Julian-day timestamps, and invalid buffer/kind +// errors. Negative epoch values are included to lock correct floor-division behavior. + +TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2Micros) { + auto type = std::make_shared(6); + std::vector values = {-1, 0, 1, 1234567, 86400000000LL - 1}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MICROS; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, + {"1969-12-31 23:59:59.999999", "1970-01-01 00:00:00.000000", + "1970-01-01 00:00:00.000001", "1970-01-01 00:00:01.234567", + "1970-01-01 23:59:59.999999"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2Millis) { + auto type = std::make_shared(6); + std::vector values = {-1, 0, 1, 1234}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MILLIS; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, + {"1969-12-31 23:59:59.999000", "1970-01-01 00:00:00.000000", + "1970-01-01 00:00:00.001000", "1970-01-01 00:00:01.234000"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2Nanos) { + auto type = std::make_shared(6); + std::vector values = {-1000, 0, 1000, 1234567890}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::NANOS; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, + {"1969-12-31 23:59:59.999999", "1970-01-01 00:00:00.000000", + "1970-01-01 00:00:00.000001", "1970-01-01 00:00:01.234567"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2UnknownUnitAsMicros) { + auto type = std::make_shared(6); + std::vector values = {1000000}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::UNKNOWN; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, {"1970-01-01 00:00:01.000000"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2UtcAdjustedDefaultUtc) { + auto type = std::make_shared(6); + std::vector values = {0}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MICROS; + view.timestamp_is_adjusted_to_utc = true; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, {"1970-01-01 00:00:00.000000"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2UtcAdjustedWithTimezones) { + auto type = std::make_shared(6); + std::vector values = {0, -1, 1234567}; + cctz::time_zone shanghai; + cctz::time_zone new_york; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("+08:00", shanghai)); + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("-05:00", new_york)); + + auto shanghai_view = make_fixed_view(DecodedValueKind::INT64, values); + shanghai_view.time_unit = DecodedTimeUnit::MICROS; + shanghai_view.timestamp_is_adjusted_to_utc = true; + shanghai_view.timezone = &shanghai; + auto shanghai_result = read_column(type, shanghai_view); + ASSERT_TRUE(shanghai_result.status.ok()) << shanghai_result.status; + expect_column_strings(*type, *shanghai_result.column, + {"1970-01-01 08:00:00.000000", "1970-01-01 07:59:59.999999", + "1970-01-01 08:00:01.234567"}); + + auto new_york_view = make_fixed_view(DecodedValueKind::INT64, values); + new_york_view.time_unit = DecodedTimeUnit::MICROS; + new_york_view.timestamp_is_adjusted_to_utc = true; + new_york_view.timezone = &new_york; + auto new_york_result = read_column(type, new_york_view); + ASSERT_TRUE(new_york_result.status.ok()) << new_york_result.status; + expect_column_strings(*type, *new_york_result.column, + {"1969-12-31 19:00:00.000000", "1969-12-31 18:59:59.999999", + "1969-12-31 19:00:01.234567"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2Int96) { + auto type = std::make_shared(std::make_shared(6)); + std::vector values = { + {0, 2440588}, + {86399999999000LL, 2440587}, + {0, 2440589}, + }; + std::vector null_map = {0, 0, 1}; + auto view = make_fixed_view(DecodedValueKind::INT96, values, &null_map); + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("+08:00", shanghai)); + view.timezone = &shanghai; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, + {"1970-01-01 08:00:00.000000", "1970-01-01 07:59:59.999999", "NULL"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadTimestampTzInt64AsUtcInstant) { + auto type = std::make_shared(6); + // 2024-12-31 16:00:00 UTC is displayed as 2025-01-01 00:00:00+08:00. + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("+08:00", shanghai)); + + std::vector micros_values = {1735660800000000LL, 1735660800123456LL}; + auto micros_view = make_fixed_view(DecodedValueKind::INT64, micros_values); + micros_view.time_unit = DecodedTimeUnit::MICROS; + auto micros_result = read_column(type, micros_view); + ASSERT_TRUE(micros_result.status.ok()) << micros_result.status; + const auto& micros_column = assert_cast(*micros_result.column); + EXPECT_EQ(micros_column.get_element(0).to_string(shanghai, 6), + "2025-01-01 00:00:00.000000+08:00"); + EXPECT_EQ(micros_column.get_element(1).to_string(shanghai, 6), + "2025-01-01 00:00:00.123456+08:00"); + + std::vector millis_values = {1735660800000LL}; + auto millis_view = make_fixed_view(DecodedValueKind::INT64, millis_values); + millis_view.time_unit = DecodedTimeUnit::MILLIS; + auto millis_result = read_column(type, millis_view); + ASSERT_TRUE(millis_result.status.ok()) << millis_result.status; + const auto& millis_column = assert_cast(*millis_result.column); + EXPECT_EQ(millis_column.get_element(0).to_string(shanghai, 6), + "2025-01-01 00:00:00.000000+08:00"); + + std::vector nanos_values = {1735660800123456000LL}; + auto nanos_view = make_fixed_view(DecodedValueKind::INT64, nanos_values); + nanos_view.time_unit = DecodedTimeUnit::NANOS; + auto nanos_result = read_column(type, nanos_view); + ASSERT_TRUE(nanos_result.status.ok()) << nanos_result.status; + const auto& nanos_column = assert_cast(*nanos_result.column); + EXPECT_EQ(nanos_column.get_element(0).to_string(shanghai, 6), + "2025-01-01 00:00:00.123456+08:00"); +} + +TEST(DataTypeSerDeDecodedValuesTest, TimestampTzReadsInt96AsUtcInstant) { + auto type = std::make_shared(6); + std::vector values = {{0, 2440588}, {123456789000LL, 2440588}}; + auto view = make_fixed_view(DecodedValueKind::INT96, values); + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("+08:00", shanghai)); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(column.get_element(0).to_string(shanghai, 6), "1970-01-01 08:00:00.000000+08:00"); + EXPECT_EQ(column.get_element(1).to_string(shanghai, 6), "1970-01-01 08:02:03.456789+08:00"); +} + +TEST(DataTypeSerDeDecodedValuesTest, DateTimeV2RejectsInvalidKind) { + auto type = std::make_shared(6); + for (auto kind : + {DecodedValueKind::INT32, DecodedValueKind::BINARY, DecodedValueKind::DOUBLE}) { + std::vector values = {0}; + auto result = read_column(type, make_fixed_view(kind, values)); + expect_not_supported(result.status); + EXPECT_EQ(0, result.column->size()); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, DateTimeV2RejectsMissingValuesWhenNonNullExists) { + auto type = std::make_shared(6); + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.row_count = 1; + + auto result = read_column(type, view); + + expect_corruption(result.status); +} + +TEST(DataTypeSerDeDecodedValuesTest, DateTimeV2RejectsOutOfRangeEpochWithoutAbort) { + auto type = std::make_shared(6); + std::vector values = {0, -377673580800000001LL}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MICROS; + + auto result = read_column(type, view); + + expect_data_quality_error(result.status); + EXPECT_EQ(0, result.column->size()); +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableDateTimeV2RejectsOutOfRangeEpochInStrictMode) { + auto type = std::make_shared(std::make_shared(6)); + std::vector values = {0, -377673580800000001LL}; + std::vector null_map = {0, 0}; + auto view = make_fixed_view(DecodedValueKind::INT64, values, &null_map); + view.time_unit = DecodedTimeUnit::MICROS; + view.enable_strict_mode = true; + + auto result = read_column(type, view); + + expect_data_quality_error(result.status); + const auto& nullable_column = assert_cast(*result.column); + EXPECT_EQ(0, nullable_column.size()); + EXPECT_EQ(0, nullable_column.get_null_map_data().size()); + EXPECT_EQ(0, nullable_column.get_nested_column().size()); +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableDateTimeV2NullsOutOfRangeEpochInNonStrictMode) { + auto type = std::make_shared(std::make_shared(6)); + std::vector values = {0, -377673580800000001LL, 1}; + std::vector null_map = {0, 0, 0}; + auto view = make_fixed_view(DecodedValueKind::INT64, values, &null_map); + view.time_unit = DecodedTimeUnit::MICROS; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(3, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + expect_column_strings(*type, *result.column, + {"1970-01-01 00:00:00.000000", "NULL", "1970-01-01 00:00:00.000001"}); +} + +// ---------------------------------------------------------------------- +// TimeV2 SerDe +// ---------------------------------------------------------------------- +// TimeV2 decodes INT32 as milliseconds and INT64 according to the supplied time unit. Negative +// durations are verified because they use a sign bit in TimeValue::TimeType rather than DateTimeV2 +// epoch arithmetic. + +TEST(DataTypeSerDeDecodedValuesTest, ReadTimeV2FromInt32Millis) { + auto type = std::make_shared(6); + std::vector values = {0, 1, -1, 3661001}; + + auto result = read_column(type, make_fixed_view(DecodedValueKind::INT32, values)); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings( + *type, *result.column, + {"00:00:00.000000", "00:00:00.001000", "-00:00:00.001000", "01:01:01.001000"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadTimeV2FromInt64Micros) { + auto type = std::make_shared(6); + std::vector values = {0, 1, -1, 3661000001LL}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MICROS; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings( + *type, *result.column, + {"00:00:00.000000", "00:00:00.000001", "-00:00:00.000001", "01:01:01.000001"}); + + view.time_unit = DecodedTimeUnit::UNKNOWN; + auto unknown_result = read_column(type, view); + ASSERT_TRUE(unknown_result.status.ok()) << unknown_result.status; + expect_column_strings( + *type, *unknown_result.column, + {"00:00:00.000000", "00:00:00.000001", "-00:00:00.000001", "01:01:01.000001"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadTimeV2FromInt64Millis) { + auto type = std::make_shared(6); + std::vector values = {1, -1, 3661001}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MILLIS; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, + {"00:00:00.001000", "-00:00:00.001000", "01:01:01.001000"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadTimeV2FromInt64Nanos) { + auto type = std::make_shared(6); + std::vector values = {1000, -1000, 3661000001000LL}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::NANOS; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, + {"00:00:00.000001", "-00:00:00.000001", "01:01:01.000001"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, TimeV2HandlesNulls) { + auto type = std::make_shared(std::make_shared(6)); + std::vector values = {0, 1, 2}; + std::vector null_map = {0, 1, 0}; + auto view = make_fixed_view(DecodedValueKind::INT64, values, &null_map); + view.time_unit = DecodedTimeUnit::MICROS; + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(3, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + expect_column_strings(*type, *result.column, {"00:00:00.000000", "NULL", "00:00:00.000002"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, TimeV2RejectsInvalidKind) { + auto type = std::make_shared(6); + for (auto kind : {DecodedValueKind::BOOL, DecodedValueKind::BINARY, DecodedValueKind::DOUBLE}) { + std::vector values = {0}; + auto result = read_column(type, make_fixed_view(kind, values)); + expect_not_supported(result.status); + EXPECT_EQ(0, result.column->size()); + } +} + +// ---------------------------------------------------------------------- +// Decimal SerDe +// ---------------------------------------------------------------------- +// Decimal cases cover integer-backed values and Parquet big-endian two's-complement binary values. +// String assertions validate the user-visible scale, while direct column checks lock the native +// unscaled value for every decimal width. + +TEST(DataTypeSerDeDecodedValuesTest, ReadDecimal32FromInt32) { + auto type = std::make_shared(9, 2); + std::vector values = {12345, -67, 0}; + auto view = make_fixed_view(DecodedValueKind::INT32, values); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(Decimal32(12345), column.get_element(0)); + EXPECT_EQ(Decimal32(-67), column.get_element(1)); + EXPECT_EQ(Decimal32(0), column.get_element(2)); + expect_column_strings(*type, *result.column, {"123.45", "-0.67", "0.00"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDecimal64FromInt64) { + auto type = std::make_shared(18, 4); + std::vector values = {123456789, -1}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(Decimal64(123456789), column.get_element(0)); + EXPECT_EQ(Decimal64(-1), column.get_element(1)); + expect_column_strings(*type, *result.column, {"12345.6789", "-0.0001"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDecimal128FromInt32AndInt64) { + auto type = std::make_shared(38, 6); + { + std::vector values = {123456, -1}; + auto result = read_column(type, make_fixed_view(DecodedValueKind::INT32, values)); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(decimal128_v3(123456), column.get_element(0)); + EXPECT_EQ(decimal128_v3(-1), column.get_element(1)); + expect_column_strings(*type, *result.column, {"0.123456", "-0.000001"}); + } + { + std::vector values = {1234567890123LL, -1234567LL}; + auto result = read_column(type, make_fixed_view(DecodedValueKind::INT64, values)); + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(decimal128_v3(1234567890123LL), column.get_element(0)); + EXPECT_EQ(decimal128_v3(-1234567LL), column.get_element(1)); + expect_column_strings(*type, *result.column, {"1234567.890123", "-1.234567"}); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDecimal256FromInt64) { + auto type = std::make_shared(76, 8); + std::vector values = {std::numeric_limits::max(), + std::numeric_limits::min()}; + auto result = read_column(type, make_fixed_view(DecodedValueKind::INT64, values)); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + EXPECT_EQ(decimal256_from_int64(std::numeric_limits::max()), column.get_element(0)); + EXPECT_EQ(decimal256_from_int64(std::numeric_limits::min()), column.get_element(1)); + expect_column_strings(*type, *result.column, {"92233720368.54775807", "-92233720368.54775808"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDecimalFromBinaryBigEndian) { + auto type = std::make_shared(18, 2); + std::vector storage = { + std::string("\x00", 1), std::string("\x7f", 1), std::string("\x80", 1), + std::string("\xff", 1), std::string("\xff\xbd", 2), std::string("\x30\x39", 2), + }; + auto refs = string_refs(storage); + + auto result = read_column(type, make_binary_view(DecodedValueKind::BINARY, refs)); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& column = assert_cast(*result.column); + std::vector expected = {decimal128_v3(0), decimal128_v3(127), + decimal128_v3(-128), decimal128_v3(-1), + decimal128_v3(-67), decimal128_v3(12345)}; + ASSERT_EQ(expected.size(), column.size()); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(expected[row], column.get_element(row)) << "row=" << row; + } + expect_column_strings(*type, *result.column, + {"0.00", "1.27", "-1.28", "-0.01", "-0.67", "123.45"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadDecimalFromFixedBinaryLengths) { + { + auto type = std::make_shared(38, 2); + std::vector storage = {std::string("\x00", 1), std::string("\x80", 1)}; + auto refs = string_refs(storage); + auto result = read_column(type, make_binary_view(DecodedValueKind::FIXED_BINARY, refs, 1)); + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, {"0.00", "-1.28"}); + } + { + auto type = std::make_shared(38, 2); + std::vector storage = {std::string("\xff\xbd", 2), std::string("\x30\x39", 2)}; + auto refs = string_refs(storage); + auto result = read_column(type, make_binary_view(DecodedValueKind::FIXED_BINARY, refs, 2)); + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, {"-0.67", "123.45"}); + } + { + auto type = std::make_shared(38, 2); + std::vector storage = {std::string("\0\0\0\0\0\0\x30\x39", 8)}; + auto refs = string_refs(storage); + auto result = read_column(type, make_binary_view(DecodedValueKind::FIXED_BINARY, refs, 8)); + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, {"123.45"}); + } + { + auto type = std::make_shared(38, 2); + std::vector storage = { + std::string("\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xbd", + 16)}; + auto refs = string_refs(storage); + auto result = read_column(type, make_binary_view(DecodedValueKind::FIXED_BINARY, refs, 16)); + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, {"-0.67"}); + } + { + auto type = std::make_shared(76, 2); + std::vector storage = {std::string(31, '\xff') + std::string("\xbd", 1)}; + auto refs = string_refs(storage); + auto result = read_column(type, make_binary_view(DecodedValueKind::FIXED_BINARY, refs, 32)); + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, {"-0.67"}); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, DecimalHandlesNulls) { + auto type = std::make_shared(std::make_shared(18, 2)); + std::vector values = {12345, -1, -67}; + std::vector null_map = {0, 1, 0}; + + auto result = read_column(type, make_fixed_view(DecodedValueKind::INT64, values, &null_map)); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + const auto& decimal_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(3, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_EQ(decimal128_v3(12345), decimal_column.get_element(0)); + EXPECT_EQ(decimal128_v3(0), decimal_column.get_element(1)); + EXPECT_EQ(decimal128_v3(-67), decimal_column.get_element(2)); +} + +TEST(DataTypeSerDeDecodedValuesTest, DecimalRejectsOutOfRangeValueInStrictMode) { + auto type = std::make_shared(std::make_shared(9, 2)); + std::vector values = {999999999, 1000000000}; + std::vector null_map = {0, 0}; + auto view = make_fixed_view(DecodedValueKind::INT64, values, &null_map); + view.enable_strict_mode = true; + + auto result = read_column(type, view); + + expect_data_quality_error(result.status); + const auto& nullable_column = assert_cast(*result.column); + EXPECT_EQ(0, nullable_column.size()); + EXPECT_EQ(0, nullable_column.get_null_map_data().size()); + EXPECT_EQ(0, nullable_column.get_nested_column().size()); +} + +TEST(DataTypeSerDeDecodedValuesTest, DecimalNullsOutOfRangeValueInNonStrictMode) { + auto type = std::make_shared(std::make_shared(9, 2)); + std::vector values = {999999999, 1000000000, -1000000000, -999999999}; + std::vector null_map = {0, 0, 0, 0}; + auto view = make_fixed_view(DecodedValueKind::INT64, values, &null_map); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + const auto& decimal_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(4, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_TRUE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_EQ(Decimal32(999999999), decimal_column.get_element(0)); + EXPECT_EQ(Decimal32(0), decimal_column.get_element(1)); + EXPECT_EQ(Decimal32(0), decimal_column.get_element(2)); + EXPECT_EQ(Decimal32(-999999999), decimal_column.get_element(3)); +} + +TEST(DataTypeSerDeDecodedValuesTest, DecimalRejectsNullBinaryDataWithPositiveLength) { + auto type = std::make_shared(18, 2); + std::vector refs = {StringRef(static_cast(nullptr), 2)}; + + auto result = read_column(type, make_binary_view(DecodedValueKind::BINARY, refs)); + + expect_corruption(result.status); + EXPECT_NE(std::string::npos, result.status.to_string().find("row 0")); +} + +TEST(DataTypeSerDeDecodedValuesTest, DecimalAllowsZeroLengthBinaryAsZero) { + auto type = std::make_shared(18, 2); + std::vector refs = {StringRef(static_cast(nullptr), 0), + StringRef("", 0)}; + + auto result = read_column(type, make_binary_view(DecodedValueKind::BINARY, refs)); + + ASSERT_TRUE(result.status.ok()) << result.status; + expect_column_strings(*type, *result.column, {"0.00", "0.00"}); +} + +TEST(DataTypeSerDeDecodedValuesTest, DecimalRejectsInvalidKind) { + auto type = std::make_shared(18, 2); + for (auto kind : {DecodedValueKind::BOOL, DecodedValueKind::FLOAT, DecodedValueKind::DOUBLE, + DecodedValueKind::UINT64}) { + std::vector values = {0}; + auto result = read_column(type, make_fixed_view(kind, values)); + expect_not_supported(result.status); + EXPECT_EQ(0, result.column->size()); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, DecimalRejectsMissingBufferWhenNonNullExists) { + auto type = std::make_shared(18, 2); + { + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.row_count = 1; + auto result = read_column(type, view); + expect_corruption(result.status); + } + { + DecodedColumnView view; + view.value_kind = DecodedValueKind::BINARY; + view.row_count = 1; + auto result = read_column(type, view); + expect_corruption(result.status); + } +} + +// ---------------------------------------------------------------------- +// Nullable SerDe wrapper +// ---------------------------------------------------------------------- +// Nullable tests focus on wrapper responsibilities: copying the outer null map, inserting default +// nested values for null rows, treating a missing null_map as all non-null, appending to existing +// columns, and rolling back outer state when the nested reader rejects the input. + +TEST(DataTypeSerDeDecodedValuesTest, NullablePropagatesNullMapAndReadsNested) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {10, 20, 30, 40}; + std::vector null_map = {0, 1, 0, 1}; + + auto result = read_column(type, make_fixed_view(DecodedValueKind::INT32, values, &null_map)); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + const auto& nested_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(4, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_TRUE(nullable_column.is_null_at(3)); + EXPECT_EQ(10, nested_column.get_element(0)); + EXPECT_EQ(0, nested_column.get_element(1)); + EXPECT_EQ(30, nested_column.get_element(2)); + EXPECT_EQ(0, nested_column.get_element(3)); +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableWithoutNullMapReadsAllNonNull) { + auto type = std::make_shared(std::make_shared()); + std::vector storage = {"alpha", "beta"}; + auto refs = string_refs(storage); + + auto result = read_column(type, make_binary_view(DecodedValueKind::BINARY, refs)); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(2, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + expect_binary_column(nullable_column.get_nested_column(), storage); +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableAllNullDoesNotRequireNestedBuffer) { + std::vector null_map = {1, 1}; + std::vector types = { + std::make_shared(std::make_shared()), + std::make_shared(std::make_shared(18, 2)), + std::make_shared(std::make_shared()), + std::make_shared(std::make_shared()), + }; + + for (const auto& type : types) { + DecodedColumnView view; + view.value_kind = type->get_name().find("String") != std::string::npos + ? DecodedValueKind::BINARY + : DecodedValueKind::INT32; + view.row_count = 2; + view.null_map = null_map.data(); + auto result = read_column(type, view); + ASSERT_TRUE(result.status.ok()) << result.status << ", type=" << type->get_name(); + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(2, nullable_column.size()); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_EQ(2, nullable_column.get_nested_column().size()); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableAppendToExistingColumn) { + auto type = std::make_shared(std::make_shared()); + auto column = type->create_column(); + + std::vector first_values = {1, 2}; + auto first_status = type->get_serde()->read_column_from_decoded_values( + *column, make_fixed_view(DecodedValueKind::INT32, first_values)); + ASSERT_TRUE(first_status.ok()) << first_status; + + std::vector second_values = {10, 20, 30}; + std::vector second_null_map = {0, 1, 0}; + auto second_status = type->get_serde()->read_column_from_decoded_values( + *column, make_fixed_view(DecodedValueKind::INT32, second_values, &second_null_map)); + ASSERT_TRUE(second_status.ok()) << second_status; + + const auto& nullable_column = assert_cast(*column); + const auto& nested_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(5, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_TRUE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + EXPECT_EQ(1, nested_column.get_element(0)); + EXPECT_EQ(2, nested_column.get_element(1)); + EXPECT_EQ(10, nested_column.get_element(2)); + EXPECT_EQ(0, nested_column.get_element(3)); + EXPECT_EQ(30, nested_column.get_element(4)); +} + +TEST(DataTypeSerDeDecodedValuesTest, NullablePropagatesNestedError) { + auto type = std::make_shared(std::make_shared()); + auto column = type->create_column(); + std::vector values = {1.0}; + std::vector null_map = {0}; + auto view = make_fixed_view(DecodedValueKind::DOUBLE, values, &null_map); + view.enable_strict_mode = true; + + auto status = type->get_serde()->read_column_from_decoded_values(*column, view); + + expect_not_supported(status); + const auto& nullable_column = assert_cast(*column); + EXPECT_EQ(0, nullable_column.size()); + EXPECT_EQ(0, nullable_column.get_null_map_data().size()); + EXPECT_EQ(0, nullable_column.get_nested_column().size()); +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableNonStrictModeNullsUnsupportedDecodedKindForAllTypes) { + struct Case { + DataTypePtr type; + DecodedValueKind kind; + }; + std::vector cases = { + {std::make_shared(std::make_shared()), + DecodedValueKind::INT32}, + {std::make_shared(std::make_shared()), + DecodedValueKind::DOUBLE}, + {std::make_shared(std::make_shared()), + DecodedValueKind::FLOAT}, + {std::make_shared(std::make_shared()), + DecodedValueKind::INT64}, + {std::make_shared(std::make_shared()), + DecodedValueKind::INT64}, + {std::make_shared(std::make_shared(6)), + DecodedValueKind::DOUBLE}, + {std::make_shared(std::make_shared(6)), + DecodedValueKind::DOUBLE}, + {std::make_shared(std::make_shared(18, 2)), + DecodedValueKind::DOUBLE}, + }; + + std::vector values = {1, 2}; + for (const auto& test_case : cases) { + auto view = make_fixed_view(test_case.kind, values); + + auto result = read_column(test_case.type, view); + + ASSERT_TRUE(result.status.ok()) << result.status << ", type=" << test_case.type->get_name(); + expect_nullable_all_null(*result.column, values.size()); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableStrictModeRejectsUnsupportedDecodedKind) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {1.0}; + std::vector null_map = {0}; + auto view = make_fixed_view(DecodedValueKind::DOUBLE, values, &null_map); + view.enable_strict_mode = true; + + auto result = read_column(type, view); + + expect_not_supported(result.status); + const auto& nullable_column = assert_cast(*result.column); + EXPECT_EQ(0, nullable_column.size()); + EXPECT_EQ(0, nullable_column.get_null_map_data().size()); + EXPECT_EQ(0, nullable_column.get_nested_column().size()); +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableNonStrictModeNullsRowLevelDecodedConversionFailure) { + { + auto type = std::make_shared(std::make_shared()); + std::vector refs = {StringRef("ok", 2), + StringRef(static_cast(nullptr), 2), + StringRef("", 0)}; + auto view = make_binary_view(DecodedValueKind::BINARY, refs); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(3, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + expect_binary_column(nullable_column.get_nested_column(), {"ok", "", ""}); + } + { + auto type = std::make_shared(std::make_shared(18, 2)); + std::vector refs = {StringRef("\x30\x39", 2), + StringRef(static_cast(nullptr), 2)}; + auto view = make_binary_view(DecodedValueKind::BINARY, refs); + + auto result = read_column(type, view); + + ASSERT_TRUE(result.status.ok()) << result.status; + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(2, nullable_column.size()); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + expect_column_strings(*type, *result.column, {"123.45", "NULL"}); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, NullableStrictModeRejectsRowLevelDecodedConversionFailure) { + auto type = std::make_shared(std::make_shared()); + std::vector refs = {StringRef("ok", 2), + StringRef(static_cast(nullptr), 2)}; + auto view = make_binary_view(DecodedValueKind::BINARY, refs); + view.enable_strict_mode = true; + + auto result = read_column(type, view); + + expect_corruption(result.status); + const auto& nullable_column = assert_cast(*result.column); + EXPECT_EQ(0, nullable_column.size()); + EXPECT_EQ(0, nullable_column.get_null_map_data().size()); + EXPECT_EQ(0, nullable_column.get_nested_column().size()); +} + +// ---------------------------------------------------------------------- +// read_field_from_decoded_value +// ---------------------------------------------------------------------- +// The field path is used by Parquet min/max and pruning code. It must be covered independently +// because it creates a one-row column, delegates to the batch reader, and extracts a Field value. + +TEST(DataTypeSerDeDecodedValuesTest, ReadFieldPrimitiveValues) { + { + std::vector values = {true}; + auto field = read_field(std::make_shared(), make_bool_view(values)); + EXPECT_EQ(TYPE_BOOLEAN, field.get_type()); + EXPECT_TRUE(field.get()); + } + { + std::vector values = {-42}; + auto field = read_field(std::make_shared(), + make_fixed_view(DecodedValueKind::INT32, values)); + EXPECT_EQ(TYPE_INT, field.get_type()); + EXPECT_EQ(-42, field.get()); + } + { + std::vector values = {1234567890123LL}; + auto field = read_field(std::make_shared(), + make_fixed_view(DecodedValueKind::INT64, values)); + EXPECT_EQ(TYPE_BIGINT, field.get_type()); + EXPECT_EQ(1234567890123LL, field.get()); + } + { + std::vector values = {-9}; + auto field = read_field(std::make_shared(), + make_fixed_view(DecodedValueKind::INT64, values)); + EXPECT_EQ(TYPE_LARGEINT, field.get_type()); + EXPECT_EQ(static_cast<__int128_t>(-9), field.get()); + } + { + std::vector values = {std::numeric_limits::quiet_NaN()}; + auto field = read_field(std::make_shared(), + make_fixed_view(DecodedValueKind::FLOAT, values)); + EXPECT_EQ(TYPE_FLOAT, field.get_type()); + EXPECT_TRUE(std::isnan(field.get())); + } + { + std::vector values = {std::numeric_limits::infinity()}; + auto field = read_field(std::make_shared(), + make_fixed_view(DecodedValueKind::DOUBLE, values)); + EXPECT_EQ(TYPE_DOUBLE, field.get_type()); + EXPECT_TRUE(std::isinf(field.get())); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadFieldLogicalIntegerCastsPhysicalValue) { + { + std::vector values = {32767}; + auto view = + with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); + auto field = read_field(std::make_shared(), view); + EXPECT_EQ(TYPE_SMALLINT, field.get_type()); + EXPECT_EQ(255, field.get()); + } + { + std::vector values = {-1}; + auto view = + with_logical_integer(make_fixed_view(DecodedValueKind::UINT32, values), 32, false); + auto field = read_field(std::make_shared(), view); + EXPECT_EQ(TYPE_BIGINT, field.get_type()); + EXPECT_EQ(4294967295LL, field.get()); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadFieldStringValues) { + auto type = std::make_shared(); + std::vector storage = {std::string("a\0b", 3)}; + auto refs = string_refs(storage); + auto field = read_field(type, make_binary_view(DecodedValueKind::BINARY, refs)); + EXPECT_EQ(TYPE_STRING, field.get_type()); + EXPECT_EQ(std::string("a\0b", 3), field.get()); + + std::vector fixed_storage = {std::string("\x00\x01\x02\x03", 4)}; + auto fixed_refs = string_refs(fixed_storage); + auto fixed_field = + read_field(type, make_binary_view(DecodedValueKind::FIXED_BINARY, fixed_refs, 4)); + EXPECT_EQ(TYPE_STRING, fixed_field.get_type()); + EXPECT_EQ(std::string("\x00\x01\x02\x03", 4), fixed_field.get()); +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadFieldDateTimeAndTimeValues) { + { + auto type = std::make_shared(); + std::vector values = {18628}; + auto field = read_field(type, make_fixed_view(DecodedValueKind::INT32, values)); + EXPECT_EQ(TYPE_DATEV2, field.get_type()); + EXPECT_EQ("2021-01-01", field.to_debug_string(0)); + } + { + auto type = std::make_shared(6); + std::vector values = {1234567}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MICROS; + auto field = read_field(type, view); + EXPECT_EQ(TYPE_DATETIMEV2, field.get_type()); + EXPECT_EQ("1970-01-01 00:00:01.234567", field.to_debug_string(6)); + } + { + auto type = std::make_shared(6); + std::vector values = {1234}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MILLIS; + auto field = read_field(type, view); + EXPECT_EQ(TYPE_DATETIMEV2, field.get_type()); + EXPECT_EQ("1970-01-01 00:00:01.234000", field.to_debug_string(6)); + } + { + auto type = std::make_shared(6); + std::vector values = {{0, 2440588}}; + auto field = read_field(type, make_fixed_view(DecodedValueKind::INT96, values)); + EXPECT_EQ(TYPE_DATETIMEV2, field.get_type()); + EXPECT_EQ("1970-01-01 00:00:00.000000", field.to_debug_string(6)); + } + { + auto type = std::make_shared(6); + std::vector values = {3661000001LL}; + auto view = make_fixed_view(DecodedValueKind::INT64, values); + view.time_unit = DecodedTimeUnit::MICROS; + auto field = read_field(type, view); + EXPECT_EQ(TYPE_TIMEV2, field.get_type()); + auto column = type->create_column(); + column->insert(field); + expect_column_strings(*type, *column, {"01:01:01.000001"}); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadFieldDecimalValues) { + { + auto type = std::make_shared(9, 2); + std::vector values = {12345}; + auto field = read_field(type, make_fixed_view(DecodedValueKind::INT32, values)); + EXPECT_EQ(TYPE_DECIMAL32, field.get_type()); + EXPECT_EQ("123.45", field.to_debug_string(2)); + } + { + auto type = std::make_shared(18, 4); + std::vector values = {-1}; + auto field = read_field(type, make_fixed_view(DecodedValueKind::INT64, values)); + EXPECT_EQ(TYPE_DECIMAL64, field.get_type()); + EXPECT_EQ("-0.0001", field.to_debug_string(4)); + } + { + auto type = std::make_shared(38, 2); + std::vector storage = {std::string("\x30\x39", 2)}; + auto refs = string_refs(storage); + auto field = read_field(type, make_binary_view(DecodedValueKind::BINARY, refs)); + EXPECT_EQ(TYPE_DECIMAL128I, field.get_type()); + EXPECT_EQ("123.45", field.to_debug_string(2)); + } + { + auto type = std::make_shared(76, 2); + std::vector storage = {std::string(31, '\xff') + std::string("\xbd", 1)}; + auto refs = string_refs(storage); + auto field = read_field(type, make_binary_view(DecodedValueKind::FIXED_BINARY, refs, 32)); + EXPECT_EQ(TYPE_DECIMAL256, field.get_type()); + EXPECT_EQ("-0.67", field.to_debug_string(2)); + } +} + +TEST(DataTypeSerDeDecodedValuesTest, ReadFieldPropagatesUnsupportedKind) { + { + auto type = std::make_shared(); + std::vector values = {1}; + expect_not_supported( + read_field_status(type, make_fixed_view(DecodedValueKind::INT32, values))); + } + { + auto type = std::make_shared(); + std::vector values = {1.0}; + expect_not_supported( + read_field_status(type, make_fixed_view(DecodedValueKind::DOUBLE, values))); + } + { + auto type = std::make_shared(); + std::vector values = {0}; + expect_not_supported( + read_field_status(type, make_fixed_view(DecodedValueKind::INT64, values))); + } +} + +TEST(DataTypeSerDeDecodedValuesDeathTest, ReadFieldRejectsInvalidRowCountDeathTest) { + auto type = std::make_shared(); + std::vector values = {1, 2}; + Field field; + + auto zero_row_view = make_fixed_view(DecodedValueKind::INT32, values); + zero_row_view.row_count = 0; + EXPECT_DEATH( + { + auto status = type->get_serde()->read_field_from_decoded_value(*type, &field, + zero_row_view); + (void)status; + }, + "view.row_count == 1"); + + auto two_row_view = make_fixed_view(DecodedValueKind::INT32, values); + two_row_view.row_count = 2; + EXPECT_DEATH( + { + auto status = type->get_serde()->read_field_from_decoded_value(*type, &field, + two_row_view); + (void)status; + }, + "view.row_count == 1"); +} + +TEST(DataTypeSerDeDecodedValuesDeathTest, ReadFieldRejectsNullFieldPointerDeathTest) { + auto type = std::make_shared(); + std::vector values = {1}; + auto view = make_fixed_view(DecodedValueKind::INT32, values); + + EXPECT_DEATH( + { + auto status = + type->get_serde()->read_field_from_decoded_value(*type, nullptr, view); + (void)status; + }, + "field != nullptr"); +} + +// ---------------------------------------------------------------------- +// Illegal kind matrix +// ---------------------------------------------------------------------- +// This compact matrix complements the focused error tests above by ensuring each decoded-aware +// family rejects representative illegal physical kinds without mutating an empty destination. + +TEST(DataTypeSerDeDecodedValuesTest, IllegalKindMatrixRejectsUnsupportedCombinations) { + struct Case { + DataTypePtr type; + std::vector illegal_kinds; + }; + std::vector cases = { + {std::make_shared(), {DecodedValueKind::INT32, DecodedValueKind::BINARY}}, + {std::make_shared(), + {DecodedValueKind::BOOL, DecodedValueKind::FLOAT, DecodedValueKind::DOUBLE, + DecodedValueKind::BINARY}}, + {std::make_shared(), + {DecodedValueKind::DOUBLE, DecodedValueKind::INT32}}, + {std::make_shared(), + {DecodedValueKind::FLOAT, DecodedValueKind::INT64}}, + {std::make_shared(), + {DecodedValueKind::INT32, DecodedValueKind::DOUBLE}}, + {std::make_shared(), + {DecodedValueKind::INT64, DecodedValueKind::BINARY}}, + {std::make_shared(6), + {DecodedValueKind::INT32, DecodedValueKind::DOUBLE, DecodedValueKind::BINARY}}, + {std::make_shared(6), + {DecodedValueKind::BOOL, DecodedValueKind::BINARY, DecodedValueKind::DOUBLE}}, + {std::make_shared(18, 2), + {DecodedValueKind::BOOL, DecodedValueKind::UINT64, DecodedValueKind::FLOAT, + DecodedValueKind::DOUBLE}}, + }; + + for (const auto& test_case : cases) { + for (auto kind : test_case.illegal_kinds) { + std::vector values = {0}; + auto result = read_column(test_case.type, make_fixed_view(kind, values)); + expect_not_supported(result.status); + EXPECT_EQ(0, result.column->size()) << test_case.type->get_name(); + } + } +} + +} // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_pb_test.cpp b/be/test/core/data_type_serde/data_type_serde_pb_test.cpp index 986583982eb2bd..c1663bf7a9dd49 100644 --- a/be/test/core/data_type_serde/data_type_serde_pb_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_pb_test.cpp @@ -54,6 +54,7 @@ #include "core/data_type/data_type_quantilestate.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" #include "core/data_type_serde/data_type_serde.h" #include "core/types.h" #include "core/value/bitmap_value.h" @@ -646,6 +647,17 @@ TEST(DataTypeSerDePbTest, DataTypeScalaSerDeTestDateTime) { } } +TEST(DataTypeSerDePbTest, DataTypeTimeStampTzToProtobufKeepsScale) { + DataTypePtr data_type(std::make_shared(6)); + PTypeDesc type_desc; + data_type->to_protobuf(&type_desc); + + ASSERT_EQ(type_desc.types_size(), 1); + const auto& scalar_type = type_desc.types(0).scalar_type(); + EXPECT_EQ(scalar_type.type(), TPrimitiveType::TIMESTAMPTZ); + EXPECT_EQ(scalar_type.scale(), 6); +} + TEST(DataTypeSerDePbTest, DataTypeScalaSerDeTestLargeInt) { std::cout << "==== LargeInt === " << std::endl; // LargeInt @@ -662,4 +674,4 @@ TEST(DataTypeSerDePbTest, DataTypeScalaSerDeTestLargeInt) { check_pb_col(data_type, *vec.get()); } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp new file mode 100644 index 00000000000000..d4bd6ab6c06360 --- /dev/null +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -0,0 +1,371 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/scan/access_path_parser.h" + +#include +#include + +#include +#include +#include +#include + +#include "common/consts.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/field.h" + +namespace doris { +namespace { + +TColumnAccessPath data_access_path(std::vector path) { + TColumnAccessPath access_path; + access_path.__set_type(TAccessPathType::DATA); + TDataAccessPath data_path; + data_path.__set_path(std::move(path)); + access_path.__set_data_access_path(std::move(data_path)); + return access_path; +} + +TColumnAccessPath data_access_path_without_payload() { + TColumnAccessPath access_path; + access_path.__set_type(TAccessPathType::DATA); + return access_path; +} + +TColumnAccessPath meta_access_path() { + TColumnAccessPath access_path; + access_path.__set_type(TAccessPathType::META); + return access_path; +} + +format::ColumnDefinition field(int32_t id, std::string name, DataTypePtr type, + std::vector children = {}, + std::vector aliases = {}) { + return { + .identifier = Field::create_field(id), + .name = std::move(name), + .name_mapping = std::move(aliases), + .type = std::move(type), + .children = std::move(children), + }; +} + +format::ColumnDefinition root_column(int32_t id, std::string name, DataTypePtr type) { + return { + .identifier = Field::create_field(id), + .name = std::move(name), + .type = std::move(type), + }; +} + +void expect_child(const format::ColumnDefinition& child, int32_t id, const std::string& name) { + ASSERT_TRUE(child.has_identifier_field_id()); + EXPECT_EQ(child.get_identifier_field_id(), id); + EXPECT_EQ(child.name, name); +} + +const format::ColumnDefinition* find_child_by_name(const format::ColumnDefinition& parent, + const std::string& name) { + for (const auto& child : parent.children) { + if (child.name == name) { + return &child; + } + } + return nullptr; +} + +} // namespace + +// Scenario: primitive columns and scanner-materialized virtual columns should not build nested +// children, even when their descriptor carries access paths that are not meaningful to the parser. +TEST(AccessPathParserTest, IgnoresPrimitiveColumnsAndScannerVirtualColumns) { + auto int_type = std::make_shared(); + auto string_type = std::make_shared(); + + // Primitive columns have no nested children, so parser should not inspect even invalid paths. + auto primitive = root_column(1, "id", int_type); + auto status = AccessPathParser::build_nested_children( + &primitive, std::vector {meta_access_path()}, nullptr); + ASSERT_TRUE(status.ok()) << status; + EXPECT_TRUE(primitive.children.empty()); + + // Iceberg rowid is materialized by scanner/table-reader logic and may carry a negative access + // path. Parser must leave it untouched. + auto rowid_type = std::make_shared( + DataTypes {string_type, std::make_shared(), + std::make_shared(), string_type}, + Strings {"file_path", "row_pos", "partition_spec_id", "partition_data_json"}); + format::ColumnDefinition rowid { + .identifier = Field::create_field(BeConsts::ICEBERG_ROWID_COL), + .name = BeConsts::ICEBERG_ROWID_COL, + .type = rowid_type, + }; + status = AccessPathParser::build_nested_children( + &rowid, std::vector {data_access_path({"-1"})}, nullptr); + ASSERT_TRUE(status.ok()) << status; + EXPECT_TRUE(rowid.children.empty()); +} + +// Scenario: reject unsupported top-level inputs before recursive type parsing, including META +// paths, missing DATA payloads, and access paths whose root does not match the projected slot. +TEST(AccessPathParserTest, RejectsUnsupportedTopLevelAccessPathInputs) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"a"}); + + struct Case { + std::string name; + format::ColumnDefinition column; + std::vector paths; + }; + std::vector cases; + cases.push_back({"meta path", root_column(100, "s", struct_type), {meta_access_path()}}); + cases.push_back({"missing DATA payload", + root_column(100, "s", struct_type), + {data_access_path_without_payload()}}); + cases.push_back({"wrong root name", + root_column(100, "s", struct_type), + {data_access_path({"other", "a"})}}); + cases.push_back({"wrong root field id", + root_column(100, "s", struct_type), + {data_access_path({"101", "a"})}}); + + for (auto& test_case : cases) { + auto status = AccessPathParser::build_nested_children(&test_case.column, test_case.paths, + nullptr); + EXPECT_FALSE(status.ok()) << test_case.name; + } +} + +// Scenario: struct access paths support field-id lookup, alias lookup, case-insensitive name +// fallback, and whole-struct expansion; reserved array/map path tokens remain invalid. +TEST(AccessPathParserTest, StructAccessPathMatrix) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + format::ColumnDefinition schema { + .identifier = Field::create_field(100), + .name = "s", + .type = struct_type, + .children = + { + field(101, "a", int_type), + field(205, "b", int_type, {}, {"old_b"}), + }, + }; + + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "A"})}, nullptr); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 0, "a"); + } + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"100", "205"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 205, "b"); + } + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "old_b"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 205, "b"); + EXPECT_EQ(column.children[0].name_mapping, std::vector({"old_b"})); + } + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 2); + expect_child(column.children[0], 101, "a"); + expect_child(column.children[1], 205, "b"); + } + + for (const auto& invalid_child : {"OFFSET", "*", "KEYS", "VALUES", "missing"}) { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", invalid_child})}, + &schema); + EXPECT_FALSE(status.ok()) << invalid_child; + } +} + +// Scenario: array access paths must pass through the "*" element token, then reuse struct child +// parsing under the element wrapper; invalid array tokens are rejected. +TEST(AccessPathParserTest, ArrayAccessPathMatrix) { + auto int_type = std::make_shared(); + auto string_type = std::make_shared(); + auto element_type = std::make_shared(DataTypes {string_type, int_type}, + Strings {"item", "quantity"}); + auto array_type = std::make_shared(element_type); + format::ColumnDefinition schema { + .identifier = Field::create_field(200), + .name = "items", + .type = array_type, + .children = + { + field(201, "element", element_type, + { + field(202, "item", string_type, {}, {"old_item"}), + field(203, "quantity", int_type), + }), + }, + }; + + { + auto column = root_column(200, "items", array_type); + auto status = AccessPathParser::build_nested_children( + &column, + std::vector {data_access_path({"items", "*", "old_item"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 201, "element"); + ASSERT_EQ(column.children[0].children.size(), 1); + expect_child(column.children[0].children[0], 202, "item"); + EXPECT_EQ(column.children[0].children[0].name_mapping, + std::vector({"old_item"})); + } + { + auto column = root_column(200, "items", array_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"items"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 201, "element"); + ASSERT_EQ(column.children[0].children.size(), 2); + expect_child(column.children[0].children[0], 202, "item"); + expect_child(column.children[0].children[1], 203, "quantity"); + } + + for (const auto& invalid_path : std::vector> { + {"items", "OFFSET"}, {"items", "item"}, {"items", "*", "missing"}}) { + auto column = root_column(200, "items", array_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path(invalid_path)}, &schema); + EXPECT_FALSE(status.ok()) << invalid_path.back(); + } +} + +// Scenario: map access paths split KEYS/VALUES, force the missing side needed for materialization, +// merge repeated value-child requests, and reject unsupported map child tokens. +TEST(AccessPathParserTest, MapAccessPathMatrix) { + auto int_type = std::make_shared(); + auto string_type = std::make_shared(); + auto value_type = std::make_shared( + DataTypes {string_type, int_type, string_type}, Strings {"full_name", "age", "gender"}); + auto map_type = std::make_shared(string_type, value_type); + format::ColumnDefinition schema { + .identifier = Field::create_field(300), + .name = "m", + .type = map_type, + .children = + { + field(301, "key", string_type), + field(302, "value", value_type, + { + field(303, "full_name", string_type, {}, {"name"}), + field(304, "age", int_type), + field(305, "gender", string_type), + }), + }, + }; + + { + auto column = root_column(300, "m", map_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"m", "KEYS"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 2); + expect_child(column.children[0], 301, "key"); + expect_child(column.children[1], 302, "value"); + ASSERT_EQ(column.children[1].children.size(), 3); + const auto* full_name = find_child_by_name(column.children[1], "full_name"); + ASSERT_NE(full_name, nullptr); + expect_child(*full_name, 303, "full_name"); + const auto* age = find_child_by_name(column.children[1], "age"); + ASSERT_NE(age, nullptr); + expect_child(*age, 304, "age"); + const auto* gender = find_child_by_name(column.children[1], "gender"); + ASSERT_NE(gender, nullptr); + expect_child(*gender, 305, "gender"); + } + { + auto column = root_column(300, "m", map_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"m", "VALUES", "age"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 2); + expect_child(column.children[0], 301, "key"); + expect_child(column.children[1], 302, "value"); + ASSERT_EQ(column.children[1].children.size(), 1); + expect_child(column.children[1].children[0], 304, "age"); + } + { + auto column = root_column(300, "m", map_type); + auto status = AccessPathParser::build_nested_children( + &column, + std::vector { + data_access_path({"m", "VALUES", "name"}), + data_access_path({"m", "*", "gender"}), + }, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 2); + ASSERT_EQ(column.children[1].children.size(), 2); + const auto* full_name = find_child_by_name(column.children[1], "full_name"); + ASSERT_NE(full_name, nullptr); + expect_child(*full_name, 303, "full_name"); + EXPECT_EQ(full_name->name_mapping, std::vector({"name"})); + const auto* gender = find_child_by_name(column.children[1], "gender"); + ASSERT_NE(gender, nullptr); + expect_child(*gender, 305, "gender"); + } + { + auto column = root_column(300, "m", map_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"m"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 2); + ASSERT_EQ(column.children[1].children.size(), 3); + } + + for (const auto& invalid_path : std::vector> { + {"m", "OFFSET"}, {"m", "ENTRY"}, {"m", "VALUES", "missing"}}) { + auto column = root_column(300, "m", map_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path(invalid_path)}, &schema); + EXPECT_FALSE(status.ok()) << invalid_path.back(); + } +} + +} // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp new file mode 100644 index 00000000000000..661298df1874ee --- /dev/null +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -0,0 +1,602 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/scan/file_scanner_v2.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exec/operator/file_scan_operator.h" +#include "exec/scan/file_scanner.h" +#include "exec/scan/split_source_connector.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vruntimefilter_wrapper.h" +#include "exprs/vslot_ref.h" +#include "format_v2/expr/cast.h" + +namespace doris { +namespace { + +TFileRangeDesc range_with_format(std::string table_format, TFileFormatType::type format_type) { + TFileRangeDesc range; + range.__set_format_type(format_type); + if (!table_format.empty()) { + TTableFormatFileDesc table_desc; + table_desc.__set_table_format_type(std::move(table_format)); + range.__set_table_format_params(std::move(table_desc)); + } + return range; +} + +TFileRangeDesc hudi_range_with_delta_logs() { + auto range = range_with_format("hudi", TFileFormatType::FORMAT_PARQUET); + THudiFileDesc hudi_params; + hudi_params.__set_delta_logs({"delta.log"}); + range.table_format_params.__set_hudi_params(std::move(hudi_params)); + return range; +} + +TFileRangeDesc paimon_cpp_jni_range() { + auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI); + TPaimonFileDesc paimon_params; + paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_CPP); + range.table_format_params.__set_paimon_params(std::move(paimon_params)); + return range; +} + +TFileRangeDesc legacy_paimon_jni_range_without_reader_type() { + auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI); + TPaimonFileDesc paimon_params; + paimon_params.__set_paimon_split("legacy-split"); + paimon_params.__set_paimon_predicate("legacy-predicate"); + range.table_format_params.__set_paimon_params(std::move(paimon_params)); + return range; +} + +struct RetryableCloseState { + int close_calls = 0; +}; + +class RetryableCloseTableReader final : public format::TableReader { +public: + explicit RetryableCloseTableReader(std::shared_ptr state) + : _state(std::move(state)) {} + + Status close() override { + ++_state->close_calls; + if (_state->close_calls == 1) { + return Status::InternalError("injected table reader close failure"); + } + return Status::OK(); + } + +private: + std::shared_ptr _state; +}; + +VExprSPtr slot_ref(int slot_id, int column_id, DataTypePtr type, const std::string& name) { + return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name); +} + +TExprNode bool_in_pred_node(); + +class UnsafePartitionPredicate final : public VExpr { +public: + UnsafePartitionPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_safe_to_execute_on_selected_rows() const override { return false; } + +private: + const std::string _expr_name = "UnsafePartitionPredicate"; +}; + +VExprContextSPtr runtime_filter_context(VExprSPtr impl, int filter_id) { + const auto node = bool_in_pred_node(); + return VExprContext::create_shared( + VRuntimeFilterWrapper::create_shared(node, std::move(impl), 0.4, false, filter_id)); +} + +TExprNode bool_in_pred_node() { + TTypeDesc bool_type; + TTypeNode bool_node; + TScalarType bool_scalar_type; + bool_scalar_type.__set_type(TPrimitiveType::BOOLEAN); + bool_node.__set_type(TTypeNodeType::SCALAR); + bool_node.__set_scalar_type(bool_scalar_type); + bool_type.types.push_back(bool_node); + + TExprNode node; + node.__set_type(bool_type); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + return node; +} + +} // namespace + +// Scenario: FileScannerV2::is_supported should honor table format, scan params format, and the +// optional per-range file format override as a single matrix. +TEST(FileScannerV2Test, SupportedFormatMatrix) { + struct Case { + std::string table_format; + TFileFormatType::type params_format; + std::optional range_format; + bool expected; + }; + + const std::vector cases { + {"", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"tvf", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"iceberg", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"paimon", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"hudi", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"jdbc", TFileFormatType::FORMAT_PARQUET, std::nullopt, false}, + {"", TFileFormatType::FORMAT_JNI, std::nullopt, false}, + {"hive", TFileFormatType::FORMAT_ORC, std::nullopt, true}, + {"transactional_hive", TFileFormatType::FORMAT_ORC, std::nullopt, false}, + {"jdbc", TFileFormatType::FORMAT_JNI, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_JNI, std::nullopt, false}, + {"", TFileFormatType::FORMAT_CSV_PLAIN, std::nullopt, true}, + {"tvf", TFileFormatType::FORMAT_CSV_GZ, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_CSV_BZ2, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_CSV_LZ4FRAME, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_CSV_LZ4BLOCK, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_CSV_LZOP, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_CSV_DEFLATE, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_CSV_SNAPPYBLOCK, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_PROTO, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_TEXT, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_JSON, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_PARQUET, TFileFormatType::FORMAT_ORC, true}, + {"hive", TFileFormatType::FORMAT_ORC, TFileFormatType::FORMAT_PARQUET, true}, + {"hive", TFileFormatType::FORMAT_PARQUET, TFileFormatType::FORMAT_CSV_PLAIN, true}, + {"hive", TFileFormatType::FORMAT_PARQUET, TFileFormatType::FORMAT_TEXT, true}, + {"hive", TFileFormatType::FORMAT_PARQUET, TFileFormatType::FORMAT_JSON, true}, + {"tvf", TFileFormatType::FORMAT_PARQUET, TFileFormatType::FORMAT_NATIVE, true}, + {"remote_doris", TFileFormatType::FORMAT_ARROW, std::nullopt, true}, + {"hive", TFileFormatType::FORMAT_ARROW, std::nullopt, false}, + {"", TFileFormatType::FORMAT_ARROW, std::nullopt, false}, + {"", TFileFormatType::FORMAT_WAL, std::nullopt, false}, + }; + + for (const auto& test_case : cases) { + TFileScanRangeParams params; + params.__set_format_type(test_case.params_format); + auto range = range_with_format(test_case.table_format, + test_case.range_format.value_or(test_case.params_format)); + if (!test_case.range_format.has_value()) { + range.__isset.format_type = false; + } + EXPECT_EQ(FileScannerV2::is_supported(params, range), test_case.expected) + << "table_format=" << test_case.table_format + << ", params_format=" << static_cast(test_case.params_format) + << ", range_has_format=" << test_case.range_format.has_value(); + } + + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + EXPECT_FALSE(FileScannerV2::is_supported(params, hudi_range_with_delta_logs())); +} + +TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesOnly) { + TQueryOptions query_options; + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + + query_options.__set_enable_file_scanner_v2(true); + EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, true, params)); + + const std::vector unsupported_formats { + TFileFormatType::FORMAT_WAL, + }; + for (const auto format : unsupported_formats) { + params.__set_format_type(format); + EXPECT_FALSE( + FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + } + + params.__set_format_type(TFileFormatType::FORMAT_ORC); + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("transactional_hive"); + params.__set_table_format_params(table_format_params); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + + params.table_format_params.__set_table_format_type("hive"); + EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + + query_options.__set_enable_file_scanner_v2(false); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); +} + +TEST(FileScannerV2Test, JniCompatibilityShapesForceLegacyScanner) { + TQueryOptions query_options; + query_options.__set_enable_file_scanner_v2(true); + query_options.__set_enable_paimon_cpp_reader(true); + + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_JNI); + // Rolling upgrades may carry the only Paimon marker and reader type on each split. Since the + // scan-level selector cannot inspect that split yet, JNI scans conservatively stay on V1. + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + EXPECT_FALSE(FileScannerV2::is_supported(params, paimon_cpp_jni_range())); + + // Older FEs can omit reader_type. The legacy scanner interprets this as Paimon JNI when the C++ + // reader is disabled, so the scan-level choice must still stay on V1. + query_options.__set_enable_paimon_cpp_reader(false); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + EXPECT_FALSE( + FileScannerV2::is_supported(params, legacy_paimon_jni_range_without_reader_type())); +} + +TEST(FileScannerV2Test, FailedTableReaderCloseCanBeRetriedThroughScanner) { + RuntimeState state; + RuntimeProfile profile("file_scanner_v2_close_retry"); + auto close_state = std::make_shared(); + FileScannerV2 scanner(&state, &profile, + std::make_unique(close_state)); + + EXPECT_FALSE(scanner.close(&state).ok()); + EXPECT_EQ(close_state->close_calls, 1); + EXPECT_TRUE(scanner.close(&state).ok()); + EXPECT_EQ(close_state->close_calls, 2); + EXPECT_TRUE(scanner.close(&state).ok()); + EXPECT_EQ(close_state->close_calls, 2); +} + +// Scenario: Once FileScannerV2 is selected, an unsupported range must fail instead of falling back +// to FileScanner. +TEST(FileScannerV2Test, ValidateScanRangeRejectsUnsupportedRange) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + + const auto supported = range_with_format("hive", TFileFormatType::FORMAT_PARQUET); + EXPECT_TRUE(FileScannerV2::TEST_validate_scan_range(params, supported).ok()); + + const auto unsupported = range_with_format("lakesoul", TFileFormatType::FORMAT_PARQUET); + const auto status = FileScannerV2::TEST_validate_scan_range(params, unsupported); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("lakesoul"), std::string::npos); +} + +// Scenario: FileScannerV2 converts only the file formats implemented by format_v2 readers and +// rejects everything else before TableReader::init sees an unsupported FileFormat. +TEST(FileScannerV2Test, FileFormatConversionMatrix) { + struct Case { + TFileFormatType::type input; + std::optional expected; + }; + const std::vector cases { + {TFileFormatType::FORMAT_PARQUET, format::FileFormat::PARQUET}, + {TFileFormatType::FORMAT_JNI, format::FileFormat::JNI}, + {TFileFormatType::FORMAT_CSV_PLAIN, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_CSV_GZ, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_CSV_BZ2, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_CSV_LZ4FRAME, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_CSV_LZ4BLOCK, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_CSV_LZOP, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_CSV_DEFLATE, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_CSV_SNAPPYBLOCK, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_PROTO, format::FileFormat::CSV}, + {TFileFormatType::FORMAT_TEXT, format::FileFormat::TEXT}, + {TFileFormatType::FORMAT_JSON, format::FileFormat::JSON}, + {TFileFormatType::FORMAT_NATIVE, format::FileFormat::NATIVE}, + {TFileFormatType::FORMAT_ARROW, format::FileFormat::ARROW}, + {TFileFormatType::FORMAT_ORC, format::FileFormat::ORC}, + }; + + for (const auto& test_case : cases) { + format::FileFormat file_format = format::FileFormat::PARQUET; + const auto status = FileScannerV2::TEST_to_file_format(test_case.input, &file_format); + if (test_case.expected.has_value()) { + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(file_format, *test_case.expected); + } else { + EXPECT_FALSE(status.ok()); + } + } +} + +TEST(FileScannerV2Test, RealtimeCounterDeltasUseReaderBytesAsRemoteWithoutCacheStats) { + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_statistics; + int64_t last_read_bytes = 0; + int64_t last_read_rows = 0; + int64_t last_bytes_read_from_local = 0; + int64_t last_bytes_read_from_remote = 0; + + file_reader_stats.read_bytes = 100; + file_reader_stats.read_rows = 7; + auto deltas = FileScannerV2::TEST_collect_realtime_counter_deltas( + file_reader_stats, file_cache_statistics, + FileScannerV2::UncachedReaderBytesStorage::REMOTE, &last_read_bytes, &last_read_rows, + &last_bytes_read_from_local, &last_bytes_read_from_remote); + EXPECT_EQ(7, deltas.scan_rows); + EXPECT_EQ(100, deltas.scan_bytes); + EXPECT_EQ(0, deltas.scan_bytes_from_local_storage); + EXPECT_EQ(100, deltas.scan_bytes_from_remote_storage); + + deltas = FileScannerV2::TEST_collect_realtime_counter_deltas( + file_reader_stats, file_cache_statistics, + FileScannerV2::UncachedReaderBytesStorage::REMOTE, &last_read_bytes, &last_read_rows, + &last_bytes_read_from_local, &last_bytes_read_from_remote); + EXPECT_EQ(0, deltas.scan_rows); + EXPECT_EQ(0, deltas.scan_bytes); + EXPECT_EQ(0, deltas.scan_bytes_from_local_storage); + EXPECT_EQ(0, deltas.scan_bytes_from_remote_storage); + + file_reader_stats.read_bytes = 160; + file_reader_stats.read_rows = 9; + deltas = FileScannerV2::TEST_collect_realtime_counter_deltas( + file_reader_stats, file_cache_statistics, + FileScannerV2::UncachedReaderBytesStorage::REMOTE, &last_read_bytes, &last_read_rows, + &last_bytes_read_from_local, &last_bytes_read_from_remote); + EXPECT_EQ(2, deltas.scan_rows); + EXPECT_EQ(60, deltas.scan_bytes); + EXPECT_EQ(0, deltas.scan_bytes_from_local_storage); + EXPECT_EQ(60, deltas.scan_bytes_from_remote_storage); +} + +TEST(FileScannerV2Test, RealtimeCounterDeltasUseFileCacheDeltasWhenAvailable) { + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_statistics; + int64_t last_read_bytes = 0; + int64_t last_read_rows = 0; + int64_t last_bytes_read_from_local = 0; + int64_t last_bytes_read_from_remote = 0; + + file_reader_stats.read_bytes = 100; + file_reader_stats.read_rows = 7; + file_cache_statistics.bytes_read_from_local = 30; + file_cache_statistics.bytes_read_from_remote = 70; + auto deltas = FileScannerV2::TEST_collect_realtime_counter_deltas( + file_reader_stats, file_cache_statistics, + FileScannerV2::UncachedReaderBytesStorage::REMOTE, &last_read_bytes, &last_read_rows, + &last_bytes_read_from_local, &last_bytes_read_from_remote); + EXPECT_EQ(7, deltas.scan_rows); + EXPECT_EQ(100, deltas.scan_bytes); + EXPECT_EQ(30, deltas.scan_bytes_from_local_storage); + EXPECT_EQ(70, deltas.scan_bytes_from_remote_storage); + + file_reader_stats.read_bytes = 125; + file_reader_stats.read_rows = 10; + file_cache_statistics.bytes_read_from_local = 35; + file_cache_statistics.bytes_read_from_remote = 90; + deltas = FileScannerV2::TEST_collect_realtime_counter_deltas( + file_reader_stats, file_cache_statistics, + FileScannerV2::UncachedReaderBytesStorage::REMOTE, &last_read_bytes, &last_read_rows, + &last_bytes_read_from_local, &last_bytes_read_from_remote); + EXPECT_EQ(3, deltas.scan_rows); + EXPECT_EQ(25, deltas.scan_bytes); + EXPECT_EQ(5, deltas.scan_bytes_from_local_storage); + EXPECT_EQ(20, deltas.scan_bytes_from_remote_storage); +} + +TEST(FileScannerV2Test, RealtimeCounterDeltasDoNotChargePeerCacheAsRemoteStorage) { + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_statistics; + int64_t last_read_bytes = 0; + int64_t last_read_rows = 0; + int64_t last_bytes_read_from_local = 0; + int64_t last_bytes_read_from_remote = 0; + + file_reader_stats.read_bytes = 100; + file_reader_stats.read_rows = 7; + file_cache_statistics.num_peer_io_total = 1; + file_cache_statistics.bytes_read_from_peer = 100; + auto deltas = FileScannerV2::TEST_collect_realtime_counter_deltas( + file_reader_stats, file_cache_statistics, + FileScannerV2::UncachedReaderBytesStorage::REMOTE, &last_read_bytes, &last_read_rows, + &last_bytes_read_from_local, &last_bytes_read_from_remote); + EXPECT_EQ(7, deltas.scan_rows); + EXPECT_EQ(100, deltas.scan_bytes); + EXPECT_EQ(0, deltas.scan_bytes_from_local_storage); + EXPECT_EQ(0, deltas.scan_bytes_from_remote_storage); +} + +TEST(FileScannerV2Test, RealtimeCounterDeltasDoNotChargeLocalFileFallbackAsRemoteStorage) { + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_statistics; + int64_t last_read_bytes = 0; + int64_t last_read_rows = 0; + int64_t last_bytes_read_from_local = 0; + int64_t last_bytes_read_from_remote = 0; + + file_reader_stats.read_bytes = 100; + file_reader_stats.read_rows = 7; + auto deltas = FileScannerV2::TEST_collect_realtime_counter_deltas( + file_reader_stats, file_cache_statistics, + FileScannerV2::UncachedReaderBytesStorage::LOCAL, &last_read_bytes, &last_read_rows, + &last_bytes_read_from_local, &last_bytes_read_from_remote); + EXPECT_EQ(7, deltas.scan_rows); + EXPECT_EQ(100, deltas.scan_bytes); + EXPECT_EQ(100, deltas.scan_bytes_from_local_storage); + EXPECT_EQ(0, deltas.scan_bytes_from_remote_storage); +} + +TEST(FileScannerV2Test, FileCacheStatisticsArePublishedToScannerProfile) { + RuntimeProfile profile("file_scanner_v2"); + io::FileCacheStatistics file_cache_statistics; + file_cache_statistics.num_local_io_total = 3; + file_cache_statistics.num_remote_io_total = 5; + file_cache_statistics.num_peer_io_total = 7; + file_cache_statistics.bytes_read_from_local = 11; + file_cache_statistics.bytes_read_from_remote = 13; + file_cache_statistics.bytes_read_from_peer = 17; + file_cache_statistics.bytes_write_into_cache = 19; + + FileScannerV2::TEST_report_file_cache_profile(&profile, file_cache_statistics); + + ASSERT_NE(profile.get_counter("FileCache"), nullptr); + EXPECT_EQ(profile.get_counter("NumLocalIOTotal")->value(), 3); + EXPECT_EQ(profile.get_counter("NumRemoteIOTotal")->value(), 5); + EXPECT_EQ(profile.get_counter("NumPeerIOTotal")->value(), 7); + EXPECT_EQ(profile.get_counter("BytesScannedFromCache")->value(), 11); + EXPECT_EQ(profile.get_counter("BytesScannedFromRemote")->value(), 13); + EXPECT_EQ(profile.get_counter("BytesScannedFromPeer")->value(), 17); + EXPECT_EQ(profile.get_counter("BytesWriteIntoCache")->value(), 19); +} + +TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { + const auto not_found = Status::NotFound("missing external file"); + EXPECT_TRUE(FileScannerV2::TEST_should_skip_not_found(not_found, true)); + EXPECT_FALSE(FileScannerV2::TEST_should_skip_not_found(not_found, false)); + EXPECT_FALSE( + FileScannerV2::TEST_should_skip_not_found(Status::InternalError("read failed"), true)); + EXPECT_FALSE(FileScannerV2::TEST_should_skip_not_found(Status::OK(), true)); +} + +// Scenario: partition slots are identified from the explicit FE category when present, otherwise +// from the legacy is_file_slot flag. Scanner-generated rowid columns must never be treated as +// partition columns even if FE marks them as non-file slots. +TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) { + TFileScanSlotInfo legacy_partition; + legacy_partition.__set_is_file_slot(false); + EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "dt")); + + TFileScanSlotInfo legacy_file; + legacy_file.__set_is_file_slot(true); + EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_file, "value")); + + EXPECT_FALSE( + FileScannerV2::TEST_is_partition_slot(legacy_partition, BeConsts::GLOBAL_ROWID_COL)); + EXPECT_FALSE( + FileScannerV2::TEST_is_partition_slot(legacy_partition, BeConsts::ICEBERG_ROWID_COL)); +} + +// Scenario: data-file slots are the complement of partition/default/synthesized columns for +// formats without embedded schema. FE may send either the new category or the old is_file_slot +// flag, and scanner-generated rowid columns must never be passed to a physical file reader. +TEST(FileScannerV2Test, DataFileSlotClassificationMatrix) { + TFileScanSlotInfo legacy_file; + legacy_file.__set_is_file_slot(true); + EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "value")); + + TFileScanSlotInfo legacy_partition; + legacy_partition.__set_is_file_slot(false); + EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_partition, "dt")); + + EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, BeConsts::GLOBAL_ROWID_COL)); + EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, BeConsts::ICEBERG_ROWID_COL)); +} + +// Scenario: table conjuncts are cloned into global-index space before they are handed to +// TableReader. Explicit slot-id mappings use the required_slots order; missing mappings fall back +// to the slot id itself for legacy descriptors. +TEST(FileScannerV2Test, RewriteSlotRefsToGlobalIndexMatrix) { + const auto int_type = std::make_shared(); + { + auto expr = slot_ref(42, 99, int_type, "value"); + const auto status = FileScannerV2::TEST_rewrite_slot_refs_to_global_index( + &expr, {{42, format::GlobalIndex(3)}}); + ASSERT_TRUE(status.ok()) << status; + const auto* rewritten = assert_cast(expr.get()); + EXPECT_EQ(rewritten->slot_id(), 3); + EXPECT_EQ(rewritten->column_id(), 3); + EXPECT_EQ(rewritten->column_name(), "value"); + } + { + auto expr = slot_ref(7, 99, int_type, "legacy_value"); + const auto status = FileScannerV2::TEST_rewrite_slot_refs_to_global_index(&expr, {}); + ASSERT_TRUE(status.ok()) << status; + const auto* rewritten = assert_cast(expr.get()); + EXPECT_EQ(rewritten->slot_id(), 7); + EXPECT_EQ(rewritten->column_id(), 7); + EXPECT_EQ(rewritten->column_name(), "legacy_value"); + } + { + auto cast_expr = format::Cast::create_shared(int_type); + cast_expr->add_child(slot_ref(9, 9, int_type, "nested_value")); + VExprSPtr expr = cast_expr; + const auto status = FileScannerV2::TEST_rewrite_slot_refs_to_global_index( + &expr, {{9, format::GlobalIndex(1)}}); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(expr->get_num_children(), 1); + const auto* rewritten_child = assert_cast(expr->children()[0].get()); + EXPECT_EQ(rewritten_child->slot_id(), 1); + EXPECT_EQ(rewritten_child->column_id(), 1); + EXPECT_EQ(rewritten_child->column_name(), "nested_value"); + } + { + const auto node = bool_in_pred_node(); + auto impl = VDirectInPredicate::create_shared(node, nullptr); + impl->add_child(slot_ref(11, 11, int_type, "rf_value")); + VExprSPtr expr = VRuntimeFilterWrapper::create_shared(node, impl, 0.4, false, 7); + const auto status = FileScannerV2::TEST_rewrite_slot_refs_to_global_index( + &expr, {{11, format::GlobalIndex(2)}}); + ASSERT_TRUE(status.ok()) << status; + + auto* runtime_filter = assert_cast(expr.get()); + auto rewritten_impl = runtime_filter->get_impl(); + ASSERT_NE(rewritten_impl, nullptr); + ASSERT_EQ(rewritten_impl->get_num_children(), 1); + const auto* rewritten_child = + assert_cast(rewritten_impl->children()[0].get()); + EXPECT_EQ(rewritten_child->slot_id(), 2); + EXPECT_EQ(rewritten_child->column_id(), 2); + EXPECT_EQ(rewritten_child->column_name(), "rf_value"); + } +} + +TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) { + const auto bool_type = std::make_shared(); + auto unsafe_predicate = std::make_shared(); + unsafe_predicate->add_child(slot_ref(1, 0, bool_type, "part")); + VExprContextSPtrs conjuncts { + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 1), + runtime_filter_context(std::move(unsafe_predicate), 2), + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 3), + }; + + RuntimeState state; + RuntimeProfile profile("file_scanner"); + FileScanner scanner(&state, &profile, nullptr, nullptr, nullptr); + scanner.TEST_init_runtime_filter_partition_prune_ctxs(conjuncts, {{1, 0}}); + + const auto& partition_conjuncts = scanner.TEST_runtime_filter_partition_prune_ctxs(); + ASSERT_EQ(partition_conjuncts.size(), 1); + EXPECT_EQ(partition_conjuncts[0], conjuncts[0]); +} + +} // namespace doris diff --git a/be/test/exec/scan/vfile_scanner_exception_test.cpp b/be/test/exec/scan/vfile_scanner_exception_test.cpp index b661f5e1088682..3a489a7d52eee5 100644 --- a/be/test/exec/scan/vfile_scanner_exception_test.cpp +++ b/be/test/exec/scan/vfile_scanner_exception_test.cpp @@ -18,13 +18,19 @@ #include #include +#include #include +#include #include #include "common/object_pool.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" #include "cpp/sync_point.h" #include "exec/operator/file_scan_operator.h" #include "exec/scan/file_scanner.h" +#include "exec/scan/split_source_connector.h" +#include "format_v2/table/hive_reader.h" #include "io/fs/local_file_system.h" #include "load/group_commit/wal/wal_manager.h" #include "runtime/cluster_info.h" @@ -34,7 +40,6 @@ #include "runtime/user_function_cache.h" namespace doris { - class TestSplitSourceConnectorStub : public SplitSourceConnector { private: std::mutex _range_lock; @@ -336,4 +341,114 @@ TEST_F(VfileScannerExceptionTest, process_late_arrival_conjuncts_retain) { WARN_IF_ERROR(scanner->close(&_runtime_state), "fail to close scanner"); } +TEST(HiveReaderPositionMappingTest, PositionMappingUsesColumnIdxsForFileSlots) { + TQueryOptions query_options; + query_options.hive_parquet_use_column_names = false; + RuntimeState runtime_state; + runtime_state.set_query_options(query_options); + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + params.__set_column_idxs({2, 0}); + format::ProjectedColumnBuildContext context { + .scan_params = ¶ms, + .runtime_state = &runtime_state, + }; + format::hive::HiveReader reader; + + TFileScanSlotInfo id_slot; + id_slot.__set_is_file_slot(true); + format::ColumnDefinition id_column { + .identifier = Field::create_field("id"), + .name = "id", + .type = std::make_shared(), + }; + + TFileScanSlotInfo name_slot; + name_slot.__set_is_file_slot(true); + format::ColumnDefinition name_column { + .identifier = Field::create_field("name"), + .name = "name", + .type = std::make_shared(), + }; + + ASSERT_TRUE(reader.annotate_projected_column(id_slot, &context, &id_column).ok()); + ASSERT_TRUE(id_column.has_identifier_field_id()); + EXPECT_EQ(id_column.get_identifier_position(), 2); + EXPECT_EQ(context.next_file_column_idx, 1); + + ASSERT_TRUE(reader.annotate_projected_column(name_slot, &context, &name_column).ok()); + ASSERT_TRUE(name_column.has_identifier_field_id()); + EXPECT_EQ(name_column.get_identifier_position(), 0); + EXPECT_EQ(context.next_file_column_idx, 2); + ASSERT_TRUE(reader.validate_projected_columns(context).ok()); +} + +TEST(HiveReaderPositionMappingTest, PositionMappingDoesNotConsumePartitionSlots) { + TQueryOptions query_options; + query_options.hive_parquet_use_column_names = false; + RuntimeState runtime_state; + runtime_state.set_query_options(query_options); + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + params.__set_column_idxs({3}); + format::ProjectedColumnBuildContext context { + .scan_params = ¶ms, + .runtime_state = &runtime_state, + }; + format::hive::HiveReader reader; + + TFileScanSlotInfo partition_slot; + partition_slot.__set_is_file_slot(false); + format::ColumnDefinition partition_column { + .identifier = Field::create_field("year"), + .name = "year", + .type = std::make_shared(), + }; + + TFileScanSlotInfo value_slot; + value_slot.__set_is_file_slot(true); + format::ColumnDefinition value_column { + .identifier = Field::create_field("value"), + .name = "value", + .type = std::make_shared(), + }; + + ASSERT_TRUE(reader.annotate_projected_column(partition_slot, &context, &partition_column).ok()); + ASSERT_TRUE(partition_column.has_identifier_name()); + EXPECT_EQ(partition_column.get_identifier_name(), "year"); + EXPECT_EQ(context.next_file_column_idx, 0); + + ASSERT_TRUE(reader.annotate_projected_column(value_slot, &context, &value_column).ok()); + ASSERT_TRUE(value_column.has_identifier_field_id()); + EXPECT_EQ(value_column.get_identifier_position(), 3); + EXPECT_EQ(context.next_file_column_idx, 1); + ASSERT_TRUE(reader.validate_projected_columns(context).ok()); +} + +TEST(HiveReaderPositionMappingTest, PositionMappingFailsWhenColumnIdxsMissing) { + TQueryOptions query_options; + query_options.hive_parquet_use_column_names = false; + RuntimeState runtime_state; + runtime_state.set_query_options(query_options); + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + format::ProjectedColumnBuildContext context { + .scan_params = ¶ms, + .runtime_state = &runtime_state, + }; + format::hive::HiveReader reader; + + TFileScanSlotInfo value_slot; + value_slot.__set_is_file_slot(true); + format::ColumnDefinition value_column { + .identifier = Field::create_field("value"), + .name = "value", + .type = std::make_shared(), + }; + + auto status = reader.annotate_projected_column(value_slot, &context, &value_column); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(context.next_file_column_idx, 0); +} + } // namespace doris diff --git a/be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet b/be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet new file mode 100644 index 00000000000000..6b9c77842582ad Binary files /dev/null and b/be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet differ diff --git a/be/test/exprs/expr_zonemap_filter_test.cpp b/be/test/exprs/expr_zonemap_filter_test.cpp index 96e5d318006c0e..338d1de79fbfc0 100644 --- a/be/test/exprs/expr_zonemap_filter_test.cpp +++ b/be/test/exprs/expr_zonemap_filter_test.cpp @@ -47,6 +47,7 @@ #include "runtime/descriptor_helper.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" +#include "storage/index/bloom_filter/block_split_bloom_filter.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/segment/segment_iterator.h" @@ -128,6 +129,36 @@ ZoneMapEvalContext make_context(segment_v2::ZoneMap zone_map, const DataTypePtr& return ctx; } +DictionaryEvalContext make_dictionary_context(std::vector values, + const DataTypePtr& data_type) { + DictionaryEvalContext ctx; + ctx.slots.emplace(0, DictionaryEvalContext::SlotDictionary { + .data_type = data_type, + .values = std::move(values), + }); + return ctx; +} + +std::unique_ptr make_int_bloom_filter( + const std::vector& values) { + auto bloom_filter = std::make_unique(); + EXPECT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok()); + for (const auto value : values) { + bloom_filter->add_bytes(reinterpret_cast(&value), sizeof(value)); + } + return bloom_filter; +} + +BloomFilterEvalContext make_bloom_filter_context(const segment_v2::BloomFilter* bloom_filter, + const DataTypePtr& data_type) { + BloomFilterEvalContext ctx; + ctx.slots.emplace(0, BloomFilterEvalContext::SlotBloomFilter { + .data_type = data_type, + .bloom_filter = bloom_filter, + }); + return ctx; +} + segment_v2::ZoneMap make_int_zonemap(int32_t min_value, int32_t max_value) { segment_v2::ZoneMap zone_map; zone_map.min_value = int_field(min_value); @@ -337,6 +368,55 @@ TEST(ExprZonemapFilterTest, ComparisonZonemapHandlesNullAndUnsupportedInputs) { EXPECT_EQ(1, pass_all_ctx.stats.unusable_zonemap_eval_count); } +TEST(ExprZonemapFilterTest, ComparisonDictionaryAndBloomUseEqualityLiterals) { + auto type = int_type(); + auto slot = make_slot(0, type); + FunctionComparison equals; + + EXPECT_TRUE(equals.can_evaluate_dictionary_filter({slot, make_int_literal(2)})); + auto dictionary_ctx = make_dictionary_context({int_field(1), int_field(3)}, type); + EXPECT_EQ(ZoneMapFilterResult::kNoMatch, + equals.evaluate_dictionary_filter(dictionary_ctx, {slot, make_int_literal(2)})); + EXPECT_EQ(ZoneMapFilterResult::kMayMatch, + equals.evaluate_dictionary_filter(dictionary_ctx, {slot, make_int_literal(3)})); + + EXPECT_TRUE(equals.can_evaluate_bloom_filter({slot, make_int_literal(2)})); + auto bloom_filter = make_int_bloom_filter({1, 3}); + auto bloom_ctx = make_bloom_filter_context(bloom_filter.get(), type); + EXPECT_EQ(ZoneMapFilterResult::kNoMatch, + equals.evaluate_bloom_filter(bloom_ctx, {slot, make_int_literal(2)})); + EXPECT_EQ(ZoneMapFilterResult::kMayMatch, + equals.evaluate_bloom_filter(bloom_ctx, {slot, make_int_literal(3)})); + + FunctionComparison not_equals; + EXPECT_FALSE(not_equals.can_evaluate_dictionary_filter({slot, make_int_literal(3)})); + EXPECT_FALSE(not_equals.can_evaluate_bloom_filter({slot, make_int_literal(3)})); +} + +TEST(ExprZonemapFilterTest, DefaultFunctionForwardsDictionaryAndBloomEvaluation) { + auto type = int_type(); + auto slot = make_slot(0, type); + auto equals = SimpleFunctionFactory::instance().get_function( + "eq", ColumnsWithTypeAndName {{nullptr, type, "slot"}, {nullptr, type, "literal"}}, + std::make_shared()); + ASSERT_NE(equals, nullptr); + + EXPECT_TRUE(equals->can_evaluate_dictionary_filter({slot, make_int_literal(2)})); + auto dictionary_ctx = make_dictionary_context({int_field(1), int_field(3)}, type); + EXPECT_EQ(ZoneMapFilterResult::kNoMatch, + equals->evaluate_dictionary_filter(dictionary_ctx, {slot, make_int_literal(2)})); + EXPECT_EQ(ZoneMapFilterResult::kMayMatch, + equals->evaluate_dictionary_filter(dictionary_ctx, {slot, make_int_literal(3)})); + + EXPECT_TRUE(equals->can_evaluate_bloom_filter({slot, make_int_literal(2)})); + auto bloom_filter = make_int_bloom_filter({1, 3}); + auto bloom_ctx = make_bloom_filter_context(bloom_filter.get(), type); + EXPECT_EQ(ZoneMapFilterResult::kNoMatch, + equals->evaluate_bloom_filter(bloom_ctx, {slot, make_int_literal(2)})); + EXPECT_EQ(ZoneMapFilterResult::kMayMatch, + equals->evaluate_bloom_filter(bloom_ctx, {slot, make_int_literal(3)})); +} + TEST(ExprZonemapFilterTest, MissingSlotTypeCountsUnsupportedZonemapEvalOnce) { auto type = int_type(); auto slot = make_slot(0, type); @@ -642,6 +722,46 @@ TEST(ExprZonemapFilterTest, VInPredicateMaterializesZonemapValues) { EXPECT_TRUE(not_in_with_null->_seg_filter_contains_null); } +TEST(ExprZonemapFilterTest, VInPredicateDictionaryAndBloomUseMaterializedValues) { + auto type = int_type(); + ObjectPool obj_pool; + DescriptorTbl* desc_tbl = nullptr; + auto thrift_desc_tbl = make_k2_scan_desc_tbl(); + ASSERT_TRUE(DescriptorTbl::create(&obj_pool, thrift_desc_tbl, &desc_tbl).ok()); + + RuntimeState runtime_state; + runtime_state.set_desc_tbl(desc_tbl); + RowDescriptor row_desc(runtime_state.desc_tbl(), {0}, {false}); + + auto in_predicate = std::make_shared(make_in_predicate_node(false, 3)); + auto in_slot = make_slot(0, type); + std::static_pointer_cast(in_slot)->set_slot_id(0); + in_predicate->add_child(in_slot); + in_predicate->add_child(make_int_literal(2)); + in_predicate->add_child(make_int_literal(4)); + VExprContext in_context(in_predicate); + ASSERT_TRUE(in_context.prepare(&runtime_state, row_desc).ok()); + ASSERT_TRUE(in_context.open(&runtime_state).ok()); + + EXPECT_TRUE(in_predicate->can_evaluate_dictionary_filter()); + auto missing_dictionary_ctx = make_dictionary_context({int_field(1), int_field(3)}, type); + EXPECT_EQ(ZoneMapFilterResult::kNoMatch, + in_predicate->evaluate_dictionary_filter(missing_dictionary_ctx)); + auto matching_dictionary_ctx = make_dictionary_context({int_field(4), int_field(5)}, type); + EXPECT_EQ(ZoneMapFilterResult::kMayMatch, + in_predicate->evaluate_dictionary_filter(matching_dictionary_ctx)); + + EXPECT_TRUE(in_predicate->can_evaluate_bloom_filter()); + auto missing_bloom_filter = make_int_bloom_filter({1, 3}); + auto missing_bloom_ctx = make_bloom_filter_context(missing_bloom_filter.get(), type); + EXPECT_EQ(ZoneMapFilterResult::kNoMatch, + in_predicate->evaluate_bloom_filter(missing_bloom_ctx)); + auto matching_bloom_filter = make_int_bloom_filter({4}); + auto matching_bloom_ctx = make_bloom_filter_context(matching_bloom_filter.get(), type); + EXPECT_EQ(ZoneMapFilterResult::kMayMatch, + in_predicate->evaluate_bloom_filter(matching_bloom_ctx)); +} + TEST(ExprZonemapFilterTest, DirectInPredicateMaterializesStringSetForZonemap) { auto type = std::make_shared(); std::shared_ptr filter(create_set(PrimitiveType::TYPE_STRING, false)); diff --git a/be/test/exprs/try_cast_expr_test.cpp b/be/test/exprs/try_cast_expr_test.cpp index 24f73dbcb355d8..41565ab894c221 100644 --- a/be/test/exprs/try_cast_expr_test.cpp +++ b/be/test/exprs/try_cast_expr_test.cpp @@ -271,4 +271,12 @@ TEST_F(TryCastExprTest, row_exec3) { EXPECT_FALSE(st.ok()) << st.msg(); } -} // namespace doris \ No newline at end of file +TEST_F(TryCastExprTest, selected_row_safety) { + VCastExpr cast_expr; + cast_expr.add_child(std::make_shared()); + + EXPECT_FALSE(cast_expr.is_safe_to_execute_on_selected_rows()); + EXPECT_TRUE(try_cast_expr.is_safe_to_execute_on_selected_rows()); +} + +} // namespace doris diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp new file mode 100644 index 00000000000000..06941b29884c72 --- /dev/null +++ b/be/test/format_v2/column_mapper_test.cpp @@ -0,0 +1,4211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/column_mapper.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vin_predicate.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/column_mapper_nested.h" +#include "format_v2/expr/cast.h" +#include "format_v2/schema_projection.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/Exprs_types.h" +#include "runtime/descriptors.h" +#include "testutil/column_helper.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris::format { +namespace { + +DataTypePtr i32() { + return std::make_shared(); +} + +DataTypePtr i64() { + return std::make_shared(); +} + +DataTypePtr f32() { + return std::make_shared(); +} + +DataTypePtr f64() { + return std::make_shared(); +} + +DataTypePtr dec32(uint32_t precision, uint32_t scale) { + return std::make_shared(precision, scale); +} + +DataTypePtr str() { + return std::make_shared(); +} + +DataTypePtr varbinary() { + return std::make_shared(); +} + +DataTypePtr timestamptz(uint32_t scale) { + return std::make_shared(scale); +} + +DataTypePtr u8() { + return std::make_shared(); +} + +ColumnDefinition field_id_col(const std::string& name, int32_t field_id, DataTypePtr type, + int32_t local_id = -1) { + ColumnDefinition column; + column.identifier = Field::create_field(field_id); + column.local_id = local_id; + column.name = name; + column.type = std::move(type); + return column; +} + +ColumnDefinition name_col(const std::string& name, DataTypePtr type, int32_t local_id = -1) { + ColumnDefinition column; + column.identifier = Field::create_field(name); + column.local_id = local_id; + column.name = name; + column.type = std::move(type); + return column; +} + +ColumnDefinition name_id_col(const std::string& name, const std::string& identifier, + DataTypePtr type, int32_t local_id = -1) { + ColumnDefinition column = name_col(name, std::move(type), local_id); + column.identifier = Field::create_field(identifier); + return column; +} + +ColumnDefinition position_col(const std::string& name, int32_t file_position, DataTypePtr type) { + return field_id_col(name, file_position, std::move(type)); +} + +ColumnDefinition struct_col(const std::string& name, int32_t field_id, + std::vector children, int32_t local_id = -1) { + DataTypes child_types; + Strings child_names; + child_types.reserve(children.size()); + child_names.reserve(children.size()); + for (const auto& child : children) { + child_types.push_back(child.type); + child_names.push_back(child.name); + } + auto column = field_id_col( + name, field_id, std::make_shared(child_types, child_names), local_id); + column.children = std::move(children); + return column; +} + +ColumnDefinition struct_name_col(const std::string& name, std::vector children, + int32_t local_id = -1) { + auto column = struct_col(name, -1, std::move(children), local_id); + column.identifier = Field::create_field(name); + return column; +} + +ColumnDefinition array_col(const std::string& name, int32_t field_id, ColumnDefinition element, + int32_t local_id = -1) { + auto column = + field_id_col(name, field_id, std::make_shared(element.type), local_id); + column.children = {std::move(element)}; + return column; +} + +ColumnDefinition map_col(const std::string& name, int32_t field_id, + std::vector children, const DataTypePtr& key_type, + const DataTypePtr& value_type, int32_t local_id = -1) { + auto column = field_id_col(name, field_id, std::make_shared(key_type, value_type), + local_id); + column.children = std::move(children); + return column; +} + +void set_name_identifiers(ColumnDefinition* column, int32_t local_id) { + DORIS_CHECK(column != nullptr); + column->identifier = Field::create_field(column->name); + column->local_id = local_id; + for (size_t idx = 0; idx < column->children.size(); ++idx) { + set_name_identifiers(&column->children[idx], static_cast(idx)); + } +} + +std::vector projection_ids(const std::vector& projections) { + std::vector ids; + ids.reserve(projections.size()); + for (const auto& projection : projections) { + ids.push_back(projection.local_id()); + } + return ids; +} + +TEST(ColumnMapperDebugTest, CoversDebugStringEnumAndNestedBranches) { + ColumnDefinition child = field_id_col("child", 2, str(), 3); + child.name_mapping = {"legacy_child"}; + + ColumnDefinition column = field_id_col( + "root", 1, + std::make_shared(DataTypes {child.type}, Strings {child.name})); + column.name_mapping = {"legacy_root"}; + column.children = {child}; + column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + std::make_shared(), Field::create_field("fallback"))); + column.is_partition_key = true; + + const auto column_debug = column.debug_string(); + EXPECT_NE(column_debug.find("ColumnDefinition{name=root"), std::string::npos); + EXPECT_NE(column_debug.find("name_mapping=[legacy_root]"), std::string::npos); + EXPECT_NE(column_debug.find("children=[ColumnDefinition{name=child"), std::string::npos); + EXPECT_NE(column_debug.find("has_default_expr=1"), std::string::npos); + EXPECT_NE(column_debug.find("is_partition_key=1"), std::string::npos); + + LocalColumnIndex projection = LocalColumnIndex::partial_local(4); + projection.children.push_back(LocalColumnIndex::local(7)); + EXPECT_NE(projection.debug_string().find("children=[LocalColumnIndex{index=7"), + std::string::npos); + + const std::vector modes {TableColumnMappingMode::BY_FIELD_ID, + TableColumnMappingMode::BY_NAME, + TableColumnMappingMode::BY_INDEX}; + const std::vector mode_names {"BY_FIELD_ID", "BY_NAME", "BY_INDEX"}; + for (size_t idx = 0; idx < modes.size(); ++idx) { + TableColumnMapperOptions options {.mode = modes[idx]}; + EXPECT_NE(options.debug_string().find(mode_names[idx]), std::string::npos); + } + + const std::vector conversions { + FilterConversionType::COPY_DIRECTLY, FilterConversionType::CAST_FILTER, + FilterConversionType::READER_EXPRESSION, FilterConversionType::FINALIZE_ONLY, + FilterConversionType::CONSTANT}; + const std::vector conversion_names { + "COPY_DIRECTLY", "CAST_FILTER", "READER_EXPRESSION", "FINALIZE_ONLY", "CONSTANT"}; + for (size_t idx = 0; idx < conversions.size(); ++idx) { + ColumnMapping mapping; + mapping.global_index = GlobalIndex(idx); + mapping.table_column_name = "table_col"; + mapping.file_local_id = 8; + mapping.constant_index = ConstantIndex(9); + mapping.file_column_name = "file_col"; + mapping.original_file_type = str(); + mapping.original_file_children = {child}; + mapping.file_type = str(); + mapping.table_type = str(); + mapping.is_trivial = idx % 2 == 0; + mapping.filter_conversion = conversions[idx]; + mapping.virtual_column_type = static_cast( + idx % (TableVirtualColumnType::ICEBERG_ROWID + 1)); + mapping.default_expr = column.default_expr; + + ColumnMapping child_mapping; + child_mapping.global_index = GlobalIndex(10 + idx); + child_mapping.table_column_name = "child_col"; + child_mapping.file_column_name = "child_file"; + child_mapping.file_type = i32(); + child_mapping.table_type = i32(); + mapping.child_mappings.push_back(std::move(child_mapping)); + + const auto debug = mapping.debug_string(); + EXPECT_NE(debug.find("file_local_id=8"), std::string::npos); + EXPECT_NE(debug.find("constant_index=9"), std::string::npos); + EXPECT_NE(debug.find(conversion_names[idx]), std::string::npos); + EXPECT_NE(debug.find("child_mappings=[ColumnMapping{global_index="), std::string::npos); + EXPECT_NE(debug.find("has_default_expr=1"), std::string::npos); + } +} + +void expect_mapping(const ColumnMapping& mapping, size_t global_index, + const std::string& table_name, int32_t file_local_id, + const std::string& file_name, const DataTypePtr& file_type, + const DataTypePtr& table_type) { + EXPECT_EQ(mapping.global_index, GlobalIndex(global_index)); + EXPECT_EQ(mapping.table_column_name, table_name); + ASSERT_TRUE(mapping.file_local_id.has_value()); + EXPECT_EQ(*mapping.file_local_id, file_local_id); + EXPECT_EQ(mapping.file_column_name, file_name); + ASSERT_NE(mapping.file_type, nullptr); + ASSERT_NE(mapping.table_type, nullptr); + EXPECT_TRUE(mapping.file_type->equals(*file_type)); + EXPECT_TRUE(mapping.table_type->equals(*table_type)); +} + +void expect_constant(const TableColumnMapper& mapper, const ColumnMapping& mapping, + size_t global_index, const DataTypePtr& table_type) { + EXPECT_FALSE(mapping.file_local_id.has_value()); + ASSERT_TRUE(mapping.constant_index.has_value()); + ASSERT_LT(mapping.constant_index->value(), mapper.constant_map().size()); + const auto& entry = mapper.constant_map().get(*mapping.constant_index); + EXPECT_EQ(entry.global_index, GlobalIndex(global_index)); + EXPECT_TRUE(entry.type->equals(*table_type)); + EXPECT_EQ(entry.expr, mapping.default_expr); +} + +void expect_missing(const ColumnMapping& mapping) { + EXPECT_FALSE(mapping.file_local_id.has_value()); + EXPECT_FALSE(mapping.constant_index.has_value()); + EXPECT_EQ(mapping.virtual_column_type, TableVirtualColumnType::INVALID); +} + +class TestFunctionExpr final : public VExpr { +public: + TestFunctionExpr(std::string function_name, DataTypePtr data_type, + TExprNodeType::type node_type = TExprNodeType::FUNCTION_CALL, + TExprOpcode::type opcode = TExprOpcode::INVALID_OPCODE) + : VExpr(std::move(data_type), false), _expr_name(std::move(function_name)) { + set_node_type(node_type); + _opcode = opcode; + TFunctionName fn_name; + fn_name.__set_function_name(_expr_name); + _fn.__set_name(fn_name); + } + + const std::string& expr_name() const override { return _expr_name; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = + std::make_shared(_expr_name, data_type(), node_type(), _opcode); + return Status::OK(); + } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::NotSupported("TestFunctionExpr is only used for ColumnMapper analysis"); + } + +private: + std::string _expr_name; +}; + +VExprSPtr table_slot(int slot_id, int column_id, DataTypePtr type, const std::string& name) { + return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name); +} + +VExprSPtr literal(DataTypePtr type, Field value) { + return VLiteral::create_shared(std::move(type), std::move(value)); +} + +VExprSPtr struct_element(const VExprSPtr& parent, DataTypePtr child_type, + const std::string& child_name) { + auto expr = std::make_shared("struct_element", child_type); + expr->add_child(parent); + expr->add_child(literal(str(), Field::create_field(child_name))); + return expr; +} + +VExprSPtr element_at(const VExprSPtr& parent, DataTypePtr child_type, + const std::string& child_name) { + auto expr = std::make_shared("element_at", std::move(child_type)); + expr->add_child(parent); + expr->add_child(literal(str(), Field::create_field(child_name))); + return expr; +} + +VExprSPtr array_element_at(const VExprSPtr& parent, DataTypePtr child_type, int64_t ordinal) { + auto expr = std::make_shared("element_at", std::move(child_type)); + expr->add_child(parent); + expr->add_child(literal(i64(), Field::create_field(ordinal))); + return expr; +} + +VExprSPtr map_values(const VExprSPtr& parent, DataTypePtr value_type) { + auto expr = std::make_shared( + "map_values", std::make_shared(std::move(value_type))); + expr->add_child(parent); + return expr; +} + +VExprSPtr map_keys(const VExprSPtr& parent, DataTypePtr key_type) { + auto expr = std::make_shared( + "map_keys", std::make_shared(std::move(key_type))); + expr->add_child(parent); + return expr; +} + +VExprSPtr array_contains(const VExprSPtr& array, const VExprSPtr& value) { + auto expr = std::make_shared("array_contains", u8()); + expr->add_child(array); + expr->add_child(value); + return expr; +} + +VExprSPtr like_expr(const VExprSPtr& left, const std::string& pattern) { + auto expr = std::make_shared("like", u8()); + expr->add_child(left); + expr->add_child(literal(str(), Field::create_field(pattern))); + return expr; +} + +VExprSPtr struct_element_by_selector(const VExprSPtr& parent, DataTypePtr child_type, + const VExprSPtr& selector) { + auto expr = std::make_shared("struct_element", std::move(child_type)); + expr->add_child(parent); + expr->add_child(selector); + return expr; +} + +VExprSPtr int_gt(const VExprSPtr& left, int32_t value) { + auto expr = std::make_shared("gt", u8(), TExprNodeType::BINARY_PRED, + TExprOpcode::GT); + expr->add_child(left); + expr->add_child(literal(i32(), Field::create_field(value))); + return expr; +} + +VExprSPtr binary_predicate(TExprOpcode::type opcode, const VExprSPtr& left, + const VExprSPtr& right) { + auto expr = std::make_shared("binary_predicate", u8(), + TExprNodeType::BINARY_PRED, opcode); + expr->add_child(left); + expr->add_child(right); + return expr; +} + +VExprSPtr in_predicate(const VExprSPtr& probe, const DataTypePtr& literal_type, + const std::vector& values) { + auto expr = std::make_shared("in", u8(), TExprNodeType::IN_PRED); + expr->add_child(probe); + for (const auto& value : values) { + expr->add_child(literal(literal_type, value)); + } + return expr; +} + +VExprSPtr null_predicate(const VExprSPtr& child, bool is_null) { + auto expr = + std::make_shared(is_null ? "is_null_pred" : "is_not_null_pred", u8()); + expr->add_child(child); + return expr; +} + +VExprSPtr cast_expr(const VExprSPtr& child, DataTypePtr target_type) { + auto expr = Cast::create_shared(std::move(target_type)); + expr->add_child(child); + return expr; +} + +VExprSPtr compound_predicate(TExprOpcode::type opcode, const VExprSPtr& left, + const VExprSPtr& right) { + auto expr = std::make_shared("compound", u8(), TExprNodeType::COMPOUND_PRED, + opcode); + expr->add_child(left); + expr->add_child(right); + return expr; +} + +std::vector collect_paths(const VExprSPtr& expr) { + std::vector paths; + collect_nested_struct_paths(expr, &paths); + return paths; +} + +void expect_name_selector(const StructChildSelector& selector, const std::string& name) { + EXPECT_TRUE(selector.by_name); + EXPECT_EQ(selector.name, name); +} + +void expect_ordinal_selector(const StructChildSelector& selector, size_t ordinal) { + EXPECT_FALSE(selector.by_name); + EXPECT_EQ(selector.ordinal, ordinal); +} + +void expect_path_root(const NestedStructPath& path, size_t global_index) { + EXPECT_EQ(path.root_global_index, GlobalIndex(global_index)); +} + +class ColumnMapperCastTest : public testing::Test { +protected: + void SetUp() override { state.set_enable_strict_cast(true); } + + Status prepare_open_execute(VExprContext* context, Block* block, int* result_column_id) { + RETURN_IF_ERROR(context->prepare(&state, RowDescriptor())); + RETURN_IF_ERROR(context->open(&state)); + return context->execute(block, result_column_id); + } + + MockRuntimeState state; +}; + +class Int64ChildGreaterThanExpr final : public VExpr { +public: + explicit Int64ChildGreaterThanExpr(int64_t value) + : VExpr(std::make_shared(), false), _value(value) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr child_column; + RETURN_IF_ERROR( + get_child(0)->execute_column_impl(context, block, selector, count, child_column)); + const auto& input = assert_cast(*child_column); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + result_data[row] = input.get_element(row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_value); + return Status::OK(); + } + +private: + const int64_t _value; + const std::string _expr_name = "Int64ChildGreaterThanExpr"; +}; + +class Int64BinaryPredicateExpr final : public VExpr { +public: + explicit Int64BinaryPredicateExpr(TExprOpcode::type opcode) + : VExpr(std::make_shared(), false) { + set_node_type(TExprNodeType::BINARY_PRED); + _opcode = opcode; + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr left_column; + RETURN_IF_ERROR( + get_child(0)->execute_column_impl(context, block, selector, count, left_column)); + ColumnPtr right_column; + RETURN_IF_ERROR( + get_child(1)->execute_column_impl(context, block, selector, count, right_column)); + + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto left = left_column->get_int(row); + const auto right = right_column->get_int(row); + switch (_opcode) { + case TExprOpcode::GT: + result_data[row] = left > right; + break; + case TExprOpcode::LT: + result_data[row] = left < right; + break; + default: + return Status::InternalError("Unsupported test opcode {}", _opcode); + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_opcode); + return Status::OK(); + } + +private: + const std::string _expr_name = "Int64BinaryPredicateExpr"; +}; + +VExprSPtr create_in_predicate() { + TExprNode node; + node.__set_node_type(TExprNodeType::IN_PRED); + node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); + node.__set_is_nullable(false); + node.__set_num_children(0); + TInPredicate in_predicate; + in_predicate.__set_is_not_in(false); + node.__set_in_predicate(in_predicate); + return VInPredicate::create_shared(node); +} + +// ---------------------------------------------------------------------- +// L0 schema projection helper tests. +// These tests isolate LocalColumnIndex projection semantics before +// TableColumnMapper starts mutating ColumnMapping state. +// ---------------------------------------------------------------------- + +TEST(ColumnMapperSchemaProjectionTest, ProjectsStructByLocalIdAndKeepsFileOrder) { + auto a = field_id_col("a", 101, i32(), 0); + auto b = field_id_col("b", 102, str(), 1); + auto root = struct_col("s", 100, {a, b}, 7); + + LocalColumnIndex projection = LocalColumnIndex::partial_local(7); + projection.children.push_back(LocalColumnIndex::local(1)); + projection.children.push_back(LocalColumnIndex::local(0)); + + ColumnDefinition projected; + ASSERT_TRUE(project_column_definition(root, projection, &projected).ok()); + ASSERT_EQ(projected.children.size(), 2); + EXPECT_EQ(projected.children[0].name, "a"); + EXPECT_EQ(projected.children[1].name, "b"); + + const auto* projected_type = + assert_cast(remove_nullable(projected.type).get()); + ASSERT_EQ(projected_type->get_elements().size(), 2); + EXPECT_EQ(projected_type->get_element_name(0), "a"); + EXPECT_EQ(projected_type->get_element_name(1), "b"); +} + +TEST(ColumnMapperSchemaProjectionTest, ProjectsArrayElementStructLeaf) { + auto a = field_id_col("a", 1, i32(), 0); + auto b = field_id_col("b", 2, str(), 1); + auto element = struct_col("element", 10, {a, b}, 0); + auto array = array_col("items", 100, element, 5); + + LocalColumnIndex projection = LocalColumnIndex::partial_local(5); + auto element_projection = LocalColumnIndex::partial_local(0); + element_projection.children.push_back(LocalColumnIndex::local(1)); + projection.children.push_back(std::move(element_projection)); + + ColumnDefinition projected; + ASSERT_TRUE(project_column_definition(array, projection, &projected).ok()); + ASSERT_EQ(projected.children.size(), 1); + ASSERT_EQ(projected.children[0].children.size(), 1); + EXPECT_EQ(projected.children[0].children[0].name, "b"); + + const auto* array_type = + assert_cast(remove_nullable(projected.type).get()); + const auto* element_type = assert_cast( + remove_nullable(array_type->get_nested_type()).get()); + ASSERT_EQ(element_type->get_elements().size(), 1); + EXPECT_EQ(element_type->get_element_name(0), "b"); +} + +TEST(ColumnMapperSchemaProjectionTest, ProjectsMapValueStructLeaf) { + auto key = field_id_col("key", 1, str(), 0); + auto value_a = field_id_col("a", 2, i32(), 0); + auto value_b = field_id_col("b", 3, str(), 1); + auto value_type = + std::make_shared(DataTypes {i32(), str()}, Strings {"a", "b"}); + ColumnDefinition value = field_id_col("value", 4, value_type, 1); + value.children = {value_a, value_b}; + auto map = map_col("m", 100, {key, value}, str(), value_type, 9); + + LocalColumnIndex projection = LocalColumnIndex::partial_local(9); + projection.children.push_back(LocalColumnIndex::local(0)); + auto value_projection = LocalColumnIndex::partial_local(1); + value_projection.children.push_back(LocalColumnIndex::local(1)); + projection.children.push_back(std::move(value_projection)); + + ColumnDefinition projected; + ASSERT_TRUE(project_column_definition(map, projection, &projected).ok()); + ASSERT_EQ(projected.children.size(), 2); + EXPECT_EQ(projected.children[0].name, "key"); + EXPECT_TRUE(projected.children[0].children.empty()); + EXPECT_EQ(projected.children[1].name, "value"); + ASSERT_EQ(projected.children[1].children.size(), 1); + EXPECT_EQ(projected.children[1].children[0].name, "b"); + + const auto* map_type = assert_cast(remove_nullable(projected.type).get()); + const auto* projected_value = + assert_cast(remove_nullable(map_type->get_value_type()).get()); + ASSERT_EQ(projected_value->get_elements().size(), 1); + EXPECT_EQ(projected_value->get_element_name(0), "b"); +} + +TEST(ColumnMapperSchemaProjectionTest, RejectsMapKeyOnlyProjection) { + auto key = field_id_col("key", 1, str(), 0); + auto value = field_id_col("value", 2, i32(), 1); + auto map = map_col("m", 100, {key, value}, str(), i32(), 9); + + LocalColumnIndex projection = LocalColumnIndex::partial_local(9); + projection.children.push_back(LocalColumnIndex::local(0)); + + ColumnDefinition projected; + const auto status = project_column_definition(map, projection, &projected); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains no value child"), std::string::npos); +} + +TEST(ColumnMapperSchemaProjectionTest, RejectsInvalidProjectionChildIdWithFieldName) { + auto root = struct_col("s", 100, {field_id_col("a", 101, i32(), 0)}, 7); + + LocalColumnIndex projection = LocalColumnIndex::partial_local(7); + projection.children.push_back(LocalColumnIndex::local(99)); + + ColumnDefinition projected; + const auto status = project_column_definition(root, projection, &projected); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Invalid projection child id 99 for field s"), + std::string::npos); +} + +TEST(ColumnMapperSchemaProjectionTest, RejectsEmptyProjectionPathWithFieldName) { + auto root = struct_col("s", 100, {field_id_col("a", 101, i32(), 0)}, 7); + + LocalColumnIndex projection = LocalColumnIndex::partial_local(7); + projection.children.push_back(LocalColumnIndex::local(-1)); + + ColumnDefinition projected; + const auto status = project_column_definition(root, projection, &projected); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Empty projection path for field s"), std::string::npos); +} + +TEST(ColumnMapperSchemaProjectionTest, RejectsInvalidChildProjectionForPrimitiveField) { + auto root = field_id_col("i", 1, i32(), 7); + LocalColumnIndex projection = LocalColumnIndex::partial_local(7); + projection.children.push_back(LocalColumnIndex::local(0)); + + ColumnDefinition projected; + const auto status = project_column_definition(root, projection, &projected); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Invalid projection child id 0 for field i"), + std::string::npos); +} + +// ---------------------------------------------------------------------- +// L0 nested helper tests. +// These tests cover child ordering, direct schema path resolution, and +// predicate-filter merging without going through create_scan_request(). +// ---------------------------------------------------------------------- + +TEST(ColumnMapperNestedHelperTest, PresentChildMappingsAreSortedByFileLocalId) { + ColumnMapping b; + b.table_column_name = "b"; + b.file_local_id = 2; + ColumnMapping missing; + missing.table_column_name = "missing"; + ColumnMapping a; + a.table_column_name = "a"; + a.file_local_id = 1; + + const std::vector child_mappings = {b, missing, a}; + const auto present = present_child_mappings_in_file_order(child_mappings); + ASSERT_EQ(present.size(), 2); + EXPECT_EQ(present[0]->table_column_name, "a"); + EXPECT_EQ(present[1]->table_column_name, "b"); +} + +TEST(ColumnMapperNestedHelperTest, BuildsProjectionByNameAndOrdinalSelectors) { + auto leaf = field_id_col("leaf", 3, i32(), 0); + auto nested = struct_col("nested", 2, {leaf}, 1); + auto first = field_id_col("first", 1, str(), 0); + const std::vector children = {first, nested}; + + const std::vector by_name = { + {.by_name = true, .name = "nested", .ordinal = 0}, + {.by_name = true, .name = "leaf", .ordinal = 0}, + }; + LocalColumnIndex named_projection; + ASSERT_TRUE(build_file_child_projection_from_schema(children, by_name, &named_projection).ok()); + EXPECT_EQ(named_projection.local_id(), 1); + ASSERT_EQ(named_projection.children.size(), 1); + EXPECT_EQ(named_projection.children[0].local_id(), 0); + + const std::vector by_ordinal = { + {.by_name = false, .name = "", .ordinal = 2}, + {.by_name = false, .name = "", .ordinal = 1}, + }; + LocalColumnIndex ordinal_projection; + ASSERT_TRUE(build_file_child_projection_from_schema(children, by_ordinal, &ordinal_projection) + .ok()); + EXPECT_EQ(ordinal_projection.local_id(), 1); + ASSERT_EQ(ordinal_projection.children.size(), 1); + EXPECT_EQ(ordinal_projection.children[0].local_id(), 0); +} + +// ---------------------------------------------------------------------- +// collect_nested_struct_paths() helper tests. +// These tests assert the entry helper for nested scan projection: it only discovers +// table-side struct paths. Later localization decides how to add scan projections. +// ---------------------------------------------------------------------- + +TEST(ColumnMapperCollectNestedStructPathsTest, CollectsNameOrdinalAndBooleanSelectors) { + const auto leaf_type = i32(); + const auto inner_type = + std::make_shared(DataTypes {leaf_type, leaf_type}, Strings {"x", "y"}); + const auto root_type = std::make_shared(DataTypes {inner_type, leaf_type}, + Strings {"nested", "missing"}); + const auto root = table_slot(0, 3, root_type, "s"); + + const auto nested_by_ordinal = struct_element_by_selector( + struct_element_by_selector(root, inner_type, + literal(i32(), Field::create_field(1))), + leaf_type, literal(i32(), Field::create_field(2))); + auto paths = collect_paths(nested_by_ordinal); + ASSERT_EQ(paths.size(), 1); + expect_path_root(paths[0], 3); + ASSERT_EQ(paths[0].selectors.size(), 2); + expect_ordinal_selector(paths[0].selectors[0], 1); + expect_ordinal_selector(paths[0].selectors[1], 2); + + const std::vector positive_ordinal_selectors = { + literal(std::make_shared(), + Field::create_field(static_cast(1))), + literal(std::make_shared(), + Field::create_field(static_cast(2))), + literal(i32(), Field::create_field(3)), + literal(i64(), Field::create_field(4)), + literal(u8(), Field::create_field(true)), + }; + for (size_t idx = 0; idx < positive_ordinal_selectors.size(); ++idx) { + const auto selected = + struct_element_by_selector(root, leaf_type, positive_ordinal_selectors[idx]); + paths = collect_paths(selected); + ASSERT_EQ(paths.size(), 1); + ASSERT_EQ(paths[0].selectors.size(), 1); + expect_ordinal_selector(paths[0].selectors[0], idx == 4 ? 1 : idx + 1); + } + + paths = collect_paths(struct_element(root, leaf_type, "missing")); + ASSERT_EQ(paths.size(), 1); + ASSERT_EQ(paths[0].selectors.size(), 1); + expect_name_selector(paths[0].selectors[0], "missing"); +} + +TEST(ColumnMapperCollectNestedStructPathsTest, IgnoresInvalidSelectorsAndNonPathRoots) { + const auto leaf_type = i32(); + const auto root_type = std::make_shared(DataTypes {leaf_type}, Strings {"a"}); + const auto root = table_slot(0, 0, root_type, "s"); + + const std::vector invalid_selectors = { + literal(i32(), Field::create_field(0)), + literal(i32(), Field::create_field(-1)), + literal(u8(), Field::create_field(false)), + literal(f32(), Field::create_field(1.0F)), + literal(f64(), Field::create_field(1.0)), + table_slot(1, 1, i32(), "selector"), + }; + for (const auto& selector : invalid_selectors) { + EXPECT_TRUE(collect_paths(struct_element_by_selector(root, leaf_type, selector)).empty()); + } + + auto wrong_arity = std::make_shared("struct_element", leaf_type); + wrong_arity->add_child(root); + EXPECT_TRUE(collect_paths(wrong_arity).empty()); + + auto not_struct_element = std::make_shared("other_function", leaf_type); + not_struct_element->add_child(root); + not_struct_element->add_child(literal(str(), Field::create_field("a"))); + EXPECT_TRUE(collect_paths(not_struct_element).empty()); + + EXPECT_TRUE(collect_paths(struct_element(literal(str(), Field::create_field("x")), + leaf_type, "a")) + .empty()); + EXPECT_TRUE(collect_paths(nullptr).empty()); +} + +TEST(ColumnMapperCollectNestedStructPathsTest, RecursesThroughExpressionsAndKeepsCompletePath) { + const auto leaf_type = i32(); + const auto inner_type = std::make_shared(DataTypes {leaf_type}, Strings {"b"}); + const auto root_type = + std::make_shared(DataTypes {inner_type, leaf_type}, Strings {"a", "c"}); + const auto root = table_slot(0, 2, root_type, "s"); + const auto path_a = struct_element_by_selector( + root, inner_type, literal(str(), Field::create_field("a"))); + const auto path_ab = struct_element_by_selector( + path_a, leaf_type, literal(str(), Field::create_field("b"))); + const auto path_c = struct_element_by_selector( + root, leaf_type, literal(str(), Field::create_field("c"))); + + auto paths = collect_paths(binary_predicate( + TExprOpcode::GT, path_ab, literal(leaf_type, Field::create_field(1)))); + ASSERT_EQ(paths.size(), 1); + expect_path_root(paths[0], 2); + ASSERT_EQ(paths[0].selectors.size(), 2); + expect_name_selector(paths[0].selectors[0], "a"); + expect_name_selector(paths[0].selectors[1], "b"); + + paths = collect_paths(compound_predicate( + TExprOpcode::COMPOUND_OR, + binary_predicate(TExprOpcode::GT, path_ab, + literal(leaf_type, Field::create_field(1))), + binary_predicate(TExprOpcode::LT, path_c, + literal(leaf_type, Field::create_field(2))))); + ASSERT_EQ(paths.size(), 2); + ASSERT_EQ(paths[0].selectors.size(), 2); + ASSERT_EQ(paths[1].selectors.size(), 1); + expect_name_selector(paths[0].selectors[0], "a"); + expect_name_selector(paths[0].selectors[1], "b"); + expect_name_selector(paths[1].selectors[0], "c"); + + auto fn = std::make_shared("fn", leaf_type); + fn->add_child(path_ab); + fn->add_child(table_slot(3, 4, leaf_type, "other")); + paths = collect_paths(fn); + ASSERT_EQ(paths.size(), 1); + ASSERT_EQ(paths[0].selectors.size(), 2); + + auto if_expr = std::make_shared("if", leaf_type); + if_expr->add_child(literal(u8(), Field::create_field(true))); + if_expr->add_child(path_ab); + if_expr->add_child(path_c); + paths = collect_paths(if_expr); + ASSERT_EQ(paths.size(), 2); + + paths = collect_paths(compound_predicate(TExprOpcode::COMPOUND_AND, path_ab, path_ab)); + ASSERT_EQ(paths.size(), 2); + + paths = collect_paths(path_ab); + ASSERT_EQ(paths.size(), 1); + ASSERT_EQ(paths[0].selectors.size(), 2); +} + +TEST(ColumnMapperCollectNestedStructPathsTest, CastBehaviorSeparatesProjectionAndPruningRules) { + const auto int_type = i32(); + const auto bigint_type = i64(); + const auto float_type = f32(); + const auto double_type = f64(); + const auto decimal_small = dec32(8, 2); + const auto decimal_wide = dec32(9, 2); + const auto decimal_changed_scale = dec32(9, 3); + + const auto root_type = std::make_shared( + DataTypes {int_type, float_type, decimal_small}, Strings {"i", "f", "d"}); + const auto root = table_slot(0, 0, root_type, "s"); + const auto int_path = struct_element(root, int_type, "i"); + const auto float_path = struct_element(root, float_type, "f"); + const auto decimal_path = struct_element(root, decimal_small, "d"); + + auto paths = collect_paths(cast_expr(int_path, bigint_type)); + ASSERT_EQ(paths.size(), 1); + expect_name_selector(paths[0].selectors[0], "i"); + + paths = collect_paths(cast_expr(float_path, double_type)); + ASSERT_EQ(paths.size(), 1); + expect_name_selector(paths[0].selectors[0], "f"); + + paths = collect_paths(cast_expr(decimal_path, decimal_wide)); + ASSERT_EQ(paths.size(), 1); + expect_name_selector(paths[0].selectors[0], "d"); + + paths = collect_paths( + cast_expr(struct_element(root, make_nullable(int_type), "i"), make_nullable(int_type))); + ASSERT_EQ(paths.size(), 1); + expect_name_selector(paths[0].selectors[0], "i"); + + // Unsafe casts are not accepted as pruning paths, but collect_nested_struct_paths() still + // recurses into children so scan projection can read the column needed by row-level filters. + paths = collect_paths(cast_expr(struct_element(root, bigint_type, "i"), int_type)); + ASSERT_EQ(paths.size(), 1); + expect_name_selector(paths[0].selectors[0], "i"); + + paths = collect_paths(cast_expr(decimal_path, decimal_changed_scale)); + ASSERT_EQ(paths.size(), 1); + expect_name_selector(paths[0].selectors[0], "d"); + + EXPECT_TRUE(collect_paths(cast_expr(table_slot(1, 1, int_type, "plain"), bigint_type)).empty()); +} + +TEST(ColumnMapperCollectNestedStructPathsTest, ProjectionMergeKeepsFilterOnlyPathAndDeduplicates) { + const auto int_type = i32(); + const auto string_type = str(); + auto table_a = name_col("a", int_type); + auto table_b = name_col("b", int_type); + auto table_output = struct_name_col("s", {table_a}); + auto full_table_struct = struct_name_col("s", {table_a, table_b}); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", int_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b, name_col("c", string_type, 2)}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_output}, {}, {file_struct}).ok()); + + const auto path_b = + struct_element(table_slot(0, 0, full_table_struct.type, "s"), int_type, "b"); + auto filter_expr = compound_predicate( + TExprOpcode::COMPOUND_AND, + binary_predicate(TExprOpcode::GT, path_b, + literal(int_type, Field::create_field(1))), + binary_predicate(TExprOpcode::LT, path_b, + literal(int_type, Field::create_field(10)))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_output}, &request).ok()); + + EXPECT_TRUE(request.non_predicate_columns.empty()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(5)); + ASSERT_FALSE(request.predicate_columns[0].project_all_children); + EXPECT_EQ(projection_ids(request.predicate_columns[0].children), std::vector({0, 1})); +} + +// Scenario: row-oriented readers such as CSV/Text cannot lazy-read predicate columns separately. +// For a complex root that is both projected and referenced by a filter, the materialized mapper +// keeps one non-predicate scan entry and asks the reader to read the full top-level struct. +TEST(ColumnMapperScanRequestTest, MaterializedMapperUsesSingleScanColumnList) { + const auto int_type = i32(); + const auto string_type = str(); + auto table_a = name_col("a", int_type, 0); + auto table_b = name_col("b", int_type, 1); + auto full_table_struct = struct_name_col("s", {table_a, table_b}); + auto table_output = struct_name_col("s", {table_a}); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", int_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b, name_col("c", string_type, 2)}, 5); + + MaterializedColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_output}, {}, {file_struct}).ok()); + + const auto path_b = + struct_element(table_slot(0, 0, full_table_struct.type, "s"), int_type, "b"); + auto filter_expr = binary_predicate(TExprOpcode::GT, path_b, + literal(int_type, Field::create_field(1))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_output}, &request).ok()); + + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.non_predicate_columns[0].project_all_children); + EXPECT_TRUE(request.non_predicate_columns[0].children.empty()); +} + +// Scenario: a FileReader must expose semantic children for complex file columns. If it returns a +// complex DataType but leaves ColumnDefinition::children empty, mapper should return a diagnostic +// error instead of aborting inside ARRAY/MAP/STRUCT child lookup. +TEST(ColumnMapperScanRequestTest, MalformedComplexFileSchemaReturnsError) { + const auto int_type = i32(); + const auto string_type = str(); + auto table_a = name_col("a", int_type, 0); + auto table_b = name_col("b", string_type, 1); + auto table_struct = struct_name_col("s", {table_a, table_b}); + auto file_struct_type = + std::make_shared(DataTypes {int_type, string_type}, Strings {"a", "b"}); + auto malformed_file_struct = name_col("s", file_struct_type, 5); + + MaterializedColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + const auto status = mapper.create_mapping({table_struct}, {}, {malformed_file_struct}); + + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Malformed complex file schema"), std::string::npos) + << status; +} + +// Scenario: when the projected table schema contains the child referenced by the filter, the +// materialized mapper can still rewrite the table-level struct child predicate into a file-local +// conjunct. It remains a single full-root scan column; only the expression is localized. +TEST(ColumnMapperScanRequestTest, MaterializedMapperLocalizesMappedStructChildConjunct) { + const auto int_type = i32(); + const auto string_type = str(); + auto table_a = name_col("a", int_type, 0); + auto table_b = name_col("b", int_type, 1); + auto table_struct = struct_name_col("s", {table_a, table_b}); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", int_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b, name_col("c", string_type, 2)}, 5); + + MaterializedColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + const auto path_b = struct_element(table_slot(0, 0, table_struct.type, "s"), int_type, "b"); + auto filter_expr = binary_predicate(TExprOpcode::GT, path_b, + literal(int_type, Field::create_field(1))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); + + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.non_predicate_columns[0].project_all_children); + EXPECT_TRUE(request.non_predicate_columns[0].children.empty()); + ASSERT_EQ(request.conjuncts.size(), 1); +} + +// Scenario: even output-only partial complex projections such as `SELECT s.a` must scan the full +// top-level struct for materialized readers, because delimited text formats cannot physically read +// only one nested child from a single text field. +TEST(ColumnMapperScanRequestTest, MaterializedMapperScansFullComplexRootForOutputOnlyProjection) { + const auto int_type = i32(); + const auto string_type = str(); + auto table_a = name_col("a", int_type, 0); + auto table_output = struct_name_col("s", {table_a}); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", int_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b, name_col("c", string_type, 2)}, 5); + + MaterializedColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_output}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_output}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.non_predicate_columns[0].project_all_children); + EXPECT_TRUE(request.non_predicate_columns[0].children.empty()); + EXPECT_TRUE(request.predicate_columns.empty()); +} + +// Scenario: array/map nested projections also scan the full top-level complex root for +// materialized readers. This keeps row-oriented formats from receiving Parquet-style partial +// projections for `array` elements or map value structs. +TEST(ColumnMapperScanRequestTest, MaterializedMapperScansFullArrayAndMapRoots) { + const auto key_type = str(); + const auto int_type = i32(); + const auto string_type = str(); + + auto table_array_child = name_col("b", string_type); + auto table_array_element = struct_name_col("element", {table_array_child}); + auto table_array = array_col("items", -1, table_array_element); + table_array.identifier = Field::create_field("items"); + set_name_identifiers(&table_array, -1); + + auto file_array_a = name_col("a", int_type, 0); + auto file_array_b = name_col("b", string_type, 1); + auto file_array_element = struct_name_col("element", {file_array_a, file_array_b}, 0); + auto file_array = array_col("items", -1, file_array_element, 4); + file_array.identifier = Field::create_field("items"); + set_name_identifiers(&file_array, 4); + + auto table_value_b = name_col("b", string_type); + auto table_value = struct_name_col("value", {table_value_b}); + auto table_map = map_col("m", -1, {table_value}, key_type, table_value.type); + table_map.identifier = Field::create_field("m"); + set_name_identifiers(&table_map, -1); + + auto file_key = name_col("key", key_type, 0); + auto file_value_a = name_col("a", int_type, 0); + auto file_value_b = name_col("b", string_type, 1); + auto file_value = struct_name_col("value", {file_value_a, file_value_b}, 1); + auto file_map = map_col("m", -1, {file_key, file_value}, key_type, file_value.type, 6); + file_map.identifier = Field::create_field("m"); + set_name_identifiers(&file_map, 6); + + MaterializedColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_array, table_map}, {}, {file_array, file_map}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_array, table_map}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 2); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(4)); + EXPECT_TRUE(request.non_predicate_columns[0].project_all_children); + EXPECT_TRUE(request.non_predicate_columns[0].children.empty()); + EXPECT_EQ(request.non_predicate_columns[1].column_id(), LocalColumnId(6)); + EXPECT_TRUE(request.non_predicate_columns[1].project_all_children); + EXPECT_TRUE(request.non_predicate_columns[1].children.empty()); + EXPECT_TRUE(request.predicate_columns.empty()); +} + +// ---------------------------------------------------------------------- +// L1 create_mapping root matching tests. +// These cases cover the three supported root matching modes and the +// missing/default behavior that each mode feeds into later scan requests. +// ---------------------------------------------------------------------- + +TEST(ColumnMapperCreateMappingTest, ByNameMatchesCaseIdentifierAndAliases) { + const auto int_type = i32(); + const std::vector table_schema = { + name_col("ID", int_type), + name_id_col("renamed", "legacy_name", int_type), + [] { + auto column = name_col("current_alias", i32()); + column.name_mapping = {"old_alias"}; + return column; + }(), + name_col("file_alias", int_type), + }; + std::vector file_schema = { + name_col("id", int_type, 0), + name_col("legacy_name", int_type, 1), + name_col("old_alias", int_type, 2), + [] { + auto column = name_col("physical_name", i32(), 3); + column.name_mapping = {"file_alias"}; + return column; + }(), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 4); + expect_mapping(mapper.mappings()[0], 0, "ID", 0, "id", int_type, int_type); + expect_mapping(mapper.mappings()[1], 1, "renamed", 1, "legacy_name", int_type, int_type); + expect_mapping(mapper.mappings()[2], 2, "current_alias", 2, "old_alias", int_type, int_type); + expect_mapping(mapper.mappings()[3], 3, "file_alias", 3, "physical_name", int_type, int_type); +} + +TEST(ColumnMapperCreateMappingTest, ByNameUsesFirstMatchingFileFieldWhenAmbiguous) { + const auto int_type = i32(); + const std::vector table_schema = { + name_col("id", int_type), + }; + const std::vector file_schema = { + name_col("ID", int_type, 0), + name_col("id", int_type, 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "id", 0, "ID", int_type, int_type); +} + +TEST(ColumnMapperCreateMappingTest, TimestampTzScaleMismatchKeepsFilterAboveReader) { + // Scenario: HDFS TVF may expose a table slot as TIMESTAMPTZ(0), while a Parquet logical UTC + // timestamp file schema is materialized as TIMESTAMPTZ(6). Finalization must not add a SQL + // cast from scale 6 to scale 0, because that cast rounds fractional seconds: + // 2025-06-01 12:34:56.789+08:00 -> 2025-06-01 12:34:57+08:00 + // Reader finalization should pass the column through; the output slot type controls display + // scale and hides the fractional part without changing the stored instant. + const auto table_type = timestamptz(0); + const auto file_type = timestamptz(6); + const std::vector table_schema = {name_col("ts_tz", table_type)}; + const std::vector file_schema = {name_col("ts_tz", file_type, 0)}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "ts_tz", 0, "ts_tz", file_type, table_type); + EXPECT_TRUE(mapper.mappings()[0].is_trivial); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::FINALIZE_ONLY); + + TableFilter filter { + .conjunct = VExprContext::create_shared(table_slot(0, 0, table_type, "ts_tz")), + .global_indices = {GlobalIndex(0)}}; + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, table_schema, &request).ok()); + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +TEST(ColumnMapperCreateMappingTest, ByNameUsesNameMappingForRenamedColumn) { + const auto int_type = i32(); + auto table_column = name_col("current_id", int_type); + table_column.name_mapping = {"legacy_id"}; + const std::vector file_schema = { + name_col("legacy_id", int_type, 0), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_column}, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "current_id", 0, "legacy_id", int_type, int_type); +} + +TEST(ColumnMapperCreateMappingTest, ByNameUsesNameMappingForNestedSchemaEvolution) { + const auto int_type = i32(); + const auto string_type = str(); + + auto table_country = name_col("country", string_type); + table_country.name_mapping = {"old_country"}; + auto table_city = name_col("city", string_type); + auto table_struct = struct_name_col("struct_column", {table_country, table_city}); + set_name_identifiers(&table_struct, -1); + + auto table_item = name_col("item", string_type); + table_item.name_mapping = {"product"}; + auto table_quantity = name_col("quantity", int_type); + auto table_element = struct_name_col("element", {table_item, table_quantity}); + auto table_array = array_col("array_column", -1, table_element); + set_name_identifiers(&table_array, -1); + + auto table_key = name_col("key", string_type); + auto table_full_name = name_col("full_name", string_type); + table_full_name.name_mapping = {"name"}; + auto table_age = name_col("age", int_type); + auto table_value = struct_name_col("value", {table_full_name, table_age}); + auto table_map = + map_col("new_map_column", -1, {table_key, table_value}, string_type, table_value.type); + table_map.name_mapping = {"map_column"}; + set_name_identifiers(&table_map, -1); + + auto file_old_country = name_col("old_country", string_type, 0); + auto file_city = name_col("city", string_type, 1); + auto file_struct = struct_name_col("struct_column", {file_old_country, file_city}, 3); + set_name_identifiers(&file_struct, 3); + + auto file_product = name_col("product", string_type, 0); + auto file_element = struct_name_col("list", {file_product}, 0); + auto file_array = array_col("array_column", -1, file_element, 4); + set_name_identifiers(&file_array, 4); + + auto file_key = name_col("key", string_type, 0); + auto file_name = name_col("name", string_type, 0); + auto file_age = name_col("age", int_type, 1); + auto file_value = struct_name_col("value", {file_name, file_age}, 1); + auto file_map = + map_col("map_column", -1, {file_key, file_value}, string_type, file_value.type, 5); + set_name_identifiers(&file_map, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct, table_array, table_map}, {}, + {file_struct, file_array, file_map}) + .ok()); + + ASSERT_EQ(mapper.mappings().size(), 3); + const auto& struct_mapping = mapper.mappings()[0]; + expect_mapping(struct_mapping, 0, "struct_column", 3, "struct_column", file_struct.type, + table_struct.type); + ASSERT_EQ(struct_mapping.child_mappings.size(), 2); + EXPECT_EQ(struct_mapping.child_mappings[0].file_column_name, "old_country"); + EXPECT_EQ(*struct_mapping.child_mappings[0].file_local_id, 0); + EXPECT_EQ(struct_mapping.child_mappings[1].file_column_name, "city"); + EXPECT_EQ(*struct_mapping.child_mappings[1].file_local_id, 1); + + const auto& array_mapping = mapper.mappings()[1]; + expect_mapping(array_mapping, 1, "array_column", 4, "array_column", file_array.type, + table_array.type); + ASSERT_EQ(array_mapping.child_mappings.size(), 1); + const auto& element_mapping = array_mapping.child_mappings[0]; + EXPECT_EQ(element_mapping.file_column_name, "list"); + EXPECT_EQ(*element_mapping.file_local_id, 0); + ASSERT_EQ(element_mapping.child_mappings.size(), 2); + EXPECT_EQ(element_mapping.child_mappings[0].file_column_name, "product"); + EXPECT_EQ(*element_mapping.child_mappings[0].file_local_id, 0); + expect_missing(element_mapping.child_mappings[1]); + + const auto& map_mapping = mapper.mappings()[2]; + expect_mapping(map_mapping, 2, "new_map_column", 5, "map_column", file_map.type, + table_map.type); + ASSERT_EQ(map_mapping.child_mappings.size(), 2); + EXPECT_EQ(map_mapping.child_mappings[0].file_column_name, "key"); + EXPECT_EQ(*map_mapping.child_mappings[0].file_local_id, 0); + const auto& value_mapping = map_mapping.child_mappings[1]; + EXPECT_EQ(value_mapping.file_column_name, "value"); + EXPECT_EQ(*value_mapping.file_local_id, 1); + ASSERT_EQ(value_mapping.child_mappings.size(), 2); + EXPECT_EQ(value_mapping.child_mappings[0].file_column_name, "name"); + EXPECT_EQ(*value_mapping.child_mappings[0].file_local_id, 0); + EXPECT_EQ(value_mapping.child_mappings[1].file_column_name, "age"); + EXPECT_EQ(*value_mapping.child_mappings[1].file_local_id, 1); +} + +// Scenario: SELECT * can carry only the full complex DataType without expanded nested +// ColumnDefinitions. When an old file has map value STRUCT and the table type is +// STRUCT, the mapper must still build child mappings instead of letting +// TableReader cast between incompatible struct shapes. +TEST(ColumnMapperCreateMappingTest, SynthesizesMissingMapValueStructChildrenFromType) { + const auto int_type = i32(); + const auto string_type = str(); + const auto table_value_type = std::make_shared( + DataTypes {int_type, string_type, string_type}, Strings {"age", "full_name", "gender"}); + const auto file_value_type = std::make_shared(DataTypes {int_type, string_type}, + Strings {"age", "name"}); + + auto table_map = name_col("new_map_column", + std::make_shared(string_type, table_value_type)); + table_map.name_mapping = {"map_column"}; + set_name_identifiers(&table_map, -1); + + auto file_age = name_col("age", int_type, 0); + auto file_name = name_col("name", string_type, 1); + auto file_value = struct_name_col("value", {file_age, file_name}, 1); + auto file_key = name_col("key", string_type, 0); + auto file_map = + map_col("map_column", -1, {file_key, file_value}, string_type, file_value_type, 5); + set_name_identifiers(&file_map, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_map}, {}, {file_map}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + const auto& map_mapping = mapper.mappings()[0]; + ASSERT_EQ(map_mapping.child_mappings.size(), 2); + EXPECT_EQ(map_mapping.child_mappings[0].table_column_name, "key"); + EXPECT_EQ(map_mapping.child_mappings[0].file_column_name, "key"); + EXPECT_EQ(*map_mapping.child_mappings[0].file_local_id, 0); + + const auto& value_mapping = map_mapping.child_mappings[1]; + EXPECT_EQ(value_mapping.table_column_name, "value"); + EXPECT_EQ(value_mapping.file_column_name, "value"); + EXPECT_EQ(*value_mapping.file_local_id, 1); + ASSERT_EQ(value_mapping.child_mappings.size(), 3); + EXPECT_EQ(value_mapping.child_mappings[0].table_column_name, "age"); + EXPECT_EQ(value_mapping.child_mappings[0].file_column_name, "age"); + EXPECT_EQ(*value_mapping.child_mappings[0].file_local_id, 0); + EXPECT_EQ(value_mapping.child_mappings[1].table_column_name, "full_name"); + EXPECT_EQ(value_mapping.child_mappings[1].file_column_name, "name"); + EXPECT_EQ(*value_mapping.child_mappings[1].file_local_id, 1); + EXPECT_EQ(value_mapping.child_mappings[2].table_column_name, "gender"); + expect_missing(value_mapping.child_mappings[2]); + EXPECT_FALSE(value_mapping.is_trivial); +} + +// Scenario: MAP_KEYS(new_map_column) may build a key-only nested projection, while SELECT * still +// needs the whole map root. The mapper must add a synthetic value child and recursively map the old +// value struct instead of treating Struct(name, age) as a leaf to CAST into the table value struct. +TEST(ColumnMapperCreateMappingTest, KeyOnlyMapProjectionStillMapsEvolvedValueStruct) { + const auto int_type = i32(); + const auto string_type = str(); + const auto table_value_type = std::make_shared( + DataTypes {int_type, string_type, string_type}, Strings {"age", "full_name", "gender"}); + const auto file_value_type = std::make_shared(DataTypes {string_type, int_type}, + Strings {"name", "age"}); + + auto table_key = name_col("key", string_type); + auto table_map = map_col("new_map_column", -1, {table_key}, string_type, table_value_type); + table_map.name_mapping = {"map_column"}; + set_name_identifiers(&table_map, -1); + + auto file_key = name_col("key", string_type, 0); + auto file_name = name_col("name", string_type, 0); + auto file_age = name_col("age", int_type, 1); + auto file_value = struct_name_col("value", {file_name, file_age}, 1); + auto file_map = + map_col("map_column", -1, {file_key, file_value}, string_type, file_value_type, 5); + set_name_identifiers(&file_map, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_map}, {}, {file_map}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + const auto& map_mapping = mapper.mappings()[0]; + ASSERT_EQ(map_mapping.child_mappings.size(), 2); + EXPECT_EQ(map_mapping.child_mappings[0].table_column_name, "key"); + EXPECT_EQ(map_mapping.child_mappings[0].file_column_name, "key"); + EXPECT_EQ(*map_mapping.child_mappings[0].file_local_id, 0); + + const auto& value_mapping = map_mapping.child_mappings[1]; + EXPECT_EQ(value_mapping.table_column_name, "value"); + EXPECT_EQ(value_mapping.file_column_name, "value"); + EXPECT_EQ(*value_mapping.file_local_id, 1); + ASSERT_EQ(value_mapping.child_mappings.size(), 3); + EXPECT_EQ(value_mapping.child_mappings[0].table_column_name, "age"); + EXPECT_EQ(value_mapping.child_mappings[0].file_column_name, "age"); + EXPECT_EQ(*value_mapping.child_mappings[0].file_local_id, 1); + EXPECT_EQ(value_mapping.child_mappings[1].table_column_name, "full_name"); + EXPECT_EQ(value_mapping.child_mappings[1].file_column_name, "name"); + EXPECT_EQ(*value_mapping.child_mappings[1].file_local_id, 0); + EXPECT_EQ(value_mapping.child_mappings[2].table_column_name, "gender"); + expect_missing(value_mapping.child_mappings[2]); + EXPECT_FALSE(value_mapping.is_trivial); +} + +// Scenario: Iceberg uses field-id mapping, but a key-only map projection may force the mapper to +// synthesize the missing value struct from DataType names, which do not carry field ids. The mapper +// must name-match synthesized children before ordinal fallback, otherwise `age` would read old +// file child `name` and the later materialization would build the value struct incorrectly. +TEST(ColumnMapperCreateMappingTest, + KeyOnlyMapProjectionSynthesizedValueStructNameMatchesBeforeOrdinalFallback) { + const auto int_type = i32(); + const auto string_type = str(); + const auto table_value_type = std::make_shared( + DataTypes {int_type, string_type, string_type}, Strings {"age", "full_name", "gender"}); + const auto file_value_type = std::make_shared(DataTypes {string_type, int_type}, + Strings {"name", "age"}); + + auto table_key = field_id_col("key", 10, string_type, 0); + auto table_map = map_col("new_map_column", 2, {table_key}, string_type, table_value_type); + + auto file_key = field_id_col("key", 10, string_type, 0); + auto file_name = field_id_col("name", 7, string_type, 0); + auto file_age = field_id_col("age", 8, int_type, 1); + auto file_value = struct_col("value", 11, {file_name, file_age}, 1); + auto file_map = + map_col("new_map_column", 2, {file_key, file_value}, string_type, file_value_type, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_map}, {}, {file_map}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + const auto& map_mapping = mapper.mappings()[0]; + ASSERT_EQ(map_mapping.child_mappings.size(), 2); + EXPECT_EQ(map_mapping.child_mappings[0].table_column_name, "key"); + EXPECT_EQ(map_mapping.child_mappings[0].file_column_name, "key"); + EXPECT_EQ(*map_mapping.child_mappings[0].file_local_id, 0); + + const auto& value_mapping = map_mapping.child_mappings[1]; + EXPECT_EQ(value_mapping.table_column_name, "value"); + EXPECT_EQ(value_mapping.file_column_name, "value"); + EXPECT_EQ(*value_mapping.file_local_id, 1); + ASSERT_EQ(value_mapping.child_mappings.size(), 3); + EXPECT_EQ(value_mapping.child_mappings[0].table_column_name, "age"); + EXPECT_EQ(value_mapping.child_mappings[0].file_column_name, "age"); + EXPECT_EQ(*value_mapping.child_mappings[0].file_local_id, 1); + EXPECT_EQ(value_mapping.child_mappings[1].table_column_name, "full_name"); + EXPECT_EQ(value_mapping.child_mappings[1].file_column_name, "name"); + EXPECT_EQ(*value_mapping.child_mappings[1].file_local_id, 0); + EXPECT_EQ(value_mapping.child_mappings[2].table_column_name, "gender"); + expect_missing(value_mapping.child_mappings[2]); + EXPECT_FALSE(value_mapping.is_trivial); +} + +TEST(ColumnMapperCreateMappingTest, ByFieldIdDoesNotFallbackToNameAndUsesFirstDuplicate) { + const auto int_type = i32(); + const std::vector table_schema = { + field_id_col("renamed", 10, int_type), + name_col("same_name", int_type), + field_id_col("negative", -7, int_type), + }; + const std::vector file_schema = { + field_id_col("first", 10, int_type, 0), + field_id_col("second", 10, int_type, 1), + field_id_col("same_name", 99, int_type, 2), + field_id_col("negative_file", -7, int_type, 3), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 3); + expect_mapping(mapper.mappings()[0], 0, "renamed", 0, "first", int_type, int_type); + expect_missing(mapper.mappings()[1]); + expect_mapping(mapper.mappings()[2], 2, "negative", 3, "negative_file", int_type, int_type); +} + +// Scenario: Iceberg TopN lazy materialization uses BY_FIELD_ID for schema evolution and also asks +// the file reader to synthesize GLOBAL_ROWID. GLOBAL_ROWID is matched by ColumnType before the +// field-id matcher, so keeping BY_FIELD_ID does not make the mapper look for a numeric field id for +// that virtual column. +TEST(ColumnMapperCreateMappingTest, ByFieldIdMapsGlobalRowIdByVirtualColumnType) { + const auto int_type = i32(); + auto table_rowid = global_rowid_column_definition(); + table_rowid.name = BeConsts::GLOBAL_ROWID_COL + "equality_delete_par_1"; + table_rowid.identifier = Field::create_field(table_rowid.name); + + const std::vector table_schema = { + field_id_col("new_new_id", 1, int_type), + table_rowid, + }; + const std::vector file_schema = { + field_id_col("id", 1, int_type, 0), + global_rowid_column_definition(), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 2); + expect_mapping(mapper.mappings()[0], 0, "new_new_id", 0, "id", int_type, int_type); + expect_mapping(mapper.mappings()[1], 1, table_rowid.name, GLOBAL_ROWID_COLUMN_ID, + BeConsts::GLOBAL_ROWID_COL, str(), str()); +} + +TEST(ColumnMapperCreateMappingTest, ByFieldIdTreatsSameNameDifferentFieldIdAsMissing) { + const auto int_type = i32(); + const std::vector table_schema = { + field_id_col("same_name", 10, int_type), + }; + const std::vector file_schema = { + field_id_col("same_name", 20, int_type, 0), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + const auto status = mapper.create_mapping(table_schema, {}, file_schema); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_missing(mapper.mappings()[0]); +} + +TEST(ColumnMapperCreateMappingTest, NestedFieldIdTreatsSameNameDifferentFieldIdAsMissing) { + const auto int_type = i32(); + auto table_child = field_id_col("child", 10, int_type); + auto table_root = struct_col("root", 1, {table_child}); + + auto file_child = field_id_col("child", 20, int_type, 0); + auto file_root = struct_col("root", 1, {file_child}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + const auto status = mapper.create_mapping({table_root}, {}, {file_root}); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "root", 0, "root", file_root.type, table_root.type); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 1); + expect_missing(mapper.mappings()[0].child_mappings[0]); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexMapsTopLevelColumnsByPositionIgnoringFileNames) { + const auto int_type = i32(); + const auto string_type = str(); + const std::vector table_schema = { + position_col("user_id", 0, int_type), + position_col("user_name", 1, string_type), + position_col("age", 2, int_type), + }; + const std::vector file_schema = { + field_id_col("_col0", 100, int_type, 0), + field_id_col("_col1", 101, string_type, 1), + field_id_col("_col2", 102, int_type, 2), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 3); + expect_mapping(mapper.mappings()[0], 0, "user_id", 0, "_col0", int_type, int_type); + expect_mapping(mapper.mappings()[1], 1, "user_name", 1, "_col1", string_type, string_type); + expect_mapping(mapper.mappings()[2], 2, "age", 2, "_col2", int_type, int_type); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexSupportsSparseProjection) { + const auto int_type = i32(); + const std::vector table_schema = { + position_col("age", 2, int_type), + position_col("score", 4, int_type), + }; + const std::vector file_schema = { + field_id_col("_col0", 100, int_type, 0), field_id_col("_col1", 101, int_type, 1), + field_id_col("_col2", 102, int_type, 2), field_id_col("_col3", 103, int_type, 3), + field_id_col("_col4", 104, int_type, 4), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 2); + expect_mapping(mapper.mappings()[0], 0, "age", 2, "_col2", int_type, int_type); + expect_mapping(mapper.mappings()[1], 1, "score", 4, "_col4", int_type, int_type); +} + +TEST(ColumnMapperCreateMappingTest, + ByIndexMatchesNestedStructChildrenByNameEvenWhenChildrenHaveFieldIds) { + const auto int_type = i32(); + const auto string_type = str(); + // Hive positional mapping only applies to top-level columns. FE/history schema metadata can + // still put field-id style integer identifiers on nested struct children. Those nested + // identifiers must not be interpreted as file positions. + auto table_root = struct_col("profile", 1, + { + field_id_col("id", 100, int_type), + field_id_col("name", 101, string_type), + }); + // Reverse the file child order so a wrong positional match either misses the child or reads + // the wrong physical child. The expected mapping below proves the children are matched by name. + auto file_root = struct_name_col("_col1", + { + name_col("name", string_type, 0), + name_col("id", int_type, 1), + }, + 1); + const std::vector table_schema = {table_root}; + const std::vector file_schema = { + field_id_col("_col0", 1000, string_type, 0), + file_root, + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + const auto status = mapper.create_mapping(table_schema, {}, file_schema); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "profile", 1, "_col1", file_root.type, table_root.type); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + expect_mapping(mapper.mappings()[0].child_mappings[0], 0, "id", 1, "id", int_type, int_type); + expect_mapping(mapper.mappings()[0].child_mappings[1], 0, "name", 0, "name", string_type, + string_type); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexNestedStructDoesNotUseChildOrdinalIdentifier) { + const auto int_type = i32(); + const auto string_type = str(); + // This is the dangerous variant of the previous case: the nested integer identifiers happen + // to be valid child ordinals. BY_INDEX must still ignore them below the top-level root. + auto table_root = struct_col("profile", 1, + { + field_id_col("id", 0, int_type), + field_id_col("name", 1, string_type), + }); + // If the implementation uses child ordinal matching, id/name will be swapped here. + auto file_root = struct_name_col("_col1", + { + name_col("name", string_type, 0), + name_col("id", int_type, 1), + }, + 1); + const std::vector table_schema = {table_root}; + const std::vector file_schema = { + field_id_col("_col0", 1000, string_type, 0), + file_root, + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + const auto status = mapper.create_mapping(table_schema, {}, file_schema); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "profile", 1, "_col1", file_root.type, table_root.type); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + expect_mapping(mapper.mappings()[0].child_mappings[0], 0, "id", 1, "id", int_type, int_type); + expect_mapping(mapper.mappings()[0].child_mappings[1], 0, "name", 0, "name", string_type, + string_type); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexArrayElementStructChildrenMatchByName) { + const auto int_type = i32(); + const auto string_type = str(); + // The top-level ARRAY column is selected by file position. After that, ARRAY has a single + // structural child, and the element STRUCT should use Hive's nested-by-name behavior. + auto table_element = struct_col("element", 10, + { + field_id_col("id", 100, int_type), + field_id_col("name", 101, string_type), + }); + auto table_root = array_col("profiles", 1, table_element); + // Reverse the element struct children to distinguish name matching from position matching. + auto file_element = struct_name_col("element", + { + name_col("name", string_type, 0), + name_col("id", int_type, 1), + }, + 0); + auto file_root = array_col("_col1", 1001, file_element, 1); + const std::vector table_schema = {table_root}; + const std::vector file_schema = { + field_id_col("_col0", 1000, string_type, 0), + file_root, + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + const auto status = mapper.create_mapping(table_schema, {}, file_schema); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "profiles", 1, "_col1", file_root.type, + table_root.type); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 1); + const auto& element_mapping = mapper.mappings()[0].child_mappings[0]; + expect_mapping(element_mapping, 0, "element", 0, "element", file_element.type, + table_element.type); + ASSERT_EQ(element_mapping.child_mappings.size(), 2); + expect_mapping(element_mapping.child_mappings[0], 0, "id", 1, "id", int_type, int_type); + expect_mapping(element_mapping.child_mappings[1], 0, "name", 0, "name", string_type, + string_type); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexMapValueStructChildrenMatchByName) { + const auto int_type = i32(); + const auto string_type = str(); + const auto key_type = str(); + // MAP key/value are structural children, so BY_INDEX should not reinterpret their nested + // integer identifiers as arbitrary positions. The value STRUCT then follows name matching. + auto table_key = field_id_col("key", 10, key_type); + auto table_value = struct_col("value", 11, + { + field_id_col("id", 100, int_type), + field_id_col("name", 101, string_type), + }); + auto table_root = map_col("profiles", 1, {table_key, table_value}, key_type, table_value.type); + auto file_key = name_col("key", key_type, 0); + // Reverse value struct children. A positional nested match would produce name/id swapped. + auto file_value = struct_name_col("value", + { + name_col("name", string_type, 0), + name_col("id", int_type, 1), + }, + 1); + auto file_root = map_col("_col1", 1001, {file_key, file_value}, key_type, file_value.type, 1); + const std::vector table_schema = {table_root}; + const std::vector file_schema = { + field_id_col("_col0", 1000, string_type, 0), + file_root, + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + const auto status = mapper.create_mapping(table_schema, {}, file_schema); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "profiles", 1, "_col1", file_root.type, + table_root.type); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + expect_mapping(mapper.mappings()[0].child_mappings[0], 0, "key", 0, "key", key_type, key_type); + const auto& value_mapping = mapper.mappings()[0].child_mappings[1]; + expect_mapping(value_mapping, 0, "value", 1, "value", file_value.type, table_value.type); + ASSERT_EQ(value_mapping.child_mappings.size(), 2); + expect_mapping(value_mapping.child_mappings[0], 0, "id", 1, "id", int_type, int_type); + expect_mapping(value_mapping.child_mappings[1], 0, "name", 0, "name", string_type, string_type); +} + +TEST(ColumnMapperCreateMappingTest, + ByIndexPartitionColumnsTakeConstantAndDoNotConsumeFilePosition) { + const auto int_type = i32(); + const auto string_type = str(); + auto partition = name_col("dt", string_type); + partition.is_partition_key = true; + const std::vector table_schema = { + partition, + position_col("user_id", 0, int_type), + position_col("score", 1, int_type), + }; + const std::vector file_schema = { + field_id_col("_col0", 100, int_type, 0), + field_id_col("_col1", 101, int_type, 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + ASSERT_TRUE(mapper.create_mapping(table_schema, + {{"dt", Field::create_field("2026-06-11")}}, + file_schema) + .ok()); + + ASSERT_EQ(mapper.mappings().size(), 3); + expect_constant(mapper, mapper.mappings()[0], 0, string_type); + expect_mapping(mapper.mappings()[1], 1, "user_id", 0, "_col0", int_type, int_type); + expect_mapping(mapper.mappings()[2], 2, "score", 1, "_col1", int_type, int_type); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexOutOfRangeFallsBackToDefaultOrMissing) { + const auto int_type = i32(); + auto with_default = position_col("extra_default", 5, int_type); + const auto literal_expr = + VExprContext::create_shared(literal(int_type, Field::create_field(42))); + with_default.default_expr = literal_expr; + const std::vector table_schema = { + position_col("a", 0, int_type), + with_default, + position_col("extra_missing", 99, int_type), + }; + const std::vector file_schema = { + field_id_col("_col0", 100, int_type, 0), + field_id_col("_col1", 101, int_type, 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 3); + expect_mapping(mapper.mappings()[0], 0, "a", 0, "_col0", int_type, int_type); + expect_constant(mapper, mapper.mappings()[1], 1, int_type); + EXPECT_EQ(mapper.mappings()[1].default_expr, literal_expr); + expect_missing(mapper.mappings()[2]); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexMissingIdentifierFallsBackToDefaultOrMissing) { + const auto int_type = i32(); + auto with_default = name_col("extra_default", int_type); + const auto literal_expr = + VExprContext::create_shared(literal(int_type, Field::create_field(42))); + with_default.default_expr = literal_expr; + const std::vector table_schema = { + position_col("a", 0, int_type), + with_default, + name_col("extra_missing", int_type), + }; + const std::vector file_schema = { + field_id_col("_col0", 100, int_type, 0), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 3); + expect_mapping(mapper.mappings()[0], 0, "a", 0, "_col0", int_type, int_type); + expect_constant(mapper, mapper.mappings()[1], 1, int_type); + EXPECT_EQ(mapper.mappings()[1].default_expr, literal_expr); + expect_missing(mapper.mappings()[2]); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexOutOfRangeFallsBackToMissing) { + const auto int_type = i32(); + const std::vector table_schema = { + position_col("a", 0, int_type), + position_col("b", 5, int_type), + }; + const std::vector file_schema = { + field_id_col("_col0", 100, int_type, 0), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + const auto status = mapper.create_mapping(table_schema, {}, file_schema); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(mapper.mappings().size(), 2); + expect_mapping(mapper.mappings()[0], 0, "a", 0, "_col0", int_type, int_type); + expect_missing(mapper.mappings()[1]); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexIgnoresExtraFileColumns) { + const auto int_type = i32(); + const std::vector table_schema = { + position_col("a", 0, int_type), + }; + const std::vector file_schema = { + field_id_col("_col0", 100, int_type, 0), + field_id_col("_col1", 101, int_type, 1), + field_id_col("_col2", 102, int_type, 2), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "a", 0, "_col0", int_type, int_type); +} + +TEST(ColumnMapperCreateMappingTest, ByIndexIgnoresFileColumnNames) { + const auto int_type = i32(); + const std::vector table_schema = { + position_col("a", 1, int_type), + }; + const std::vector file_schema = { + field_id_col("a", 100, int_type, 10), + field_id_col("b", 101, int_type, 20), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_INDEX}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_mapping(mapper.mappings()[0], 0, "a", 20, "b", int_type, int_type); +} + +TEST(ColumnMapperCreateMappingTest, MissingColumnFallsBackToMissingMapping) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + const auto status = mapper.create_mapping({name_col("missing", i32())}, {}, + {name_col("present", i32(), 0)}); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_missing(mapper.mappings()[0]); +} + +// ---------------------------------------------------------------------- +// L1 constants and virtual columns. +// These tests verify non-file-backed mappings before TableReader materializes +// their final values. +// ---------------------------------------------------------------------- + +TEST(ColumnMapperConstantTest, PartitionDefaultAndVirtualColumnsUseDedicatedBranches) { + auto partition_column = name_col("dt", str()); + partition_column.is_partition_key = true; + + auto default_column = name_col("new_value", i32()); + default_column.default_expr = + VExprContext::create_shared(literal(i32(), Field::create_field(42))); + + auto row_id_column = name_col("_row_id", make_nullable(i64())); + auto sequence_column = name_col("_last_updated_sequence_number", make_nullable(i64())); + auto iceberg_rowid_column = name_col(BeConsts::ICEBERG_ROWID_COL, str()); + + const std::vector table_schema = { + partition_column, default_column, row_id_column, sequence_column, iceberg_rowid_column}; + const std::map partition_values = { + {"dt", Field::create_field("2026-06-11")}, + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, partition_values, {}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 5); + expect_constant(mapper, mapper.mappings()[0], 0, str()); + expect_constant(mapper, mapper.mappings()[1], 1, i32()); + EXPECT_EQ(mapper.mappings()[2].virtual_column_type, TableVirtualColumnType::ROW_ID); + EXPECT_EQ(mapper.mappings()[3].virtual_column_type, + TableVirtualColumnType::LAST_UPDATED_SEQUENCE_NUMBER); + EXPECT_EQ(mapper.mappings()[4].virtual_column_type, TableVirtualColumnType::ICEBERG_ROWID); +} + +TEST(ColumnMapperConstantTest, PhysicalRowLineageFiltersStayFinalizeOnly) { + auto row_id_column = name_col("_row_id", make_nullable(i64())); + auto sequence_column = name_col("_last_updated_sequence_number", make_nullable(i64())); + const std::vector table_schema = {row_id_column, sequence_column}; + const std::vector file_schema = { + name_col("_row_id", make_nullable(i64()), 2147483540), + name_col("_last_updated_sequence_number", make_nullable(i64()), 2147483539), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 2); + EXPECT_EQ(mapper.mappings()[0].virtual_column_type, TableVirtualColumnType::ROW_ID); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::FINALIZE_ONLY); + EXPECT_EQ(mapper.mappings()[1].virtual_column_type, + TableVirtualColumnType::LAST_UPDATED_SEQUENCE_NUMBER); + EXPECT_EQ(mapper.mappings()[1].filter_conversion, FilterConversionType::FINALIZE_ONLY); + + auto row_id_filter = + binary_predicate(TExprOpcode::EQ, table_slot(0, 0, make_nullable(i64()), "_row_id"), + literal(i64(), Field::create_field(1001))); + auto sequence_filter = binary_predicate( + TExprOpcode::EQ, + table_slot(1, 1, make_nullable(i64()), "_last_updated_sequence_number"), + literal(i64(), Field::create_field(77))); + TableFilter row_id_table_filter {.conjunct = VExprContext::create_shared(row_id_filter), + .global_indices = {GlobalIndex(0)}}; + TableFilter sequence_table_filter {.conjunct = VExprContext::create_shared(sequence_filter), + .global_indices = {GlobalIndex(1)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({row_id_table_filter, sequence_table_filter}, + table_schema, &request) + .ok()); + + EXPECT_TRUE(request.conjuncts.empty()); + EXPECT_TRUE(request.predicate_columns.empty()); + EXPECT_EQ(projection_ids(request.non_predicate_columns), + std::vector({2147483540, 2147483539})); +} + +TEST(ColumnMapperConstantTest, MissingRowLineageDefaultExprStillUsesVirtualMapping) { + auto id_column = field_id_col("id", 1, make_nullable(i32())); + auto row_id_column = field_id_col("renamed_row_id", 2147483540, make_nullable(i64())); + row_id_column.default_expr = VExprContext::create_shared( + literal(make_nullable(i64()), Field::create_field(0))); + auto sequence_column = + field_id_col("renamed_last_updated_sequence_number", 2147483539, make_nullable(i64())); + sequence_column.default_expr = VExprContext::create_shared( + literal(make_nullable(i64()), Field::create_field(0))); + + const std::vector table_schema = {id_column, row_id_column, sequence_column}; + const std::vector file_schema = { + field_id_col("id", 1, make_nullable(i32()), 0), + field_id_col("name", 2, make_nullable(str()), 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 3); + expect_mapping(mapper.mappings()[0], 0, "id", 0, "id", make_nullable(i32()), + make_nullable(i32())); + EXPECT_EQ(mapper.mappings()[1].virtual_column_type, TableVirtualColumnType::ROW_ID); + EXPECT_FALSE(mapper.mappings()[1].constant_index.has_value()); + EXPECT_EQ(mapper.mappings()[2].virtual_column_type, + TableVirtualColumnType::LAST_UPDATED_SEQUENCE_NUMBER); + EXPECT_FALSE(mapper.mappings()[2].constant_index.has_value()); + EXPECT_TRUE(mapper.constant_map().empty()); +} + +TEST(ColumnMapperConstantTest, ByFieldIdDoesNotTreatSameNameDifferentIdAsRowLineage) { + const std::vector table_schema = { + field_id_col("_row_id", 100, make_nullable(i64())), + field_id_col("_last_updated_sequence_number", 101, make_nullable(i64())), + }; + const std::vector file_schema = { + field_id_col("_row_id", 100, make_nullable(i64()), 0), + field_id_col("_last_updated_sequence_number", 101, make_nullable(i64()), 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + ASSERT_EQ(mapper.mappings().size(), 2); + expect_mapping(mapper.mappings()[0], 0, "_row_id", 0, "_row_id", make_nullable(i64()), + make_nullable(i64())); + EXPECT_EQ(mapper.mappings()[0].virtual_column_type, TableVirtualColumnType::INVALID); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::COPY_DIRECTLY); + expect_mapping(mapper.mappings()[1], 1, "_last_updated_sequence_number", 1, + "_last_updated_sequence_number", make_nullable(i64()), make_nullable(i64())); + EXPECT_EQ(mapper.mappings()[1].virtual_column_type, TableVirtualColumnType::INVALID); + EXPECT_EQ(mapper.mappings()[1].filter_conversion, FilterConversionType::COPY_DIRECTLY); +} + +TEST(ColumnMapperConstantTest, PartitionAliasResolvesRenamedValue) { + auto partition_column = name_col("current_dt", str()); + partition_column.name_mapping = {"legacy_dt"}; + partition_column.is_partition_key = true; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping( + {partition_column}, + {{"legacy_dt", Field::create_field("2026-06-11")}}, {}) + .ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + expect_constant(mapper, mapper.mappings()[0], 0, str()); +} + +TEST(ColumnMapperConstantTest, PartitionConstantFilterEntryDoesNotReadFileColumns) { + auto partition_column = name_col("part", i32()); + partition_column.is_partition_key = true; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({partition_column}, + {{"part", Field::create_field(7)}}, {}) + .ok()); + + TableFilter filter { + .conjunct = VExprContext::create_shared(int_gt(table_slot(0, 0, i32(), "part"), 1)), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {partition_column}, &request).ok()); + + ASSERT_EQ(mapper.filter_entries().size(), 1); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(0)).is_constant()); + EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(0)).constant_index(), + *mapper.mappings()[0].constant_index); + EXPECT_TRUE(request.local_positions.empty()); + EXPECT_TRUE(request.predicate_columns.empty()); + EXPECT_TRUE(request.non_predicate_columns.empty()); + EXPECT_TRUE(request.conjuncts.empty()); +} + +TEST(ColumnMapperConstantTest, DefaultConstantFilterEntryUsesDefaultExpression) { + auto default_column = name_col("new_value", i32()); + default_column.default_expr = + VExprContext::create_shared(literal(i32(), Field::create_field(42))); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({default_column}, {}, {}).ok()); + + TableFilter filter {.conjunct = VExprContext::create_shared( + int_gt(table_slot(0, 0, i32(), "new_value"), 1)), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {default_column}, &request).ok()); + + ASSERT_EQ(mapper.filter_entries().size(), 1); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(0)).is_constant()); + const auto constant_index = mapper.filter_entries().at(GlobalIndex(0)).constant_index(); + EXPECT_EQ(constant_index, *mapper.mappings()[0].constant_index); + EXPECT_EQ(mapper.constant_map().get(constant_index).expr, default_column.default_expr); + EXPECT_TRUE(request.local_positions.empty()); + EXPECT_TRUE(request.predicate_columns.empty()); + EXPECT_TRUE(request.non_predicate_columns.empty()); + EXPECT_TRUE(request.conjuncts.empty()); +} + +TEST(ColumnMapperConstantTest, MixedConstantAndFileFilterKeepsOnlyFileScanColumn) { + auto partition_column = name_col("part", i32()); + partition_column.is_partition_key = true; + const auto file_column = name_col("score", i32(), 3); + const std::vector table_schema = {partition_column, file_column}; + const std::vector file_schema = {file_column}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {{"part", Field::create_field(7)}}, + file_schema) + .ok()); + + TableFilter constant_filter { + .conjunct = VExprContext::create_shared(int_gt(table_slot(0, 0, i32(), "part"), 1)), + .global_indices = {GlobalIndex(0)}}; + TableFilter file_filter { + .conjunct = VExprContext::create_shared(int_gt(table_slot(1, 1, i32(), "score"), 10)), + .global_indices = {GlobalIndex(1)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({constant_filter, file_filter}, table_schema, &request) + .ok()); + + ASSERT_EQ(mapper.filter_entries().size(), 2); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(0)).is_constant()); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(1)).is_local()); + EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(0)); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(3)); + EXPECT_TRUE(request.non_predicate_columns.empty()); +} + +// ---------------------------------------------------------------------- +// L1 direct filter localization tests. +// These tests call localize_filters() directly to pin the core interface +// contract apart from create_scan_request() initialization. +// ---------------------------------------------------------------------- + +TEST(ColumnMapperLocalizeFiltersTest, VisibleLocalFilterAddsPredicateColumnAndConjunct) { + const auto int_type = i32(); + const std::vector table_schema = {name_col("id", int_type)}; + const std::vector file_schema = {name_col("id", int_type, 7)}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + TableFilter filter {.conjunct = VExprContext::create_shared(table_slot(11, 0, int_type, "id")), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.localize_filters({filter}, &request).ok()); + + EXPECT_TRUE(request.non_predicate_columns.empty()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(7)); + ASSERT_EQ(request.local_positions.size(), 1); + EXPECT_EQ(request.local_positions.at(LocalColumnId(7)), LocalIndex(0)); + ASSERT_EQ(mapper.filter_entries().size(), 1); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(0)).is_local()); + EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(0)).local_index(), LocalIndex(0)); + + ASSERT_EQ(request.conjuncts.size(), 1); + const auto* localized_slot = assert_cast(request.conjuncts[0]->root().get()); + EXPECT_EQ(localized_slot->slot_id(), 11); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_EQ(localized_slot->column_name(), "id"); + EXPECT_TRUE(localized_slot->data_type()->equals(*int_type)); +} + +TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { + const auto binary_type = varbinary(); + const auto table_column = name_col("partition_key", binary_type); + const auto file_column = name_col("partition_key", binary_type, 7); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_column}, {}, {file_column}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_TRUE(mapper.mappings()[0].is_trivial); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::FINALIZE_ONLY); + + const auto value = Field::create_field(StringView("binary-value")); + TableFilter filter {.conjunct = VExprContext::create_shared(binary_predicate( + TExprOpcode::EQ, table_slot(0, 0, binary_type, "partition_key"), + literal(binary_type, value))), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_column}, &request).ok()); + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(7)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +TEST(ColumnMapperLocalizeFiltersTest, NestedVarbinaryFilterStaysAboveFileReader) { + const auto table_column = struct_name_col( + "payload", {name_col("id", i32()), name_col("binary_value", varbinary())}); + const auto file_column = struct_name_col( + "payload", {name_col("id", i32(), 0), name_col("binary_value", varbinary(), 1)}, 7); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_column}, {}, {file_column}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::FINALIZE_ONLY); + + TableFilter filter { + .conjunct = VExprContext::create_shared(table_slot(0, 0, table_column.type, "payload")), + .global_indices = {GlobalIndex(0)}}; + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_column}, &request).ok()); + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(7)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +TEST(ColumnMapperLocalizeFiltersTest, ConstantFilterBuildsEntryWithoutFileScanColumn) { + auto partition_column = name_col("part", i32()); + partition_column.is_partition_key = true; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({partition_column}, + {{"part", Field::create_field(7)}}, {}) + .ok()); + + TableFilter filter {.conjunct = VExprContext::create_shared(table_slot(3, 0, i32(), "part")), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.localize_filters({filter}, &request).ok()); + + EXPECT_TRUE(request.predicate_columns.empty()); + EXPECT_TRUE(request.non_predicate_columns.empty()); + EXPECT_TRUE(request.local_positions.empty()); + EXPECT_TRUE(request.conjuncts.empty()); + ASSERT_EQ(mapper.filter_entries().size(), 1); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(0)).is_constant()); + EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(0)).constant_index(), + mapper.mappings()[0].constant_index); +} + +TEST(ColumnMapperLocalizeFiltersTest, NestedFilterOnlyChildMergesIntoPredicateProjection) { + const auto int_type = i32(); + const auto string_type = str(); + + auto table_a = name_col("a", int_type); + auto table_b = name_col("b", string_type); + auto table_struct = struct_name_col("s", {table_b}); + auto full_table_struct = struct_name_col("s", {table_a, table_b}); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", string_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + auto filter_expr = int_gt( + struct_element(table_slot(0, 0, full_table_struct.type, "s"), int_type, "a"), 10); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.localize_filters({filter}, &request).ok()); + + EXPECT_TRUE(request.non_predicate_columns.empty()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(5)); + ASSERT_FALSE(request.predicate_columns[0].project_all_children); + EXPECT_EQ(projection_ids(request.predicate_columns[0].children), std::vector({0, 1})); + ASSERT_EQ(request.local_positions.size(), 1); + EXPECT_EQ(request.local_positions.at(LocalColumnId(5)), LocalIndex(0)); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(0)).is_local()); + EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(0)).local_index(), LocalIndex(0)); +} + +TEST(ColumnMapperLocalizeFiltersTest, PreservesExistingScanStateWhenAddingPredicateColumn) { + const auto int_type = i32(); + const std::vector table_schema = { + name_col("id", int_type), + name_col("score", int_type), + }; + const std::vector file_schema = { + name_col("id", int_type, 3), + name_col("score", int_type, 4), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + TableFilter filter {.conjunct = VExprContext::create_shared(table_slot(2, 0, int_type, "id")), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + request.non_predicate_columns.push_back(LocalColumnIndex::top_level(LocalColumnId(4))); + request.local_positions.emplace(LocalColumnId(4), LocalIndex(0)); + ASSERT_TRUE(mapper.localize_filters({filter}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(4)); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(3)); + ASSERT_EQ(request.local_positions.size(), 2); + EXPECT_EQ(request.local_positions.at(LocalColumnId(4)), LocalIndex(0)); + EXPECT_EQ(request.local_positions.at(LocalColumnId(3)), LocalIndex(1)); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(0)).is_local()); + EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(0)).local_index(), LocalIndex(1)); +} + +// ---------------------------------------------------------------------- +// L1 scan request and filter localization tests. +// These tests assert predicate/non-predicate split, local positions, hidden +// filter mappings, and nested predicate targets. +// ---------------------------------------------------------------------- + +TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback) { + const auto int_type = i32(); + const std::vector table_schema = { + field_id_col("id", 1, int_type), + }; + const std::vector file_schema = { + field_id_col("id", 1, int_type, 0), + field_id_col("score", 2, int_type, 1), + }; + + auto filter_expr = int_gt(table_slot(7, 1, int_type, "score"), 10); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(1)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, table_schema, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(1)).is_local()); + EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(1)); +} + +TEST(ColumnMapperScanRequestTest, StructOutputAndFilterOnlyChildAreMerged) { + const auto int_type = i32(); + const auto string_type = str(); + + auto table_a = name_col("a", int_type); + auto table_b = name_col("b", string_type); + auto table_struct = struct_name_col("s", {table_b}); + auto full_table_struct = struct_name_col("s", {table_a, table_b}); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", string_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + auto filter_expr = int_gt( + struct_element(table_slot(0, 0, full_table_struct.type, "s"), int_type, "a"), 10); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); + + EXPECT_TRUE(request.non_predicate_columns.empty()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(5)); + ASSERT_FALSE(request.predicate_columns[0].project_all_children); + EXPECT_EQ(projection_ids(request.predicate_columns[0].children), std::vector({0, 1})); +} + +TEST(ColumnMapperScanRequestTest, RenamedNestedPredicateTargetsMappedFileChild) { + const auto int_type = i32(); + + auto table_a = field_id_col("a", 1, int_type); + auto table_renamed_b = field_id_col("renamed_b", 2, int_type); + auto table_struct = struct_col("s", 10, {table_a, table_renamed_b}); + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, int_type, 1); + auto file_struct = struct_col("s", 10, {file_a, file_b}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + auto filter_expr = int_gt( + struct_element(table_slot(0, 0, table_struct.type, "s"), int_type, "renamed_b"), 10); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); +} + +TEST(ColumnMapperScanRequestTest, NestedInNullAndReverseComparisonFiltersAreMerged) { + const auto int_type = i32(); + const auto string_type = str(); + + auto table_a = name_col("a", int_type); + auto table_b = name_col("b", string_type); + auto table_struct = struct_name_col("s", {table_b}); + auto full_table_struct = struct_name_col("s", {table_a, table_b}); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", string_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + const auto nested_a = + struct_element(table_slot(0, 0, full_table_struct.type, "s"), int_type, "a"); + auto in_filter = + in_predicate(nested_a, int_type, + {Field::create_field(5), Field::create_field(7)}); + auto reverse_filter = binary_predicate( + TExprOpcode::LT, literal(int_type, Field::create_field(3)), nested_a); + auto null_filter = null_predicate(nested_a, true); + auto not_null_filter = null_predicate(nested_a, false); + auto filter_expr = compound_predicate( + TExprOpcode::COMPOUND_AND, + compound_predicate(TExprOpcode::COMPOUND_AND, in_filter, reverse_filter), + compound_predicate(TExprOpcode::COMPOUND_AND, null_filter, not_null_filter)); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); +} + +TEST(ColumnMapperScanRequestTest, NestedPredicateFilterThroughSafeCast) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + const auto string_type = str(); + + auto table_b = name_col("b", string_type); + auto table_struct = struct_name_col("s", {table_b}); + auto full_table_struct = std::make_shared( + DataTypes {table_bigint_type, string_type}, Strings {"a", "b"}); + + auto file_a = name_col("a", file_int_type, 0); + auto file_b = name_col("b", string_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + const auto nested_a = + struct_element(table_slot(0, 0, full_table_struct, "s"), file_int_type, "a"); + auto filter_expr = + binary_predicate(TExprOpcode::GT, cast_expr(nested_a, table_bigint_type), + literal(table_bigint_type, Field::create_field(5))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); +} + +TEST(ColumnMapperScanRequestTest, UnsafeCastDoesNotBuildNestedPredicateFilter) { + const auto file_bigint_type = i64(); + const auto table_int_type = i32(); + const auto string_type = str(); + + auto table_b = name_col("b", string_type); + auto table_struct = struct_name_col("s", {table_b}); + auto full_table_struct = std::make_shared( + DataTypes {table_int_type, string_type}, Strings {"a", "b"}); + + auto file_a = name_col("a", file_bigint_type, 0); + auto file_b = name_col("b", string_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + const auto nested_a = + struct_element(table_slot(0, 0, full_table_struct, "s"), file_bigint_type, "a"); + auto filter_expr = binary_predicate(TExprOpcode::GT, cast_expr(nested_a, table_int_type), + literal(table_int_type, Field::create_field(5))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(5)); + EXPECT_EQ(projection_ids(request.predicate_columns[0].children), std::vector({0, 1})); +} + +TEST(ColumnMapperScanRequestTest, DeepNestedPredicateTargetsLeafPath) { + const auto id_type = i32(); + const auto name_type = str(); + const auto string_type = str(); + auto table_b = name_col("b", string_type); + auto table_struct = struct_name_col("s", {table_b}); + + auto full_table_inner_type = + std::make_shared(DataTypes {id_type, name_type}, Strings {"id", "n"}); + auto full_table_struct_type = std::make_shared( + DataTypes {full_table_inner_type, string_type}, Strings {"a", "b"}); + + auto file_id = name_col("id", id_type, 0); + auto file_name = name_col("n", name_type, 1); + auto file_a = struct_name_col("a", {file_id, file_name}, 0); + auto file_b = name_col("b", string_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + const auto nested_id = + struct_element(struct_element(table_slot(0, 0, full_table_struct_type, "s"), + full_table_inner_type, "a"), + id_type, "id"); + auto filter_expr = + in_predicate(nested_id, id_type, + {Field::create_field(5), Field::create_field(7)}); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); +} + +TEST(ColumnMapperScanRequestTest, ArrayStructProjectionPrunesElementChildren) { + const auto int_type = i32(); + const auto string_type = str(); + auto table_b = name_col("b", string_type); + auto table_element = struct_name_col("element", {table_b}); + auto table_array = array_col("items", -1, table_element); + table_array.identifier = Field::create_field("items"); + set_name_identifiers(&table_array, -1); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", string_type, 1); + auto file_element = struct_name_col("element", {file_a, file_b}, 0); + auto file_array = array_col("items", -1, file_element, 4); + file_array.identifier = Field::create_field("items"); + set_name_identifiers(&file_array, 4); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_array}, {}, {file_array}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_array}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + const auto& projection = request.non_predicate_columns[0]; + EXPECT_EQ(projection.column_id(), LocalColumnId(4)); + ASSERT_FALSE(projection.project_all_children); + ASSERT_EQ(projection.children.size(), 1); + EXPECT_EQ(projection.children[0].local_id(), 0); + ASSERT_EQ(projection.children[0].children.size(), 1); + EXPECT_EQ(projection.children[0].children[0].local_id(), 1); + + const auto* mapped_array = assert_cast( + remove_nullable(mapper.mappings()[0].file_type).get()); + const auto* mapped_element = assert_cast( + remove_nullable(mapped_array->get_nested_type()).get()); + ASSERT_EQ(mapped_element->get_elements().size(), 1); + EXPECT_EQ(mapped_element->get_element_name(0), "b"); +} + +TEST(ColumnMapperScanRequestTest, MapValueStructProjectionPrunesValueChildren) { + const auto key_type = str(); + const auto int_type = i32(); + const auto string_type = str(); + + auto table_value_b = name_col("b", string_type); + auto table_value = struct_name_col("value", {table_value_b}); + auto table_map = map_col("m", -1, {table_value}, key_type, table_value.type); + table_map.identifier = Field::create_field("m"); + set_name_identifiers(&table_map, -1); + + auto file_key = name_col("key", key_type, 0); + auto file_value_a = name_col("a", int_type, 0); + auto file_value_b = name_col("b", string_type, 1); + auto file_value = struct_name_col("value", {file_value_a, file_value_b}, 1); + auto file_map = map_col("m", -1, {file_key, file_value}, key_type, file_value.type, 6); + file_map.identifier = Field::create_field("m"); + set_name_identifiers(&file_map, 6); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_map}, {}, {file_map}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_map}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + const auto& projection = request.non_predicate_columns[0]; + EXPECT_EQ(projection.column_id(), LocalColumnId(6)); + ASSERT_FALSE(projection.project_all_children); + ASSERT_EQ(projection.children.size(), 1); + EXPECT_EQ(projection.children[0].local_id(), 1); + ASSERT_EQ(projection.children[0].children.size(), 1); + EXPECT_EQ(projection.children[0].children[0].local_id(), 1); + + const auto* mapped_map = + assert_cast(remove_nullable(mapper.mappings()[0].file_type).get()); + const auto* mapped_value = + assert_cast(remove_nullable(mapped_map->get_value_type()).get()); + ASSERT_EQ(mapped_value->get_elements().size(), 1); + EXPECT_EQ(mapped_value->get_element_name(0), "b"); +} + +// Scenario: a table struct projects only child `b`, while the file struct stores `a,b`. +// BY_NAME mapping should read only the physical child `b` and rebuild the mapped file type to the +// projected struct shape. +TEST(ColumnMapperScanRequestTest, StructProjectionPrunesChildrenByName) { + const auto int_type = i32(); + const auto string_type = str(); + + auto table_b = name_col("b", string_type); + auto table_struct = struct_name_col("s", {table_b}); + set_name_identifiers(&table_struct, 0); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", string_type, 1); + auto file_struct = struct_name_col("s", {file_a, file_b}, 0); + set_name_identifiers(&file_struct, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_struct}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + const auto& projection = request.non_predicate_columns[0]; + EXPECT_EQ(projection.column_id(), LocalColumnId(0)); + ASSERT_FALSE(projection.project_all_children); + ASSERT_EQ(projection.children.size(), 1); + EXPECT_EQ(projection.children[0].local_id(), 1); + + ASSERT_EQ(mapper.mappings().size(), 1); + const auto* projected_type = assert_cast( + remove_nullable(mapper.mappings()[0].file_type).get()); + ASSERT_EQ(projected_type->get_elements().size(), 1); + EXPECT_EQ(projected_type->get_element_name(0), "b"); +} + +// Scenario: a row filter reaches a struct child through an array wrapper +// (`items.item.a > 5`). The mapper keeps this as a row predicate and reads the full array root for +// predicate evaluation. +TEST(ColumnMapperScanRequestTest, ArrayWrapperDoesNotBuildNestedPredicateFilter) { + const auto int_type = i32(); + const auto string_type = str(); + + auto file_a = name_col("a", int_type, 0); + auto file_b = name_col("b", string_type, 1); + auto file_element = struct_name_col("item", {file_a, file_b}, 0); + auto file_array = array_col("items", -1, file_element, 0); + set_name_identifiers(&file_array, 0); + + auto table_array = file_array; + + const auto item_type = file_element.type; + auto item_expr = struct_element(table_slot(0, 0, table_array.type, "items"), item_type, "item"); + auto filter_expr = int_gt(struct_element(item_expr, int_type, "a"), 5); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_array}, {}, {file_array}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_array}, &request).ok()); + + EXPECT_TRUE(request.non_predicate_columns.empty()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(0)); + EXPECT_TRUE(request.predicate_columns[0].project_all_children); + EXPECT_TRUE(request.predicate_columns[0].children.empty()); +} + +// Scenario: a map value struct projects child `b`, while a row filter reads value child `a`. +// The filter is too complex to become a file-local nested predicate, but the predicate projection +// must replace the output projection for the same map root and contain both physical value children. +TEST(ColumnMapperScanRequestTest, MapFilterOnlyValueChildMergesWithOutputProjection) { + const auto key_type = i32(); + const auto int_type = i32(); + const auto string_type = str(); + + auto table_value_b = name_col("b", string_type); + auto table_value = struct_name_col("value", {table_value_b}); + auto table_map = map_col("m", -1, {table_value}, key_type, table_value.type); + set_name_identifiers(&table_map, 0); + + auto file_key = name_col("key", key_type, 0); + auto file_value_a = name_col("a", int_type, 0); + auto file_value_b = name_col("b", string_type, 1); + auto file_value = struct_name_col("value", {file_value_a, file_value_b}, 1); + auto file_map = map_col("m", -1, {file_key, file_value}, key_type, file_value.type, 0); + set_name_identifiers(&file_map, 0); + + auto full_value_type = + std::make_shared(DataTypes {int_type, string_type}, Strings {"a", "b"}); + auto full_map_type = std::make_shared(key_type, full_value_type); + auto value_expr = + struct_element(table_slot(0, 0, full_map_type, "m"), full_value_type, "value"); + auto filter_expr = int_gt(struct_element(value_expr, int_type, "a"), 5); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_map}, {}, {file_map}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_map}, &request).ok()); + + EXPECT_TRUE(request.non_predicate_columns.empty()); + ASSERT_EQ(request.predicate_columns.size(), 1); + const auto& projection = request.predicate_columns[0]; + EXPECT_EQ(projection.column_id(), LocalColumnId(0)); + ASSERT_FALSE(projection.project_all_children); + ASSERT_EQ(projection.children.size(), 1); + EXPECT_EQ(projection.children[0].local_id(), 1); + EXPECT_EQ(projection_ids(projection.children[0].children), std::vector({0, 1})); +} + +// Scenario: when projected struct children are an in-order prefix of the file struct, the mapper can +// read those physical children directly without rebuilding the file-side complex type. +TEST(ColumnMapperScanRequestTest, MatchingProjectedStructDoesNotNeedComplexRematerialize) { + const auto int_type = i32(); + const auto string_type = str(); + + auto table_a = field_id_col("a", 1, int_type); + auto table_b = field_id_col("b", 2, string_type); + auto table_struct = struct_col("s", 10, {table_a, table_b}); + + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, string_type, 1); + auto file_c = field_id_col("c", 3, int_type, 2); + auto file_struct = struct_col("s", 10, {file_a, file_b, file_c}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_TRUE(mapper.mappings()[0].is_trivial); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_struct}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + const auto& projection = request.non_predicate_columns[0]; + EXPECT_FALSE(projection.project_all_children); + EXPECT_EQ(projection_ids(projection.children), std::vector({0, 1})); + EXPECT_TRUE(mapper.mappings()[0].is_trivial); +} + +// Scenario: Iceberg field-id mapping sees a renamed struct child, but the physical child order and +// types still match, so projection remains a full physical read instead of rebuilding a new type. +TEST(ColumnMapperScanRequestTest, RenameOnlyProjectedStructDoesNotRebuildFileProjection) { + const auto int_type = i32(); + + auto table_a = field_id_col("a", 1, int_type); + auto table_renamed_b = field_id_col("renamed_b", 2, int_type); + auto table_struct = struct_col("s", 10, {table_a, table_renamed_b}); + + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, int_type, 1); + auto file_struct = struct_col("s", 10, {file_a, file_b}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_TRUE(mapper.mappings()[0].is_trivial); + EXPECT_EQ(mapper.mappings()[0].projected_file_children.size(), + mapper.mappings()[0].original_file_children.size()); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + EXPECT_EQ(mapper.mappings()[0].child_mappings[1].table_column_name, "renamed_b"); + EXPECT_EQ(mapper.mappings()[0].child_mappings[1].file_column_name, "b"); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_struct}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_TRUE(request.non_predicate_columns[0].project_all_children); + EXPECT_TRUE(request.non_predicate_columns[0].children.empty()); + EXPECT_TRUE(mapper.mappings()[0].is_trivial); +} + +// Scenario: a row filter references an unprojected struct child, so the predicate projection is +// merged with the output projection and the mapper rebuilds the projected file struct type. +TEST(ColumnMapperScanRequestTest, PredicateProjectionRebuildsProjectedStructFileType) { + const auto int_type = i32(); + const auto string_type = str(); + + auto table_a = field_id_col("a", 1, int_type); + auto table_b = field_id_col("b", 2, string_type); + auto table_struct = struct_col("s", 10, {table_a, table_b}); + auto full_table_c = field_id_col("c", 3, int_type); + auto full_table_struct = struct_col("s", 10, {table_a, table_b, full_table_c}); + + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, string_type, 1); + auto file_c = field_id_col("c", 3, int_type, 2); + auto file_struct = struct_col("s", 10, {file_a, file_b, file_c}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + auto filter_expr = + int_gt(struct_element(table_slot(0, 0, full_table_struct.type, "s"), int_type, "c"), 0); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); + + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_TRUE(request.non_predicate_columns.empty()); + const auto& projection = request.predicate_columns[0]; + EXPECT_FALSE(projection.project_all_children); + EXPECT_EQ(projection_ids(projection.children), std::vector({0, 1, 2})); + + const auto* mapped_type = assert_cast( + remove_nullable(mapper.mappings()[0].file_type).get()); + ASSERT_EQ(mapped_type->get_elements().size(), 3); + EXPECT_EQ(mapped_type->get_element_name(0), "a"); + EXPECT_EQ(mapped_type->get_element_name(1), "b"); + EXPECT_EQ(mapped_type->get_element_name(2), "c"); + EXPECT_FALSE(mapper.mappings()[0].is_trivial); +} + +// Scenario: a filter references a top-level column that is not projected by the query; the mapper +// creates a hidden filter mapping without adding that hidden column to visible table mappings. +TEST(ColumnMapperScanRequestTest, PredicateOnlyTopLevelColumnUsesHiddenMapping) { + const auto int_type = i32(); + + auto table_id = field_id_col("id", 0, int_type); + auto table_c = field_id_col("c", 11, int_type); + auto table_struct = struct_col("s", 10, {table_c}); + + auto file_id = field_id_col("id", 0, int_type, 0); + auto file_c = field_id_col("c", 11, int_type, 0); + auto file_struct = struct_col("s", 10, {file_c}, 10); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_id}, {}, {file_id, file_struct}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_EQ(mapper.mappings()[0].table_column_name, "id"); + + auto filter_expr = + int_gt(struct_element(table_slot(7, 1, table_struct.type, "s"), int_type, "c"), 0); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(1)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_id}, &request).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_EQ(mapper.mappings()[0].table_column_name, "id"); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(10)); + EXPECT_TRUE(request.predicate_columns[0].project_all_children); + EXPECT_TRUE(request.predicate_columns[0].children.empty()); + + ASSERT_EQ(request.conjuncts.size(), 1); +} + +// Scenario: a nested predicate targets a table-side renamed struct field; scan projection must +// resolve that field to the old physical file child. +TEST(ColumnMapperScanRequestTest, NestedPredicateProjectionUsesMappedRenamedChild) { + const auto int_type = i32(); + + auto table_a = field_id_col("a", 1, int_type); + auto table_renamed_b = field_id_col("renamed_b", 2, int_type); + auto table_struct = struct_col("s", 10, {table_a, table_renamed_b}); + + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, int_type, 1); + auto file_struct = struct_col("s", 10, {file_a, file_b}, 10); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + auto filter_expr = int_gt( + struct_element(table_slot(0, 0, table_struct.type, "s"), int_type, "renamed_b"), 0); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_TRUE(request.predicate_columns[0].project_all_children); + EXPECT_TRUE(request.predicate_columns[0].children.empty()); +} + +// Scenario: element_at(struct, 'table_name') in a row filter is localized to the physical file +// child name, matching the struct_element rewrite path. +TEST(ColumnMapperScanRequestTest, + FileLocalElementAtConjunctUsesFileChildNameForRenamedStructField) { + const auto int_type = i32(); + + auto table_a = field_id_col("a", 1, int_type); + auto table_renamed_b = field_id_col("renamed_b", 2, int_type); + auto table_struct = struct_col("s", 10, {table_a, table_renamed_b}); + + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, int_type, 1); + auto file_struct = struct_col("s", 10, {file_a, file_b}, 10); + + auto child_expr = element_at(table_slot(0, 0, table_struct.type, table_struct.name), int_type, + "renamed_b"); + auto filter_expr = int_gt(child_expr, 0); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); + + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_child = request.conjuncts[0]->root()->children()[0]; + EXPECT_EQ(localized_child->expr_name(), "element_at"); + const auto* localized_slot = assert_cast(localized_child->children()[0].get()); + EXPECT_EQ(localized_slot->column_name(), "s"); + EXPECT_EQ(localized_slot->column_id(), 0); + + const auto* localized_literal = + assert_cast(localized_child->children()[1].get()); + Field localized_field; + localized_literal->get_column_ptr()->get(0, localized_field); + ASSERT_EQ(localized_field.get_type(), TYPE_STRING); + EXPECT_EQ(std::string(localized_field.as_string_view()), "b"); +} + +// Scenario: nested element_at(struct, name) localization rewrites both selector names and +// intermediate return types. The outer selector must be prepared against the projected file child +// struct, not the table child struct or the full historical file child struct. +TEST(ColumnMapperScanRequestTest, NestedElementAtConjunctUsesFileChildTypeForRenamedLeaf) { + const auto int_type = i32(); + const auto string_type = str(); + + auto table_new_aa = field_id_col("new_aa", 23, int_type); + auto table_bb = field_id_col("bb", 24, string_type); + auto table_new_a = struct_col("new_a", 20, {table_new_aa, table_bb}); + auto table_struct = struct_col("struct_column2", 19, {table_new_a}); + + auto file_aa = field_id_col("aa", 23, int_type, 0); + auto file_bb = field_id_col("bb", 24, string_type, 1); + auto file_new_a = struct_col("new_a", 20, {file_aa, file_bb}, 0); + auto file_struct = struct_col("struct_column2", 19, {file_new_a}, 10); + + const auto table_slot_expr = table_slot(0, 0, table_struct.type, "struct_column2"); + const auto table_parent_expr = element_at(table_slot_expr, table_new_a.type, "new_a"); + const auto table_leaf_expr = element_at(table_parent_expr, int_type, "new_aa"); + auto filter_expr = binary_predicate(TExprOpcode::EQ, table_leaf_expr, + literal(int_type, Field::create_field(50))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request).ok()); + ASSERT_EQ(request.conjuncts.size(), 1); + + const auto& localized_leaf = request.conjuncts[0]->root()->children()[0]; + ASSERT_EQ(localized_leaf->expr_name(), "element_at"); + const auto& localized_parent = localized_leaf->children()[0]; + ASSERT_EQ(localized_parent->expr_name(), "element_at"); + + const auto* localized_leaf_selector = + assert_cast(localized_leaf->children()[1].get()); + Field localized_leaf_field; + localized_leaf_selector->get_column_ptr()->get(0, localized_leaf_field); + ASSERT_EQ(localized_leaf_field.get_type(), TYPE_STRING); + EXPECT_EQ(std::string(localized_leaf_field.as_string_view()), "aa"); + + const auto* localized_parent_type = assert_cast( + remove_nullable(localized_parent->data_type()).get()); + ASSERT_EQ(localized_parent_type->get_elements().size(), 2); + EXPECT_EQ(localized_parent_type->get_element_name(0), "aa"); + EXPECT_EQ(localized_parent_type->get_element_name(1), "bb"); +} + +// Scenario: output projection reads one struct child while the row filter reads a different nested +// struct child. File-local conjunct rewrite must use the merged scan projection type. In the SQL +// shape below, `SELECT element_at(s, 'c') WHERE element_at(element_at(s, 'b'), 'cc') LIKE ...` +// reads file children `b.cc` and `c`; the localized inner `element_at(s, 'b')` returns +// `Struct(cc)`, not the full old file child `Struct(cc, new_dd)`. +TEST(ColumnMapperScanRequestTest, NestedElementAtConjunctUsesMergedScanProjectionChildType) { + const auto string_type = str(); + const auto int_type = i32(); + + auto table_cc = field_id_col("cc", 23, string_type); + auto table_new_dd = field_id_col("new_dd", 24, int_type); + auto table_b = struct_col("b", 20, {table_cc, table_new_dd}); + auto table_c = field_id_col("c", 25, string_type); + auto full_table_struct = struct_col("struct_column2", 19, {table_b, table_c}); + auto projected_table_struct = struct_col("struct_column2", 19, {table_c}); + + auto file_cc = field_id_col("cc", 23, string_type, 0); + auto file_new_dd = field_id_col("new_dd", 24, int_type, 1); + auto file_b = struct_col("b", 20, {file_cc, file_new_dd}, 0); + auto file_c = field_id_col("c", 25, string_type, 1); + auto file_struct = struct_col("new_struct_column", 19, {file_b, file_c}, 10); + + const auto table_slot_expr = table_slot(0, 0, full_table_struct.type, "struct_column2"); + const auto table_parent_expr = element_at(table_slot_expr, table_b.type, "b"); + const auto table_leaf_expr = element_at(table_parent_expr, string_type, "cc"); + auto filter_expr = like_expr(table_leaf_expr, "NestedC%"); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({projected_table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {projected_table_struct}, &request).ok()); + ASSERT_EQ(request.conjuncts.size(), 1); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(10)); + + const auto& localized_leaf = request.conjuncts[0]->root()->children()[0]; + ASSERT_EQ(localized_leaf->expr_name(), "element_at"); + const auto& localized_parent = localized_leaf->children()[0]; + ASSERT_EQ(localized_parent->expr_name(), "element_at"); + + const auto* localized_slot = + assert_cast(localized_parent->children()[0].get()); + EXPECT_EQ(localized_slot->column_name(), "new_struct_column"); + // The scan projection keeps the top-level file column id above, while the localized conjunct + // executes on the file-reader Block. The VSlotRef column id is therefore the block position of + // `new_struct_column` in this request, not the file schema id 10. + EXPECT_EQ(localized_slot->column_id(), 0); + + const auto* localized_parent_type = assert_cast( + remove_nullable(localized_parent->data_type()).get()); + ASSERT_EQ(localized_parent_type->get_elements().size(), 1); + EXPECT_EQ(localized_parent_type->get_element_name(0), "cc"); +} + +// Scenario: struct child access through a computed map/array parent is not localized as a file +// conjunct, because the projected value struct can have a different physical child order. +TEST(ColumnMapperScanRequestTest, MapValuesStructChildConjunctStaysTableLevel) { + const auto key_type = str(); + const auto string_type = str(); + const auto int_type = i32(); + + auto table_gender = field_id_col("gender", 17, string_type); + auto table_full_name = field_id_col("full_name", 7, string_type); + auto table_value = struct_col("value", 6, {table_gender, table_full_name}); + auto table_map = map_col("new_map_column", 2, {table_value}, key_type, table_value.type); + + auto file_key = field_id_col("key", 5, key_type, 0); + auto file_age = field_id_col("age", 8, int_type, 0); + auto file_full_name = field_id_col("full_name", 7, string_type, 1); + auto file_gender = field_id_col("gender", 17, string_type, 2); + auto file_value = struct_col("value", 6, {file_age, file_full_name, file_gender}, 1); + auto file_map = + map_col("new_map_column", 2, {file_key, file_value}, key_type, file_value.type, 1); + + const auto map_slot = table_slot(0, 0, table_map.type, "new_map_column"); + const auto values_expr = map_values(map_slot, table_value.type); + const auto first_value = array_element_at(values_expr, table_value.type, 1); + const auto full_name_expr = element_at(first_value, string_type, "full_name"); + auto filter_expr = like_expr(full_name_expr, "J%"); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_map}, {}, {file_map}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_map}, &request).ok()); + + EXPECT_TRUE(request.conjuncts.empty()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + ASSERT_FALSE(request.predicate_columns[0].project_all_children); + ASSERT_EQ(request.predicate_columns[0].children.size(), 1); + EXPECT_EQ(request.predicate_columns[0].children[0].local_id(), 1); +} + +// Scenario: MAP_KEYS only reads map keys, but localizing it by wrapping the evolved file map slot +// in CAST(file_map AS table_map) would still cast the old value struct to the new value struct. +// Keep the conjunct table-level when the map value schema changed. +TEST(ColumnMapperScanRequestTest, MapKeysConjunctWithEvolvedValueStructStaysTableLevel) { + const auto key_type = str(); + const auto string_type = str(); + const auto int_type = i32(); + + auto table_age = field_id_col("age", 8, int_type); + auto table_full_name = field_id_col("full_name", 7, string_type); + auto table_gender = field_id_col("gender", 17, string_type); + auto table_value = struct_col("value", 6, {table_age, table_full_name, table_gender}); + auto table_key = field_id_col("key", 5, key_type); + auto table_map = + map_col("new_map_column", 2, {table_key, table_value}, key_type, table_value.type); + + auto file_key = field_id_col("key", 5, key_type, 0); + auto file_name = field_id_col("name", 18, string_type, 0); + auto file_age = field_id_col("age", 8, int_type, 1); + auto file_value = struct_col("value", 6, {file_name, file_age}, 1); + auto file_map = map_col("map_column", 2, {file_key, file_value}, key_type, file_value.type, 1); + + const auto map_slot = table_slot(0, 0, table_map.type, "new_map_column"); + const auto keys_expr = map_keys(map_slot, key_type); + auto filter_expr = array_contains( + keys_expr, literal(key_type, Field::create_field("person5"))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_map}, {}, {file_map}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_map}, &request).ok()); + + EXPECT_TRUE(request.conjuncts.empty()); + EXPECT_TRUE(request.non_predicate_columns.empty()); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); +} + +// Scenario: an array element struct projection only contains missing/default children; the mapper +// falls back to reading the full physical element so the reader never gets an empty projection. +TEST(ColumnMapperScanRequestTest, ArrayStructOnlyMissingElementChildUsesFullFileProjection) { + const auto int_type = i32(); + const auto string_type = str(); + + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, int_type, 1); + auto file_element = struct_col("element", 0, {file_a, file_b}, 0); + auto file_array = array_col("xs", 10, file_element, 10); + + auto missing_child = field_id_col("missing_child", 99, string_type); + auto table_element = struct_col("element", 0, {missing_child}); + auto table_array = array_col("xs", 10, table_element); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_array}, {}, {file_array}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_array}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(10)); + EXPECT_TRUE(request.non_predicate_columns[0].project_all_children); + EXPECT_TRUE(request.non_predicate_columns[0].children.empty()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_FALSE(mapper.mappings()[0].is_trivial); +} + +// Scenario: a map value struct projection only contains missing/default children; the mapper keeps +// the map key/value shape and reads the full physical value struct instead of an empty value child. +TEST(ColumnMapperScanRequestTest, MapValueStructOnlyMissingChildUsesFullValueProjection) { + const auto key_type = i32(); + const auto int_type = i32(); + const auto string_type = str(); + + auto file_key = field_id_col("key", 0, key_type, 0); + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, int_type, 1); + auto file_value = struct_col("value", 1, {file_a, file_b}, 1); + auto file_map = map_col("m", 10, {file_key, file_value}, key_type, file_value.type, 10); + + auto missing_child = field_id_col("missing_child", 99, string_type); + auto table_value = struct_col("value", 1, {missing_child}); + auto table_map = map_col("m", 10, {table_value}, key_type, table_value.type); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_map}, {}, {file_map}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_map}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + const auto& projection = request.non_predicate_columns[0]; + EXPECT_EQ(projection.column_id(), LocalColumnId(10)); + ASSERT_FALSE(projection.project_all_children); + ASSERT_EQ(projection.children.size(), 1); + EXPECT_EQ(projection.children[0].local_id(), 1); + EXPECT_TRUE(projection.children[0].project_all_children); + EXPECT_TRUE(projection.children[0].children.empty()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_FALSE(mapper.mappings()[0].is_trivial); +} + +// ---------------------------------------------------------------------- +// L1 complex schema evolution and split isolation. +// These tests call the mapper repeatedly with different file schemas and +// verify that split-local state is rebuilt instead of leaked. +// ---------------------------------------------------------------------- + +TEST(ColumnMapperSchemaEvolutionTest, StructChildrenHandleMissingRenameReorderAndDroppedFields) { + const auto int_type = i32(); + const auto string_type = str(); + auto table_a = field_id_col("a", 1, int_type); + auto table_renamed_b = field_id_col("renamed_b", 2, string_type); + auto table_c = field_id_col("c", 3, int_type); + auto table_struct = struct_col("s", 10, {table_a, table_renamed_b, table_c}); + + auto v1_a = field_id_col("a", 1, int_type, 0); + auto v1_b = field_id_col("b", 2, string_type, 1); + auto file_v1 = struct_col("s", 10, {v1_a, v1_b}, 5); + + auto v2_b = field_id_col("b", 2, string_type, 0); + auto v2_a = field_id_col("a", 1, int_type, 1); + auto v2_c = field_id_col("c", 3, int_type, 2); + auto file_v2 = struct_col("s", 10, {v2_b, v2_a, v2_c}, 8); + + TableColumnMapper v1_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(v1_mapper.create_mapping({table_struct}, {}, {file_v1}).ok()); + FileScanRequest v1_request; + ASSERT_TRUE(v1_mapper.create_scan_request({}, {table_struct}, &v1_request).ok()); + + const auto& v1_mapping = v1_mapper.mappings()[0]; + ASSERT_EQ(v1_mapping.child_mappings.size(), 3); + EXPECT_EQ(*v1_mapping.child_mappings[0].file_local_id, 0); + EXPECT_EQ(*v1_mapping.child_mappings[1].file_local_id, 1); + EXPECT_FALSE(v1_mapping.child_mappings[2].file_local_id.has_value()); + ASSERT_EQ(v1_request.non_predicate_columns.size(), 1); + EXPECT_EQ(v1_request.non_predicate_columns[0].column_id(), LocalColumnId(5)); + EXPECT_TRUE(v1_request.non_predicate_columns[0].project_all_children); + + TableColumnMapper v2_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(v2_mapper.create_mapping({table_struct}, {}, {file_v2}).ok()); + FileScanRequest v2_request; + ASSERT_TRUE(v2_mapper.create_scan_request({}, {table_struct}, &v2_request).ok()); + + const auto& v2_mapping = v2_mapper.mappings()[0]; + ASSERT_EQ(v2_mapping.child_mappings.size(), 3); + EXPECT_EQ(*v2_mapping.child_mappings[0].file_local_id, 1); + EXPECT_EQ(*v2_mapping.child_mappings[1].file_local_id, 0); + EXPECT_EQ(*v2_mapping.child_mappings[2].file_local_id, 2); + ASSERT_EQ(v2_request.non_predicate_columns.size(), 1); + EXPECT_EQ(v2_request.non_predicate_columns[0].column_id(), LocalColumnId(8)); + EXPECT_TRUE(v2_request.non_predicate_columns[0].project_all_children); +} + +TEST(ColumnMapperSchemaEvolutionTest, DroppedStructChildrenAreNotRead) { + const auto int_type = i32(); + const auto string_type = str(); + auto table_a = field_id_col("a", 1, int_type); + auto table_struct = struct_col("s", 10, {table_a}); + + auto file_a = field_id_col("a", 1, int_type, 0); + auto file_b = field_id_col("b", 2, string_type, 1); + auto file_c = field_id_col("c", 3, int_type, 2); + auto file_struct = struct_col("s", 10, {file_a, file_b, file_c}, 5); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({}, {table_struct}, &request).ok()); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + const auto& projection = request.non_predicate_columns[0]; + EXPECT_EQ(projection.column_id(), LocalColumnId(5)); + ASSERT_FALSE(projection.project_all_children); + EXPECT_EQ(projection_ids(projection.children), std::vector({0})); +} + +TEST(ColumnMapperSchemaEvolutionTest, ReusedMapperClearsSplitLocalConstantsAndFileIds) { + const auto int_type = i32(); + auto id = name_col("id", int_type); + auto added = name_col("added", int_type); + added.default_expr = + VExprContext::create_shared(literal(int_type, Field::create_field(7))); + const std::vector table_schema = {id, added}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, {name_col("id", int_type, 0)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 2); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); + expect_constant(mapper, mapper.mappings()[1], 1, int_type); + + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, + {name_col("id", int_type, 3), name_col("added", int_type, 4)}) + .ok()); + ASSERT_EQ(mapper.mappings().size(), 2); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 3); + EXPECT_EQ(*mapper.mappings()[1].file_local_id, 4); + EXPECT_TRUE(mapper.constant_map().empty()); +} + +// ---------------------------------------------------------------------- +// L2 cast-aware filter localization tests. +// These tests belong to TableColumnMapper rather than Cast: they assert when the mapper builds +// projection casts, rewrites table predicates to file-local slot casts, converts literals to the +// current split's file type, and keeps repeated scan-request rewrites idempotent. +// ---------------------------------------------------------------------- + +// Scenario: table/file primitive types differ, so the visible mapping must build a cast projection. +TEST_F(ColumnMapperCastTest, ColumnMapperBuildsCastProjectionForTypeMismatch) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(mapper.mappings().size(), 1); + FileScanRequest file_request; + status = mapper.create_scan_request({}, projected_columns, &file_request); + ASSERT_TRUE(status.ok()) << status; + const auto& mapping = mapper.mappings()[0]; + EXPECT_FALSE(mapping.is_trivial); + ASSERT_NE(mapping.projection, nullptr); + + Block block; + block.insert(ColumnHelper::create_column_with_name({11, 22})); + int result_column_id = -1; + status = prepare_open_execute(mapping.projection.get(), &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + const auto& result_column = + assert_cast(*block.get_by_position(result_column_id).column); + EXPECT_EQ(result_column.get_data()[0], 11); + EXPECT_EQ(result_column.get_data()[1], 22); + + mapping.projection->close(); +} + +// Scenario: equivalent table/file types keep the mapping trivial and avoid unnecessary projection casts. +TEST_F(ColumnMapperCastTest, ColumnMapperTreatsEquivalentTypesAsTrivial) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i32()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_TRUE(mapper.mappings()[0].is_trivial); +} + +// Scenario: a table predicate on a widened type is localized by casting the file slot to table type. +TEST_F(ColumnMapperCastTest, ColumnMapperBuildsCastFilterForTypeMismatch) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = std::make_shared(15); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + ASSERT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 1); + const auto& localized_child = localized_expr->children()[0]; + ASSERT_NE(dynamic_cast(localized_child.get()), nullptr); + ASSERT_EQ(localized_child->get_num_children(), 1); + const auto* localized_slot = assert_cast(localized_child->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_child->data_type()->equals(*table_column.type)); + + Block block; + block.insert(ColumnHelper::create_column_with_name({11, 22})); + auto* conjunct = file_request.conjuncts[0].get(); + status = conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + IColumn::Filter filter(block.rows(), 1); + bool can_filter_all = false; + status = conjunct->execute_filter(&block, filter.data(), block.rows(), false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(can_filter_all); + ASSERT_EQ(filter.size(), 2); + EXPECT_EQ(filter[0], 0); + EXPECT_EQ(filter[1], 1); + + file_request.conjuncts[0]->close(); +} + +// Scenario: an already prepared table filter can still be cloned, rewritten, prepared, and opened as a file-local filter. +TEST_F(ColumnMapperCastTest, ColumnMapperRepreparesRewrittenPreparedFilter) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto cast = Cast::create_shared(table_column.type); + cast->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(cast); + table_filter.global_indices = {GlobalIndex(0)}; + status = table_filter.conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = table_filter.conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_NE(dynamic_cast(localized_expr.get()), nullptr); + ASSERT_EQ(localized_expr->get_num_children(), 1); + const auto* localized_slot = assert_cast(localized_expr->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + + status = file_request.conjuncts[0]->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = file_request.conjuncts[0]->open(&state); + ASSERT_TRUE(status.ok()) << status; + + file_request.conjuncts[0]->close(); +} + +// Scenario: slot-literal comparison rewrites the literal to the current file type when conversion is safe. +TEST_F(ColumnMapperCastTest, ColumnMapperCastsLiteralForSlotLiteralPredicateTypeMismatch) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = std::make_shared(TExprOpcode::GT); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(15))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + ASSERT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + const auto* localized_slot = assert_cast(localized_expr->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + const auto& localized_literal = localized_expr->children()[1]; + EXPECT_TRUE(localized_literal->is_literal()); + EXPECT_TRUE(localized_literal->data_type()->equals(*file_field.type)); + + Block block; + block.insert(ColumnHelper::create_column_with_name({11, 22})); + auto* conjunct = file_request.conjuncts[0].get(); + status = conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + IColumn::Filter filter(block.rows(), 1); + bool can_filter_all = false; + status = conjunct->execute_filter(&block, filter.data(), block.rows(), false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(can_filter_all); + ASSERT_EQ(filter.size(), 2); + EXPECT_EQ(filter[0], 0); + EXPECT_EQ(filter[1], 1); + + file_request.conjuncts[0]->close(); +} + +// Scenario: literal-slot comparison also rewrites the literal side and preserves operand order. +TEST_F(ColumnMapperCastTest, ColumnMapperCastsLiteralForLiteralSlotPredicateTypeMismatch) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = std::make_shared(TExprOpcode::LT); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(15))); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + const auto& localized_literal = localized_expr->children()[0]; + EXPECT_TRUE(localized_literal->is_literal()); + EXPECT_TRUE(localized_literal->data_type()->equals(*file_field.type)); + const auto* localized_slot = assert_cast(localized_expr->children()[1].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + + Block block; + block.insert(ColumnHelper::create_column_with_name({11, 22})); + auto* conjunct = file_request.conjuncts[0].get(); + status = conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + IColumn::Filter filter(block.rows(), 1); + bool can_filter_all = false; + status = conjunct->execute_filter(&block, filter.data(), block.rows(), false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(can_filter_all); + ASSERT_EQ(filter.size(), 2); + EXPECT_EQ(filter[0], 0); + EXPECT_EQ(filter[1], 1); + + file_request.conjuncts[0]->close(); +} + +// Scenario: Nereids may represent `BIGINT value = 1` as `value = CAST(INT 1 AS BIGINT)`. +// Strip a provably lossless literal widening so metadata readers can recognize slot-literal +// predicates for zone-map pruning. +TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesImplicitlyCastLiteral) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + auto file_field = name_col("value", i64(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = std::make_shared(TExprOpcode::GT); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child(cast_expr(VLiteral::create_shared(i32(), Field::create_field(1)), + table_column.type)); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + EXPECT_TRUE(localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); + + Block block; + block.insert(ColumnHelper::create_column_with_name({1, 2})); + auto* conjunct = file_request.conjuncts[0].get(); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor()).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + IColumn::Filter filter(block.rows(), 1); + bool can_filter_all = false; + ASSERT_TRUE( + conjunct->execute_filter(&block, filter.data(), block.rows(), false, &can_filter_all) + .ok()); + EXPECT_EQ(filter, IColumn::Filter({0, 1})); + conjunct->close(); +} + +// Scenario: Nereids may keep a narrow literal directly under an implicitly coerced comparison, +// for example `nullable INT value = TINYINT 1`. Normalize the literal to the table type so the +// file predicate remains recognizable to Parquet zone-map pruning. +TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesImplicitlyTypedLiteral) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto column_type = make_nullable(i32()); + auto table_column = name_col("value", column_type); + std::vector projected_columns {table_column}; + auto file_field = name_col("value", column_type, 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = std::make_shared(TExprOpcode::GT); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(std::make_shared(), + Field::create_field(static_cast(1)))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + EXPECT_TRUE(localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); +} + +// Scenario: branch-4.1 Nereids comparisons can be resolved by function name without retaining the +// legacy opcode. The node kind still identifies a binary comparison, so its narrow literal must be +// normalized exactly like the opcode-bearing form used above. +TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesBinaryPredicateWithoutLegacyOpcode) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto column_type = make_nullable(i32()); + auto table_column = name_col("value", column_type); + std::vector projected_columns {table_column}; + auto file_field = name_col("value", column_type, 0); + + auto status = mapper.create_mapping(projected_columns, {}, {file_field}); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = std::make_shared(TExprOpcode::INVALID_OPCODE); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(std::make_shared(), + Field::create_field(static_cast(1)))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + EXPECT_TRUE(localized_expr->children()[0]->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); +} + +// Scenario: a fractional table literal cannot be localized to an integral file type without +// changing the predicate boundary, so the mapper must cast the file slot instead. +TEST_F(ColumnMapperCastTest, ColumnMapperRejectsLossyBinaryLiteralConversion) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", f64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = binary_predicate( + TExprOpcode::LT, VSlotRef::create_shared(0, 0, -1, table_column.type, "value"), + VLiteral::create_shared(table_column.type, Field::create_field(1.5))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + const auto& localized_slot_cast = localized_expr->children()[0]; + ASSERT_NE(dynamic_cast(localized_slot_cast.get()), nullptr); + EXPECT_TRUE(localized_slot_cast->data_type()->equals(*table_column.type)); + ASSERT_EQ(localized_slot_cast->get_num_children(), 1); + const auto* localized_slot = + assert_cast(localized_slot_cast->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*table_column.type)); +} + +// Scenario: an exactly representable literal is still unsafe to localize when arbitrary file +// values lose information during materialization to the table type. +TEST_F(ColumnMapperCastTest, ColumnMapperRejectsLossyFileToTableConversion) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", f64(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = binary_predicate( + TExprOpcode::EQ, VSlotRef::create_shared(0, 0, -1, table_column.type, "value"), + VLiteral::create_shared(table_column.type, Field::create_field(1))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + const auto& localized_slot_cast = localized_expr->children()[0]; + ASSERT_NE(dynamic_cast(localized_slot_cast.get()), nullptr); + EXPECT_TRUE(localized_slot_cast->data_type()->equals(*table_column.type)); + ASSERT_EQ(localized_slot_cast->get_num_children(), 1); + const auto* localized_slot = + assert_cast(localized_slot_cast->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*table_column.type)); +} + +// Scenario: complex Field equality does not compare nested values, so complex literals must not +// use the scalar round-trip guard. +TEST_F(ColumnMapperCastTest, ColumnMapperRejectsComplexLiteralLocalization) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = array_col("value", -1, name_col("element", f64())); + set_name_identifiers(&table_column, -1); + const auto& table_type = table_column.type; + std::vector projected_columns {table_column}; + + auto file_field = array_col("value", -1, name_col("element", i32()), 0); + set_name_identifiers(&file_field, 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + Array literal_values {Field::create_field(1.5)}; + auto predicate = binary_predicate( + TExprOpcode::EQ, VSlotRef::create_shared(0, 0, -1, table_type, "value"), + VLiteral::create_shared(table_type, Field::create_field(literal_values))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + EXPECT_TRUE(file_request.conjuncts.empty()); +} + +// Scenario: IN predicate literals are all rewritten to file type when every literal conversion is safe. +TEST_F(ColumnMapperCastTest, ColumnMapperCastsInPredicateLiteralsForTypeMismatch) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = create_in_predicate(); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(15))); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(22))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + ASSERT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 3); + const auto* localized_slot = assert_cast(localized_expr->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_expr->children()[2]->is_literal()); + EXPECT_TRUE(localized_expr->children()[2]->data_type()->equals(*file_field.type)); +} + +// Scenario: one lossy IN literal prevents the entire predicate from being localized to file type. +TEST_F(ColumnMapperCastTest, ColumnMapperRejectsLossyInPredicateLiteralConversion) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", f64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = create_in_predicate(); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(1.0))); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(1.5))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 3); + const auto& localized_slot_cast = localized_expr->children()[0]; + ASSERT_NE(dynamic_cast(localized_slot_cast.get()), nullptr); + EXPECT_TRUE(localized_slot_cast->data_type()->equals(*table_column.type)); + ASSERT_EQ(localized_slot_cast->get_num_children(), 1); + const auto* localized_slot = + assert_cast(localized_slot_cast->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*table_column.type)); + EXPECT_TRUE(localized_expr->children()[2]->is_literal()); + EXPECT_TRUE(localized_expr->children()[2]->data_type()->equals(*table_column.type)); +} + +// Scenario: IN predicate falls back to casting the file slot when any literal cannot be converted safely. +TEST_F(ColumnMapperCastTest, ColumnMapperFallsBackToSlotCastWhenInPredicateLiteralRewriteFails) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", str()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = create_in_predicate(); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field("10"))); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field("bad"))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 3); + const auto& localized_child = localized_expr->children()[0]; + ASSERT_NE(dynamic_cast(localized_child.get()), nullptr); + ASSERT_EQ(localized_child->get_num_children(), 1); + const auto* localized_slot = assert_cast(localized_child->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_child->data_type()->equals(*table_column.type)); + EXPECT_TRUE(localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*table_column.type)); + EXPECT_TRUE(localized_expr->children()[2]->is_literal()); + EXPECT_TRUE(localized_expr->children()[2]->data_type()->equals(*table_column.type)); +} + +// Scenario: split-local IN literal rewrites do not mutate the original table filter across different file schemas. +TEST_F(ColumnMapperCastTest, ColumnMapperDoesNotLeakRewrittenInPredicateLiteralAcrossSplits) { + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto predicate = create_in_predicate(); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(15))); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(22))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + auto int_file_field = name_col("value", i32(), 0); + TableColumnMapper int_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(int_mapper.create_mapping(projected_columns, {}, {int_file_field}).ok()); + FileScanRequest int_request; + ASSERT_TRUE( + int_mapper.create_scan_request({table_filter}, projected_columns, &int_request, &state) + .ok()); + ASSERT_EQ(int_request.conjuncts.size(), 1); + const auto& int_localized_expr = int_request.conjuncts[0]->root(); + ASSERT_EQ(int_localized_expr->get_num_children(), 3); + EXPECT_TRUE(int_localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(int_localized_expr->children()[1]->data_type()->equals(*int_file_field.type)); + EXPECT_TRUE(int_localized_expr->children()[2]->is_literal()); + EXPECT_TRUE(int_localized_expr->children()[2]->data_type()->equals(*int_file_field.type)); + + auto bigint_file_field = name_col("value", i64(), 0); + TableColumnMapper bigint_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(bigint_mapper.create_mapping(projected_columns, {}, {bigint_file_field}).ok()); + FileScanRequest bigint_request; + ASSERT_TRUE( + bigint_mapper + .create_scan_request({table_filter}, projected_columns, &bigint_request, &state) + .ok()); + ASSERT_EQ(bigint_request.conjuncts.size(), 1); + const auto& bigint_localized_expr = bigint_request.conjuncts[0]->root(); + ASSERT_EQ(bigint_localized_expr->get_num_children(), 3); + const auto* localized_slot = + assert_cast(bigint_localized_expr->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*bigint_file_field.type)); + EXPECT_TRUE(bigint_localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(bigint_localized_expr->children()[1]->data_type()->equals(*bigint_file_field.type)); + EXPECT_TRUE(bigint_localized_expr->children()[2]->is_literal()); + EXPECT_TRUE(bigint_localized_expr->children()[2]->data_type()->equals(*bigint_file_field.type)); +} + +// Scenario: binary predicate falls back to casting the file slot when literal conversion fails. +TEST_F(ColumnMapperCastTest, ColumnMapperFallsBackToSlotCastWhenLiteralRewriteFails) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", str()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = std::make_shared(TExprOpcode::GT); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field("bad"))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + const auto& localized_child = localized_expr->children()[0]; + ASSERT_NE(dynamic_cast(localized_child.get()), nullptr); + ASSERT_EQ(localized_child->get_num_children(), 1); + const auto* localized_slot = assert_cast(localized_child->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); + EXPECT_TRUE(localized_child->data_type()->equals(*table_column.type)); + EXPECT_TRUE(localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*table_column.type)); +} + +// Scenario: split-local binary literal rewrite does not leak into a later split with a different file type. +TEST_F(ColumnMapperCastTest, ColumnMapperDoesNotLeakRewrittenLiteralAcrossSplits) { + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto predicate = std::make_shared(TExprOpcode::GT); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(15))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + auto int_file_field = name_col("value", i32(), 0); + TableColumnMapper int_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(int_mapper.create_mapping(projected_columns, {}, {int_file_field}).ok()); + FileScanRequest int_request; + ASSERT_TRUE( + int_mapper.create_scan_request({table_filter}, projected_columns, &int_request, &state) + .ok()); + ASSERT_EQ(int_request.conjuncts.size(), 1); + const auto& int_localized_expr = int_request.conjuncts[0]->root(); + ASSERT_EQ(int_localized_expr->get_num_children(), 2); + EXPECT_TRUE(int_localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(int_localized_expr->children()[1]->data_type()->equals(*int_file_field.type)); + + auto bigint_file_field = name_col("value", i64(), 0); + TableColumnMapper bigint_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(bigint_mapper.create_mapping(projected_columns, {}, {bigint_file_field}).ok()); + FileScanRequest bigint_request; + ASSERT_TRUE( + bigint_mapper + .create_scan_request({table_filter}, projected_columns, &bigint_request, &state) + .ok()); + ASSERT_EQ(bigint_request.conjuncts.size(), 1); + const auto& bigint_localized_expr = bigint_request.conjuncts[0]->root(); + ASSERT_EQ(bigint_localized_expr->get_num_children(), 2); + const auto* localized_slot = + assert_cast(bigint_localized_expr->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*bigint_file_field.type)); + EXPECT_TRUE(bigint_localized_expr->children()[1]->is_literal()); + EXPECT_TRUE(bigint_localized_expr->children()[1]->data_type()->equals(*bigint_file_field.type)); +} + +// Scenario: an explicit user/table cast is preserved while the underlying slot is localized correctly. +TEST_F(ColumnMapperCastTest, ColumnMapperKeepsExplicitSlotCastInSlotLiteralPredicate) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto explicit_cast = Cast::create_shared(std::make_shared()); + explicit_cast->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + auto predicate = std::make_shared(TExprOpcode::GT); + predicate->add_child(explicit_cast); + predicate->add_child( + VLiteral::create_shared(table_column.type, Field::create_field(15))); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) + .ok()); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto& localized_expr = file_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 2); + const auto& localized_cast = localized_expr->children()[0]; + ASSERT_NE(dynamic_cast(localized_cast.get()), nullptr); + EXPECT_TRUE(localized_cast->data_type()->equals(DataTypeString())); + ASSERT_EQ(localized_cast->get_num_children(), 1); + ASSERT_NE(dynamic_cast(localized_cast->children()[0].get()), nullptr); + const auto* localized_slot = + assert_cast(localized_cast->children()[0]->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type)); +} + +// Scenario: repeated scan request creation stays idempotent and does not wrap Cast(Cast(slot)). +TEST_F(ColumnMapperCastTest, ColumnMapperDoesNotNestCastFilterAcrossScanRequests) { + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i32(), 0); + std::vector file_schema {file_field}; + + auto status = mapper.create_mapping(projected_columns, {}, file_schema); + ASSERT_TRUE(status.ok()) << status; + + auto predicate = std::make_shared(15); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest first_request; + ASSERT_TRUE( + mapper.create_scan_request({table_filter}, projected_columns, &first_request, &state) + .ok()); + FileScanRequest second_request; + ASSERT_TRUE( + mapper.create_scan_request({table_filter}, projected_columns, &second_request, &state) + .ok()); + + ASSERT_EQ(second_request.conjuncts.size(), 1); + const auto& localized_expr = second_request.conjuncts[0]->root(); + ASSERT_EQ(localized_expr->get_num_children(), 1); + const auto& localized_child = localized_expr->children()[0]; + ASSERT_NE(dynamic_cast(localized_child.get()), nullptr); + ASSERT_EQ(localized_child->get_num_children(), 1); + const auto* localized_slot = assert_cast(localized_child->children()[0].get()); + EXPECT_EQ(localized_slot->column_id(), 0); +} + +// Scenario: a filter cloned from a previous cast rewrite is adjusted to the next split's matching file type. +TEST_F(ColumnMapperCastTest, ColumnMapperRewritesPreviousCastFilterToMatchingSplitType) { + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto predicate = std::make_shared(15); + predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + auto int_file_field = name_col("value", i32(), 0); + + TableColumnMapper int_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(int_mapper.create_mapping(projected_columns, {}, {int_file_field}).ok()); + FileScanRequest int_request; + ASSERT_TRUE( + int_mapper.create_scan_request({table_filter}, projected_columns, &int_request, &state) + .ok()); + + const auto& int_localized_expr = int_request.conjuncts[0]->root(); + ASSERT_EQ(int_localized_expr->get_num_children(), 1); + ASSERT_NE(dynamic_cast(int_localized_expr->children()[0].get()), nullptr); + + auto bigint_file_field = name_col("value", i64(), 0); + + TableColumnMapper bigint_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(bigint_mapper.create_mapping(projected_columns, {}, {bigint_file_field}).ok()); + FileScanRequest bigint_request; + ASSERT_TRUE( + bigint_mapper + .create_scan_request({table_filter}, projected_columns, &bigint_request, &state) + .ok()); + + const auto& bigint_localized_expr = bigint_request.conjuncts[0]->root(); + ASSERT_EQ(bigint_localized_expr->get_num_children(), 1); + const auto& bigint_localized_child = bigint_localized_expr->children()[0]; + const auto* localized_slot = assert_cast(bigint_localized_child.get()); + EXPECT_EQ(localized_slot->column_id(), 0); + EXPECT_TRUE(localized_slot->data_type()->equals(*bigint_file_field.type)); + + Block block; + block.insert(ColumnHelper::create_column_with_name({11, 22})); + auto* conjunct = bigint_request.conjuncts[0].get(); + auto status = conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + IColumn::Filter filter(block.rows(), 1); + bool can_filter_all = false; + status = conjunct->execute_filter(&block, filter.data(), block.rows(), false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(can_filter_all); + ASSERT_EQ(filter.size(), 2); + EXPECT_EQ(filter[0], 0); + EXPECT_EQ(filter[1], 1); + conjunct->close(); +} + +// Scenario: localized slot keeps table slot id while column id tracks the file block position. +TEST_F(ColumnMapperCastTest, ColumnMapperKeepsTableSlotIdWhenFileBlockPositionChanges) { + auto table_column = name_col("value", i64()); + std::vector projected_columns {table_column}; + + auto file_field = name_col("value", i64(), 10); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(projected_columns, {}, {file_field}).ok()); + + auto predicate = std::make_shared(15); + predicate->add_child(VSlotRef::create_shared(7, 0, -1, table_column.type, "value")); + TableFilter table_filter; + table_filter.conjunct = VExprContext::create_shared(predicate); + table_filter.global_indices = {GlobalIndex(0)}; + + FileScanRequest first_request; + ASSERT_TRUE(mapper.localize_filters({table_filter}, &first_request, &state).ok()); + ASSERT_EQ(first_request.conjuncts.size(), 1); + const auto* first_slot = + assert_cast(first_request.conjuncts[0]->root()->children()[0].get()); + EXPECT_EQ(first_slot->slot_id(), 7); + EXPECT_EQ(first_slot->column_id(), 0); + + FileScanRequest second_request; + second_request.local_positions.emplace(LocalColumnId(9), LocalIndex(0)); + second_request.local_positions.emplace(LocalColumnId(10), LocalIndex(1)); + second_request.non_predicate_columns.push_back(LocalColumnIndex::top_level(LocalColumnId(9))); + ASSERT_TRUE(mapper.localize_filters({table_filter}, &second_request, &state).ok()); + ASSERT_EQ(second_request.conjuncts.size(), 1); + const auto* second_slot = + assert_cast(second_request.conjuncts[0]->root()->children()[0].get()); + EXPECT_EQ(second_slot->slot_id(), 7); + EXPECT_EQ(second_slot->column_id(), 1); + + Block block; + block.insert(ColumnHelper::create_column_with_name({100, 100})); + block.insert(ColumnHelper::create_column_with_name({11, 22})); + auto* conjunct = second_request.conjuncts[0].get(); + auto status = conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + IColumn::Filter filter(block.rows(), 1); + bool can_filter_all = false; + status = conjunct->execute_filter(&block, filter.data(), block.rows(), false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(can_filter_all); + ASSERT_EQ(filter.size(), 2); + EXPECT_EQ(filter[0], 0); + EXPECT_EQ(filter[1], 1); + conjunct->close(); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp b/be/test/format_v2/delimited_text/csv_reader_test.cpp new file mode 100644 index 00000000000000..0ce99630c7af8b --- /dev/null +++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp @@ -0,0 +1,1367 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/delimited_text/csv_reader.h" + +#include + +#include +#include +#include +#include + +#include "common/config.h" +#include "common/consts.h" +#include "common/object_pool.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/column_mapper.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "testutil/desc_tbl_builder.h" +#include "testutil/mock/mock_runtime_state.h" +#include "util/debug_points.h" +#include "util/defer_op.h" + +namespace doris::format::csv { +namespace { + +TFileScanRangeParams csv_scan_params() { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_CSV_PLAIN); + params.__set_file_type(TFileType::FILE_LOCAL); + TFileAttributes attributes; + TFileTextScanRangeParams text_params; + text_params.__set_column_separator(","); + text_params.__set_line_delimiter("\n"); + attributes.__set_text_params(std::move(text_params)); + attributes.__set_header_type(BeConsts::CSV_WITH_NAMES); + params.__set_file_attributes(std::move(attributes)); + params.__set_column_idxs({0, 1, 2}); + return params; +} + +std::unique_ptr file_description(const std::string& path, + int64_t range_start_offset = 0, + int64_t range_size = -1) { + auto desc = std::make_unique(); + desc->path = path; + desc->range_start_offset = range_start_offset; + desc->range_size = range_size; + desc->file_size = static_cast(std::filesystem::file_size(path)); + return desc; +} + +std::unique_ptr unknown_size_file_description(const std::string& path) { + auto desc = std::make_unique(); + desc->path = path; + desc->range_start_offset = 0; + desc->range_size = -1; + desc->file_size = -1; + return desc; +} + +std::vector build_slots(ObjectPool* pool) { + DescriptorTblBuilder builder(pool); + builder.declare_tuple() + << TupleDescBuilder::SlotType {make_nullable(std::make_shared()), "id"} + << TupleDescBuilder::SlotType {make_nullable(std::make_shared()), + "name"} + << TupleDescBuilder::SlotType {make_nullable(std::make_shared()), + "score"}; + auto* desc_tbl = builder.build(); + return desc_tbl->get_tuple_descriptor(0)->slots(); +} + +SlotDescriptor* make_test_slot(ObjectPool* pool, int slot_id, int slot_idx, DataTypePtr type, + const std::string& name) { + TSlotDescriptor slot_desc; + slot_desc.__set_id(slot_id); + slot_desc.__set_parent(0); + slot_desc.__set_slotType(type->to_thrift()); + slot_desc.__set_columnPos(slot_idx); + slot_desc.__set_byteOffset(0); + slot_desc.__set_nullIndicatorByte(slot_idx / 8); + slot_desc.__set_nullIndicatorBit(slot_idx % 8); + slot_desc.__set_slotIdx(slot_idx); + slot_desc.__set_isMaterialized(true); + slot_desc.__set_colName(name); + return pool->add(new SlotDescriptor(slot_desc)); +} + +std::vector build_struct_slots(ObjectPool* pool) { + const auto nullable_int = make_nullable(std::make_shared()); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {nullable_int, nullable_int}, Strings {"a", "b"})); + return {make_test_slot(pool, 0, 0, make_nullable(std::make_shared()), "id"), + make_test_slot(pool, 1, 1, struct_type, "s"), + make_test_slot(pool, 2, 2, make_nullable(std::make_shared()), "score")}; +} + +std::vector build_nested_complex_slots(ObjectPool* pool) { + const auto nullable_int = make_nullable(std::make_shared()); + const auto nullable_string = make_nullable(std::make_shared()); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {nullable_int, nullable_string}, Strings {"a", "b"})); + const auto array_type = make_nullable(std::make_shared(struct_type)); + const auto map_type = + make_nullable(std::make_shared(nullable_string, struct_type)); + return {make_test_slot(pool, 0, 0, make_nullable(std::make_shared()), "id"), + make_test_slot(pool, 1, 1, array_type, "xs"), + make_test_slot(pool, 2, 2, map_type, "kv")}; +} + +std::vector build_char_varchar_slots(ObjectPool* pool) { + const auto nullable_char3 = + make_nullable(std::make_shared(3, PrimitiveType::TYPE_CHAR)); + const auto nullable_varchar4 = + make_nullable(std::make_shared(4, PrimitiveType::TYPE_VARCHAR)); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {nullable_char3, nullable_varchar4}, Strings {"city", "country"})); + return {make_test_slot(pool, 0, 0, make_nullable(std::make_shared()), "id"), + make_test_slot(pool, 1, 1, nullable_char3, "city"), + make_test_slot(pool, 2, 2, struct_type, "region")}; +} + +std::unique_ptr create_reader( + const std::string& path, TFileScanRangeParams* params, + const std::vector& slots, MockRuntimeState* state, RuntimeProfile* profile, + int64_t range_start_offset = 0, int64_t range_size = -1, + TFileCompressType::type range_compress_type = TFileCompressType::UNKNOWN, + std::shared_ptr io_ctx = nullptr) { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto desc = file_description(path, range_start_offset, range_size); + auto reader = std::make_unique(system_properties, desc, std::move(io_ctx), profile, + params, slots, range_compress_type); + EXPECT_TRUE(reader->init(state).ok()); + return reader; +} + +std::unique_ptr create_unknown_size_reader(const std::string& path, + TFileScanRangeParams* params, + const std::vector& slots, + MockRuntimeState* state, + RuntimeProfile* profile) { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto desc = unknown_size_file_description(path); + auto reader = + std::make_unique(system_properties, desc, nullptr, profile, params, slots); + EXPECT_TRUE(reader->init(state).ok()); + return reader; +} + +Block make_block(const std::vector& schema, + const std::vector& local_ids) { + Block block; + for (const auto local_id : local_ids) { + const auto it = std::find_if(schema.begin(), schema.end(), [&](const auto& column) { + return column.local_id == local_id; + }); + EXPECT_TRUE(it != schema.end()); + block.insert({it->type->create_column(), it->type, it->name}); + } + return block; +} + +std::string nullable_string_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data_at(row).to_string(); +} + +bool is_null_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + return nullable.is_null_at(row); +} + +int32_t nullable_int_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data()[row]; +} + +int32_t nullable_struct_int_child_at(const IColumn& column, size_t child_index, size_t row) { + const auto& nullable = assert_cast(column); + const auto& struct_column = assert_cast(nullable.get_nested_column()); + const auto& child_nullable = + assert_cast(struct_column.get_column(child_index)); + const auto& nested = assert_cast(child_nullable.get_nested_column()); + return nested.get_data()[row]; +} + +int64_t counter_value(RuntimeProfile* profile, const std::string& name) { + auto* counter = profile->get_counter(name); + EXPECT_NE(counter, nullptr) << name; + return counter == nullptr ? 0 : counter->value(); +} + +class NullableIntGreaterThanExpr final : public VExpr { +public: + NullableIntGreaterThanExpr(size_t block_position, int32_t value) + : VExpr(std::make_shared(), false), + _block_position(block_position), + _value(value) {} + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_block_position).column); + const auto& data = assert_cast(nullable.get_nested_column()); + + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto source_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable.is_null_at(source_row) && data.get_element(source_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_block_position, _value); + return Status::OK(); + } + +private: + size_t _block_position; + int32_t _value; + const std::string _name = "NullableIntGreaterThanExpr"; +}; + +class StructIntChildGreaterThanExpr final : public VExpr { +public: + StructIntChildGreaterThanExpr(size_t block_position, size_t child_index, int32_t value) + : VExpr(std::make_shared(), false), + _block_position(block_position), + _child_index(child_index), + _value(value) {} + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_block_position).column); + const auto& struct_column = assert_cast(nullable.get_nested_column()); + const auto& child_nullable = + assert_cast(struct_column.get_column(_child_index)); + const auto& child_data = + assert_cast(child_nullable.get_nested_column()); + + auto result = ColumnUInt8::create(); + auto& data = result->get_data(); + data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto source_row = selector == nullptr ? row : (*selector)[row]; + data[row] = !nullable.is_null_at(source_row) && + !child_nullable.is_null_at(source_row) && + child_data.get_element(source_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_block_position, + _child_index, _value); + return Status::OK(); + } + +private: + size_t _block_position; + size_t _child_index; + int32_t _value; + const std::string _name = "StructIntChildGreaterThanExpr"; +}; + +VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { + auto context = VExprContext::create_shared(expr); + auto status = context->prepare(state, RowDescriptor()); + EXPECT_TRUE(status.ok()) << status; + status = context->open(state); + EXPECT_TRUE(status.ok()) << status; + return context; +} + +class CsvV2ReaderTest : public testing::Test { +public: + void SetUp() override { + _test_dir = std::filesystem::temp_directory_path() / "doris_format_v2_csv_reader_test"; + std::filesystem::remove_all(_test_dir); + std::filesystem::create_directories(_test_dir); + _file_path = (_test_dir / "reader.csv").string(); + std::ofstream output(_file_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,alice,10\n"; + output << "2,bob,20\n"; + output.close(); + _slots = build_slots(&_pool); + _params = csv_scan_params(); + } + + void TearDown() override { std::filesystem::remove_all(_test_dir); } + +protected: + ObjectPool _pool; + MockRuntimeState _state; + RuntimeProfile _profile {"csv_v2_reader_test"}; + std::filesystem::path _test_dir; + std::string _file_path; + std::vector _slots; + TFileScanRangeParams _params; +}; + +// Scenario: CSV v2 exposes FE-provided file slots as nullable file-local schema using column_idxs +// as CSV field ordinals. +TEST_F(CsvV2ReaderTest, SchemaUsesSlotTypesAndColumnIdxs) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + EXPECT_EQ(schema[0].name, "id"); + EXPECT_EQ(schema[0].local_id, 0); + EXPECT_TRUE(schema[0].type->is_nullable()); + EXPECT_EQ(schema[1].name, "name"); + EXPECT_EQ(schema[1].local_id, 1); + EXPECT_TRUE(schema[1].type->is_nullable()); +} + +// Scenario: FE slot types for CSV are table target types. CHAR/VARCHAR length is not stored in the +// CSV file, so the file schema must expose bounded strings as unbounded STRING. Otherwise +// TableReader believes the file value already satisfies the table length and skips truncation. +TEST_F(CsvV2ReaderTest, SchemaTreatsCharVarcharSlotsAsUnboundedFileStrings) { + auto slots = build_char_varchar_slots(&_pool); + auto reader = create_reader(_file_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + + const auto city_type = remove_nullable(schema[1].type); + EXPECT_EQ(city_type->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(assert_cast(city_type.get())->len(), -1); + + const auto region_type = remove_nullable(schema[2].type); + ASSERT_EQ(region_type->get_primitive_type(), TYPE_STRUCT); + const auto* region_struct = assert_cast(region_type.get()); + ASSERT_EQ(region_struct->get_elements().size(), 2); + EXPECT_EQ(remove_nullable(region_struct->get_element(0))->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(remove_nullable(region_struct->get_element(1))->get_primitive_type(), TYPE_STRING); + ASSERT_EQ(schema[2].children.size(), 2); + EXPECT_EQ(remove_nullable(schema[2].children[0].type)->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(remove_nullable(schema[2].children[1].type)->get_primitive_type(), TYPE_STRING); +} + +// Scenario: CSV is row-oriented and cannot lazy-read predicate columns separately. The reader +// declares that capability by choosing MaterializedColumnMapper itself. +TEST_F(CsvV2ReaderTest, CreatesMaterializedColumnMapper) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto mapper = reader->create_column_mapper({.mode = TableColumnMappingMode::BY_NAME}); + + ASSERT_NE(dynamic_cast(mapper.get()), nullptr); +} + +// Scenario: CSV v2 exposes delimited-text profile counters for read, parse, deserialize, and +// file-local conjunct filtering, so scanner profiles can explain where row-reader time is spent. +TEST_F(CsvV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { + const auto profile_path = (_test_dir / "profile.csv").string(); + std::ofstream output(profile_path, std::ios::binary); + output << "id,name,score\n"; + output << "\n"; + output << "1,alice,10\n"; + output << "2,bob,20\n"; + output.close(); + + _state._query_options.__set_read_csv_empty_line_as_null(true); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + auto reader = create_reader(profile_path, &_params, _slots, &_state, &_profile, 0, -1, + TFileCompressType::UNKNOWN, io_ctx); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(2), LocalIndex(1)); + request->conjuncts = { + prepared_conjunct(&_state, std::make_shared(1, 15))}; + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2); + + EXPECT_NE(_profile.get_counter("OpenFileTime"), nullptr); + EXPECT_NE(_profile.get_counter("CreateLineReaderTime"), nullptr); + EXPECT_NE(_profile.get_counter("ReadLineTime"), nullptr); + EXPECT_NE(_profile.get_counter("SplitLineTime"), nullptr); + EXPECT_NE(_profile.get_counter("DeserializeTime"), nullptr); + EXPECT_NE(_profile.get_counter("ConjunctFilterTime"), nullptr); + EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); + EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); + EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); + EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); + EXPECT_EQ(file_reader_stats.read_rows, 3); + EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); + EXPECT_EQ(counter_value(&_profile, "RowsReturned"), 1); + EXPECT_EQ(counter_value(&_profile, "EmptyLinesRead"), 1); + EXPECT_EQ(counter_value(&_profile, "SkippedLines"), 1); + EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6); +} + +// Scenario: CSV has no embedded nested schema, but TableColumnMapper still needs semantic children +// for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children from the slot type +// while keeping the top-level local id as the CSV field ordinal from column_idxs. +TEST_F(CsvV2ReaderTest, SchemaSynthesizesComplexChildrenForColumnMapper) { + _params.__set_column_idxs({4, 7, 9}); + auto slots = build_nested_complex_slots(&_pool); + auto reader = create_reader(_file_path, &_params, slots, &_state, &_profile); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + + EXPECT_EQ(schema[1].name, "xs"); + EXPECT_EQ(schema[1].local_id, 7); + ASSERT_EQ(schema[1].children.size(), 1); + EXPECT_EQ(schema[1].children[0].name, "element"); + EXPECT_EQ(schema[1].children[0].local_id, 0); + ASSERT_EQ(schema[1].children[0].children.size(), 2); + EXPECT_EQ(schema[1].children[0].children[0].name, "a"); + EXPECT_EQ(schema[1].children[0].children[0].local_id, 0); + EXPECT_EQ(schema[1].children[0].children[1].name, "b"); + EXPECT_EQ(schema[1].children[0].children[1].local_id, 1); + + EXPECT_EQ(schema[2].name, "kv"); + EXPECT_EQ(schema[2].local_id, 9); + ASSERT_EQ(schema[2].children.size(), 2); + EXPECT_EQ(schema[2].children[0].name, "key"); + EXPECT_EQ(schema[2].children[0].local_id, 0); + EXPECT_EQ(schema[2].children[1].name, "value"); + EXPECT_EQ(schema[2].children[1].local_id, 1); + ASSERT_EQ(schema[2].children[1].children.size(), 2); + EXPECT_EQ(schema[2].children[1].children[0].name, "a"); + EXPECT_EQ(schema[2].children[1].children[1].name, "b"); +} + +// Scenario: CSV v2 honors FileScanRequest local positions, so TableReader can request a subset of +// CSV fields in an order different from the physical CSV field order. +TEST_F(CsvV2ReaderTest, ReadsRequestedColumnsInFileLocalBlockOrder) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1)), + LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(0), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1, 0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "alice"); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 1), "bob"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 0), 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 1), 2); +} + +// Scenario: CSV v2 defaults to the same strict UTF-8 validation as the old query reader. Invalid +// bytes should fail fast unless the scan params explicitly disable text UTF-8 validation. +TEST_F(CsvV2ReaderTest, InvalidUtf8FailsWhenValidationEnabled) { + const auto invalid_path = (_test_dir / "invalid_utf8.csv").string(); + std::ofstream output(invalid_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,"; + output.write("\xff", 1); + output << ",10\n"; + output.close(); + + auto reader = create_reader(invalid_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.to_string().find("Only support csv data in utf8 codec") != std::string::npos) + << status; +} + +// Scenario: external CSV scans can opt out of UTF-8 validation through +// `enable_text_validate_utf8=false`. In that mode the reader preserves the original bytes instead +// of rejecting the row. +TEST_F(CsvV2ReaderTest, DisableTextValidateUtf8ReadsRawBytes) { + const auto invalid_path = (_test_dir / "invalid_utf8_disabled.csv").string(); + std::ofstream output(invalid_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,"; + output.write("\xff", 1); + output << ",10\n"; + output.close(); + + _params.file_attributes.__set_enable_text_validate_utf8(false); + auto reader = create_reader(invalid_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), std::string("\xff", 1)); +} + +// Scenario: file TVF can keep the logical CSV format as FORMAT_CSV_PLAIN and put the actual gzip +// compression on the scan range. CSV v2 must honor that range-level compression before validating +// UTF-8; otherwise the gzip bytes are misread as CSV text. +TEST_F(CsvV2ReaderTest, RangeCompressTypeGzipDecompressesPlainCsvFormat) { + const auto gz_path = (_test_dir / "reader.csv.gz").string(); + static constexpr unsigned char gzipped_csv[] = { + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcb, 0x4c, + 0xd1, 0xc9, 0x4b, 0xcc, 0x4d, 0xd5, 0x29, 0x4e, 0xce, 0x2f, 0x4a, 0xe5, + 0x32, 0xd4, 0x49, 0xcc, 0xc9, 0x4c, 0x4e, 0xd5, 0x31, 0x34, 0xe0, 0x02, + 0x00, 0x0b, 0xed, 0x5c, 0xa2, 0x19, 0x00, 0x00, 0x00}; + std::ofstream output(gz_path, std::ios::binary); + output.write(reinterpret_cast(gzipped_csv), sizeof(gzipped_csv)); + output.close(); + + _params.__set_format_type(TFileFormatType::FORMAT_CSV_PLAIN); + _params.__isset.compress_type = false; + auto reader = create_reader(gz_path, &_params, _slots, &_state, &_profile, 0, -1, + TFileCompressType::GZ); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "alice"); +} + +// Scenario: FE column_idxs define the CSV field ordinal for each physical file slot. The mapping +// can be non-identity when FE reorders projected file slots, so the reader must use the local id +// from FileScanRequest instead of the slot vector position. +TEST_F(CsvV2ReaderTest, ColumnIdxsMapSlotsToCsvOrdinals) { + const auto remap_path = (_test_dir / "remapped.csv").string(); + std::ofstream output(remap_path, std::ios::binary); + output << "name,score,id\n"; + output << "alice,10,1\n"; + output.close(); + + _params.__set_column_idxs({2, 0, 1}); + auto reader = create_reader(remap_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + EXPECT_EQ(schema[0].name, "id"); + EXPECT_EQ(schema[0].local_id, 2); + EXPECT_EQ(schema[1].name, "name"); + EXPECT_EQ(schema[1].local_id, 0); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(2)), + LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(2), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(0), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {2, 0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "alice"); +} + +// Scenario: CSV stores one complex column as one text field, so v2 must read the whole struct +// field before evaluating a file-local predicate on one child. This covers `SELECT s.a WHERE +// s.b > 10` style scans after CsvReader's MaterializedColumnMapper has requested the full +// top-level `s`. +TEST_F(CsvV2ReaderTest, FullStructColumnSupportsChildConjunctFiltering) { + const auto complex_path = (_test_dir / "complex.csv").string(); + std::ofstream output(complex_path, std::ios::binary); + output << "id|s|score\n"; + output << "1|{\"a\": 11, \"b\": 5}|10\n"; + output << "2|{\"a\": 22, \"b\": 20}|20\n"; + output.close(); + + _params.file_attributes.text_params.__set_column_separator("|"); + _params.__set_column_idxs({0, 1, 2}); + auto slots = build_struct_slots(&_pool); + auto reader = create_reader(complex_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + request->conjuncts = {prepared_conjunct( + &_state, std::make_shared( + /*block_position=*/0, /*child_index=*/1, /*value=*/10))}; + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_struct_int_child_at(*block.get_by_position(0).column, 0, 0), 22); + EXPECT_EQ(nullable_struct_int_child_at(*block.get_by_position(0).column, 1, 0), 20); +} + +// Scenario: a table-level scan can need only partition/default columns, leaving the CSV +// FileScanRequest with no file-local columns. The reader must still report the number of rows read. +TEST_F(CsvV2ReaderTest, EmptyFileLocalProjectionStillReportsRows) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + Block block; + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 2); + EXPECT_FALSE(eof); +} + +// Scenario: stream-load/http_stream inputs do not have a known split size or file size. A first +// split must still read until EOF instead of rejecting the request before opening the stream. +TEST_F(CsvV2ReaderTest, UnknownFirstSplitSizeReadsUntilEof) { + auto reader = create_unknown_size_reader(_file_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 1), "bob"); +} + +// Scenario: stream load/http_stream CSV input is not backed by a filesystem. If TableReader fails +// to preserve the stream load id, the v2 reader should report that directly instead of calling the +// generic FileFactory path and returning "unsupported file reader type: 2". +TEST_F(CsvV2ReaderTest, StreamInputRequiresLoadIdBeforeOpeningPipe) { + _params.__set_file_type(TFileType::FILE_STREAM); + auto reader = create_unknown_size_reader(_file_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + const auto status = reader->open(request); + + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("stream reader requires load id"), std::string::npos) + << status; +} + +// Scenario: CSV has no footer row count, so v2 COUNT pushdown scans the split and returns the +// counted row count through FileAggregateResult. +TEST_F(CsvV2ReaderTest, CountAggregateScansRows) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + FileAggregateResult aggregate_result; + ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result).ok()); + EXPECT_EQ(aggregate_result.count, 2); +} + +// Scenario: CSV v2 parses enclosed fields itself instead of delegating to the old CsvReader. A +// separator inside an enclosed string must stay inside the same CSV field. +TEST_F(CsvV2ReaderTest, EnclosedFieldKeepsSeparatorInsideStringValue) { + const auto quoted_path = (_test_dir / "quoted.csv").string(); + std::ofstream output(quoted_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,\"alice,team\",10\n"; + output.close(); + + _params.file_attributes.text_params.__set_enclose('"'); + _params.file_attributes.text_params.__set_escape('\\'); + auto reader = create_reader(quoted_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "alice,team"); +} + +// Bare quotes and escapes outside enclosed fields stay in NORMAL state in +// EncloseCsvLineReaderCtx. Field slicing must reuse those exact separator positions. +TEST_F(CsvV2ReaderTest, EncloseFieldSplittingMatchesLineReaderStateMachine) { + const auto quoted_path = (_test_dir / "enclose_state.csv").string(); + std::ofstream output(quoted_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,ab\"cd,20\n"; + output << "2,C:\\dir\\,30,40\n"; + output.close(); + + _params.file_attributes.text_params.__set_enclose('"'); + _params.file_attributes.text_params.__set_escape('\\'); + auto reader = create_reader(quoted_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1)), + LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(2), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1, 2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "ab\"cd"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 0), 20); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 1), 30); +} + +// OpenCSV permits escape and enclose to use the same character. The shared line-reader state +// machine handles doubled quotes without treating every quote as a generic escape. +TEST_F(CsvV2ReaderTest, MatchingEscapeAndEncloseStillSplitQuotedFields) { + const auto quoted_path = (_test_dir / "matching_escape_enclose.csv").string(); + std::ofstream output(quoted_path, std::ios::binary); + output << "id,name,score\n"; + output << "\"1\",\"alice\",\"10\"\n"; + output.close(); + + _params.file_attributes.text_params.__set_enclose('"'); + _params.file_attributes.text_params.__set_escape('"'); + auto reader = create_reader(quoted_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1)), + LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + request->local_positions.emplace(LocalColumnId(2), LocalIndex(2)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 1, 2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "alice"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(2).column, 0), 10); +} + +// OpenCSV custom quote characters only remove that configured enclosure. Literal double quotes in +// a field must remain when the configured enclosure is a single quote. +TEST_F(CsvV2ReaderTest, CustomEnclosePreservesLiteralDoubleQuotes) { + const auto quoted_path = (_test_dir / "custom_enclose.csv").string(); + std::ofstream output(quoted_path, std::ios::binary); + output << R"("Project Manager")" << '\n'; + output.close(); + + auto value_slot = make_test_slot(&_pool, 10, 0, + make_nullable(std::make_shared()), "value"); + std::vector slots {value_slot}; + _params.__set_column_idxs({0}); + _params.file_attributes.__isset.header_type = false; + _params.file_attributes.text_params.__set_column_separator("\t"); + _params.file_attributes.text_params.__set_enclose('\''); + _params.file_attributes.text_params.__set_escape('|'); + auto reader = create_reader(quoted_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), R"("Project Manager")"); +} + +// A column separator that overlaps the line delimiter must not be recorded because CsvReader +// splits a Slice that excludes the line delimiter bytes. +TEST_F(CsvV2ReaderTest, ColumnSeparatorOverlappingLineDelimiterStaysOutsideReturnedLine) { + const auto overlap_path = (_test_dir / "overlapping_delimiters.csv").string(); + std::ofstream output(overlap_path, std::ios::binary); + output << "value|\n"; + output.close(); + + auto value_slot = make_test_slot(&_pool, 10, 0, + make_nullable(std::make_shared()), "value"); + std::vector slots {value_slot}; + _params.__set_column_idxs({0}); + _params.file_attributes.__isset.header_type = false; + _params.file_attributes.text_params.__set_column_separator("|\n"); + _params.file_attributes.text_params.__set_enclose('"'); + _params.file_attributes.text_params.__set_escape('\\'); + auto reader = create_reader(overlap_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "value|"); +} + +// When a logical row is refilled, partial-separator backtracking must not cross the end of a +// separator that was already accepted from the previous buffer contents. +TEST_F(CsvV2ReaderTest, MultiCharacterSeparatorAcrossOutputBufferRefill) { + const auto refill_path = (_test_dir / "separator_refill.csv").string(); + std::ofstream output(refill_path, std::ios::binary); + output << std::string(28, 'a') << "|||||b\n"; + output.close(); + + auto first_slot = make_test_slot(&_pool, 10, 0, + make_nullable(std::make_shared()), "first"); + auto second_slot = make_test_slot(&_pool, 11, 1, + make_nullable(std::make_shared()), "second"); + std::vector slots {first_slot, second_slot}; + _params.__set_column_idxs({0, 1}); + _params.file_attributes.__isset.header_type = false; + _params.file_attributes.text_params.__set_column_separator("|||"); + _params.file_attributes.text_params.__set_enclose('"'); + _params.file_attributes.text_params.__set_escape('\\'); + + const bool enable_debug_points = config::enable_debug_points; + config::enable_debug_points = true; + DebugPoints::instance()->add_with_params("NewPlainTextLineReader.shrink_output_buf", + {{"output_buf_size", "32"}}); + Defer restore_debug_point([enable_debug_points]() { + DebugPoints::instance()->remove("NewPlainTextLineReader.shrink_output_buf"); + config::enable_debug_points = enable_debug_points; + }); + + auto reader = create_reader(refill_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), std::string(28, 'a')); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "||b"); +} + +// Scenario: when the CSV row has fewer fields than the FE-provided file slot list, v2 fills the +// missing requested field with NULL instead of failing or shifting later columns. +TEST_F(CsvV2ReaderTest, MissingRequestedFieldUsesNullFormat) { + const auto missing_path = (_test_dir / "missing.csv").string(); + std::ofstream output(missing_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,alice\n"; + output.close(); + + auto reader = create_reader(missing_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(2), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 0)); +} + +// Scenario: the first line may contain UTF-8 BOM and CSV_WITH_NAMES_AND_TYPES has two header +// records. Both must be skipped before materializing the first data row. +TEST_F(CsvV2ReaderTest, HeaderNamesAndTypesSkipsTwoLinesAndBom) { + const auto header_path = (_test_dir / "header_names_types.csv").string(); + std::ofstream output(header_path, std::ios::binary); + output.write("\xEF\xBB\xBF", 3); + output << "id,name,score\n"; + output << "INT,STRING,INT\n"; + output << "7,carol,70\n"; + output.close(); + + _params.file_attributes.__set_header_type(BeConsts::CSV_WITH_NAMES_AND_TYPES); + auto reader = create_reader(header_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 7); +} + +// Scenario: when the first returned data line starts with UTF-8 BOM, CSV v2 strips the BOM before +// passing the cell to the serde. This matters for headerless files whose first column is numeric. +TEST_F(CsvV2ReaderTest, BomIsRemovedFromFirstDataLineWithoutHeader) { + const auto bom_path = (_test_dir / "bom_data.csv").string(); + std::ofstream output(bom_path, std::ios::binary); + output.write("\xEF\xBB\xBF", 3); + output << "5,bom,50\n"; + output.close(); + + _params.file_attributes.__isset.header_type = false; + _params.file_attributes.text_params.__set_enclose('"'); + _params.file_attributes.text_params.__set_escape('\\'); + auto reader = create_reader(bom_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 5); +} + +// The enclosed line reader sees bytes before DelimitedTextReader removes the BOM. It must skip the +// BOM structurally so the quote immediately after it still starts an enclosed first field and the +// comma inside that field is not recorded as a column separator. +TEST_F(CsvV2ReaderTest, BomBeforeQuotedFirstFieldWithComma) { + const auto bom_path = (_test_dir / "bom_quoted_comma.csv").string(); + std::ofstream output(bom_path, std::ios::binary); + output.write("\xEF\xBB\xBF", 3); + output << "\"alice,team\",20\n"; + output.close(); + + auto string_slot = make_test_slot(&_pool, 10, 0, + make_nullable(std::make_shared()), "name"); + auto score_slot = make_test_slot(&_pool, 11, 1, + make_nullable(std::make_shared()), "score"); + std::vector slots {string_slot, score_slot}; + _params.__set_column_idxs({0, 1}); + _params.file_attributes.__isset.header_type = false; + _params.file_attributes.text_params.__set_enclose('"'); + _params.file_attributes.text_params.__set_escape('\\'); + auto reader = create_reader(bom_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "alice,team"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 0), 20); +} + +// The same ordering matters before logical-row detection: a newline inside the first enclosed +// field belongs to the field, even when the opening quote follows a file BOM. +TEST_F(CsvV2ReaderTest, BomBeforeQuotedFirstFieldWithNewline) { + const auto bom_path = (_test_dir / "bom_quoted_newline.csv").string(); + std::ofstream output(bom_path, std::ios::binary); + output.write("\xEF\xBB\xBF", 3); + output << "\"alice\nteam\",20\n"; + output.close(); + + auto string_slot = make_test_slot(&_pool, 10, 0, + make_nullable(std::make_shared()), "name"); + auto score_slot = make_test_slot(&_pool, 11, 1, + make_nullable(std::make_shared()), "score"); + std::vector slots {string_slot, score_slot}; + _params.__set_column_idxs({0, 1}); + _params.file_attributes.__isset.header_type = false; + _params.file_attributes.text_params.__set_enclose('"'); + _params.file_attributes.text_params.__set_escape('\\'); + auto reader = create_reader(bom_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "alice\nteam"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 0), 20); +} + +// Scenario: when FE does not set header_type, CSV v2 must honor skip_lines exactly as the old +// reader does. +TEST_F(CsvV2ReaderTest, SkipLinesUsedWhenHeaderTypeUnset) { + const auto skip_path = (_test_dir / "skip_lines.csv").string(); + std::ofstream output(skip_path, std::ios::binary); + output << "skip me\n"; + output << "skip me too\n"; + output << "3,dan,30\n"; + output.close(); + + _params.file_attributes.__isset.header_type = false; + _params.file_attributes.__set_skip_lines(2); + auto reader = create_reader(skip_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 3); +} + +// Scenario: empty physical lines are skipped by default, but read_csv_empty_line_as_null turns one +// empty line into one all-null logical row. +TEST_F(CsvV2ReaderTest, EmptyLineAsNullWhenQueryOptionEnabled) { + const auto empty_line_path = (_test_dir / "empty_line.csv").string(); + std::ofstream output(empty_line_path, std::ios::binary); + output << "id,name,score\n"; + output << "\n"; + output << "4,erin,40\n"; + output.close(); + + _state._query_options.__set_read_csv_empty_line_as_null(true); + auto reader = create_reader(empty_line_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 1), 4); +} + +// Scenario: FE-provided CSV text parameters define NULL semantics. Explicit null_format and +// empty_field_as_null should both produce nullable values without throwing serde errors. +TEST_F(CsvV2ReaderTest, NullFormatAndEmptyFieldAsNullProduceNullableValues) { + const auto null_path = (_test_dir / "null_format.csv").string(); + std::ofstream output(null_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,NULL,\n"; + output.close(); + + _params.file_attributes.text_params.__set_null_format("NULL"); + _params.file_attributes.text_params.__set_empty_field_as_null(true); + auto reader = create_reader(null_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1)), + LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(2), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1, 2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_TRUE(is_null_at(*block.get_by_position(1).column, 0)); +} + +// trim_double_quotes preserves quoted null literals as strings while an unquoted marker remains +// NULL, matching the V1 CSV reader's converted-from-string behavior. +TEST_F(CsvV2ReaderTest, QuotedNullFormatIsAStringLiteral) { + const auto null_path = (_test_dir / "quoted_null_format.csv").string(); + std::ofstream output(null_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,\"\\N\",10\n"; + output << "2,\\N,20\n"; + output.close(); + + _params.file_attributes.__set_trim_double_quotes(true); + _params.file_attributes.text_params.__set_null_format("\\N"); + auto reader = create_reader(null_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_FALSE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "\\N"); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 1)); +} + +// Scenario: OpenCSV keeps an empty field as an empty string when empty_field_as_null is false, +// even if FE passes an empty null_format. This differs from Hive text serde, where an empty +// serialization.null.format is a real NULL marker. +TEST_F(CsvV2ReaderTest, EmptyNullFormatKeepsCsvEmptyFieldAsEmptyString) { + const auto null_path = (_test_dir / "empty_null_format.csv").string(); + std::ofstream output(null_path, std::ios::binary); + output << "id,name,score\n"; + output << "1,alice,10\n"; + output << "2,,20\n"; + output << "3,NULL,30\n"; + output.close(); + + _params.file_attributes.text_params.__set_null_format(""); + _params.file_attributes.text_params.__set_empty_field_as_null(false); + auto reader = create_reader(null_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 3); + EXPECT_FALSE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_FALSE(is_null_at(*block.get_by_position(0).column, 1)); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 1), ""); + EXPECT_FALSE(is_null_at(*block.get_by_position(0).column, 2)); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 2), "NULL"); +} + +// Scenario: a non-first split starts inside a record. CSV v2 pre-reads enough delimiter bytes and +// skips the partial first line so the split begins at the next complete row. +TEST_F(CsvV2ReaderTest, NonFirstSplitSkipsPartialFirstRecord) { + const auto split_path = (_test_dir / "split.csv").string(); + std::ofstream output(split_path, std::ios::binary); + output << "1,skip,10\n"; + output << "2,bob,20\n"; + output.close(); + + _params.file_attributes.__isset.header_type = false; + auto reader = create_reader(split_path, &_params, _slots, &_state, &_profile, + /*range_start_offset=*/3); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2); +} + +// Scenario: compressed CSV cannot be split at arbitrary byte offsets because the decompressor needs +// the stream from the beginning. V2 should reject such a split before constructing the line reader. +TEST_F(CsvV2ReaderTest, NonFirstCompressedSplitReturnsError) { + _params.__set_format_type(TFileFormatType::FORMAT_CSV_GZ); + _params.file_attributes.__isset.header_type = false; + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile, + /*range_start_offset=*/1); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + EXPECT_FALSE(reader->open(request).ok()); +} + +// Scenario: FileScanRequest is a TableReader-to-FileReader contract. Unknown CSV ordinals, +// out-of-range block positions, and sparse block-position maps must fail during reader open. +TEST_F(CsvV2ReaderTest, InvalidScanRequestReturnsError) { + { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(99))}; + request->local_positions.emplace(LocalColumnId(99), LocalIndex(0)); + EXPECT_FALSE(reader->open(request).ok()); + } + { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(2)); + EXPECT_FALSE(reader->open(request).ok()); + } + { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + EXPECT_FALSE(reader->open(request).ok()); + } +} + +// Scenario: CSV v2 can count rows by scanning, but it cannot answer min/max or mixed aggregate +// requests from metadata. +TEST_F(CsvV2ReaderTest, UnsupportedAggregateReturnsNotSupported) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + FileAggregateResult aggregate_result; + EXPECT_FALSE(reader->get_aggregate_result(aggregate_request, &aggregate_result).ok()); +} + +} // namespace +} // namespace doris::format::csv diff --git a/be/test/format_v2/delimited_text/text_reader_test.cpp b/be/test/format_v2/delimited_text/text_reader_test.cpp new file mode 100644 index 00000000000000..f5f7309e490ff1 --- /dev/null +++ b/be/test/format_v2/delimited_text/text_reader_test.cpp @@ -0,0 +1,994 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/delimited_text/text_reader.h" + +#include + +#include +#include +#include +#include + +#include "common/consts.h" +#include "common/object_pool.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/column_mapper.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "testutil/desc_tbl_builder.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris::format::text { +namespace { + +TFileScanRangeParams text_scan_params() { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_TEXT); + params.__set_file_type(TFileType::FILE_LOCAL); + TFileAttributes attributes; + TFileTextScanRangeParams text_params; + text_params.__set_column_separator(","); + text_params.__set_line_delimiter("\n"); + text_params.__set_escape('\\'); + attributes.__set_text_params(std::move(text_params)); + params.__set_file_attributes(std::move(attributes)); + params.__set_column_idxs({0, 1, 2}); + return params; +} + +std::unique_ptr file_description(const std::string& path, + int64_t range_start_offset = 0, + int64_t range_size = -1) { + auto desc = std::make_unique(); + desc->path = path; + desc->range_start_offset = range_start_offset; + desc->range_size = range_size; + desc->file_size = static_cast(std::filesystem::file_size(path)); + return desc; +} + +std::vector build_slots(ObjectPool* pool) { + DescriptorTblBuilder builder(pool); + builder.declare_tuple() + << TupleDescBuilder::SlotType {make_nullable(std::make_shared()), "id"} + << TupleDescBuilder::SlotType {make_nullable(std::make_shared()), + "name"} + << TupleDescBuilder::SlotType {make_nullable(std::make_shared()), + "score"}; + auto* desc_tbl = builder.build(); + return desc_tbl->get_tuple_descriptor(0)->slots(); +} + +SlotDescriptor* make_test_slot(ObjectPool* pool, int slot_id, int slot_idx, DataTypePtr type, + const std::string& name) { + TSlotDescriptor slot_desc; + slot_desc.__set_id(slot_id); + slot_desc.__set_parent(0); + slot_desc.__set_slotType(type->to_thrift()); + slot_desc.__set_columnPos(slot_idx); + slot_desc.__set_byteOffset(0); + slot_desc.__set_nullIndicatorByte(slot_idx / 8); + slot_desc.__set_nullIndicatorBit(slot_idx % 8); + slot_desc.__set_slotIdx(slot_idx); + slot_desc.__set_isMaterialized(true); + slot_desc.__set_colName(name); + return pool->add(new SlotDescriptor(slot_desc)); +} + +std::vector build_struct_slots(ObjectPool* pool) { + const auto nullable_int = make_nullable(std::make_shared()); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {nullable_int, nullable_int}, Strings {"a", "b"})); + return {make_test_slot(pool, 0, 0, make_nullable(std::make_shared()), "id"), + make_test_slot(pool, 1, 1, struct_type, "s"), + make_test_slot(pool, 2, 2, make_nullable(std::make_shared()), "score")}; +} + +std::vector build_nested_complex_slots(ObjectPool* pool) { + const auto nullable_int = make_nullable(std::make_shared()); + const auto nullable_string = make_nullable(std::make_shared()); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {nullable_int, nullable_string}, Strings {"a", "b"})); + const auto array_type = make_nullable(std::make_shared(struct_type)); + const auto map_type = + make_nullable(std::make_shared(nullable_string, struct_type)); + return {make_test_slot(pool, 0, 0, make_nullable(std::make_shared()), "id"), + make_test_slot(pool, 1, 1, array_type, "xs"), + make_test_slot(pool, 2, 2, map_type, "kv")}; +} + +std::vector build_char_varchar_slots(ObjectPool* pool) { + const auto nullable_char3 = + make_nullable(std::make_shared(3, PrimitiveType::TYPE_CHAR)); + const auto nullable_varchar4 = + make_nullable(std::make_shared(4, PrimitiveType::TYPE_VARCHAR)); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {nullable_char3, nullable_varchar4}, Strings {"city", "country"})); + return {make_test_slot(pool, 0, 0, make_nullable(std::make_shared()), "id"), + make_test_slot(pool, 1, 1, nullable_char3, "city"), + make_test_slot(pool, 2, 2, struct_type, "region")}; +} + +std::unique_ptr create_reader(const std::string& path, TFileScanRangeParams* params, + const std::vector& slots, + MockRuntimeState* state, RuntimeProfile* profile, + int64_t range_start_offset = 0, int64_t range_size = -1, + std::shared_ptr io_ctx = nullptr) { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto desc = file_description(path, range_start_offset, range_size); + auto reader = std::make_unique(system_properties, desc, std::move(io_ctx), profile, + params, slots); + EXPECT_TRUE(reader->init(state).ok()); + return reader; +} + +Block make_block(const std::vector& schema, + const std::vector& local_ids) { + Block block; + for (const auto local_id : local_ids) { + const auto it = std::find_if(schema.begin(), schema.end(), [&](const auto& column) { + return column.local_id == local_id; + }); + EXPECT_TRUE(it != schema.end()); + block.insert({it->type->create_column(), it->type, it->name}); + } + return block; +} + +std::string nullable_string_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data_at(row).to_string(); +} + +int32_t nullable_int_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data()[row]; +} + +bool is_null_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + return nullable.is_null_at(row); +} + +int32_t nullable_struct_int_child_at(const IColumn& column, size_t child_index, size_t row) { + const auto& nullable = assert_cast(column); + const auto& struct_column = assert_cast(nullable.get_nested_column()); + const auto& child_nullable = + assert_cast(struct_column.get_column(child_index)); + const auto& nested = assert_cast(child_nullable.get_nested_column()); + return nested.get_data()[row]; +} + +int64_t counter_value(RuntimeProfile* profile, const std::string& name) { + auto* counter = profile->get_counter(name); + EXPECT_NE(counter, nullptr) << name; + return counter == nullptr ? 0 : counter->value(); +} + +class NullableIntGreaterThanExpr final : public VExpr { +public: + NullableIntGreaterThanExpr(size_t block_position, int32_t value) + : VExpr(std::make_shared(), false), + _block_position(block_position), + _value(value) {} + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_block_position).column); + const auto& data = assert_cast(nullable.get_nested_column()); + + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto source_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable.is_null_at(source_row) && data.get_element(source_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_block_position, _value); + return Status::OK(); + } + +private: + size_t _block_position; + int32_t _value; + const std::string _name = "NullableIntGreaterThanExpr"; +}; + +class StructIntChildGreaterThanExpr final : public VExpr { +public: + StructIntChildGreaterThanExpr(size_t block_position, size_t child_index, int32_t value) + : VExpr(std::make_shared(), false), + _block_position(block_position), + _child_index(child_index), + _value(value) {} + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_block_position).column); + const auto& struct_column = assert_cast(nullable.get_nested_column()); + const auto& child_nullable = + assert_cast(struct_column.get_column(_child_index)); + const auto& child_data = + assert_cast(child_nullable.get_nested_column()); + + auto result = ColumnUInt8::create(); + auto& data = result->get_data(); + data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto source_row = selector == nullptr ? row : (*selector)[row]; + data[row] = !nullable.is_null_at(source_row) && + !child_nullable.is_null_at(source_row) && + child_data.get_element(source_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_block_position, + _child_index, _value); + return Status::OK(); + } + +private: + size_t _block_position; + size_t _child_index; + int32_t _value; + const std::string _name = "StructIntChildGreaterThanExpr"; +}; + +VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { + auto context = VExprContext::create_shared(expr); + auto status = context->prepare(state, RowDescriptor()); + EXPECT_TRUE(status.ok()) << status; + status = context->open(state); + EXPECT_TRUE(status.ok()) << status; + return context; +} + +class TextV2ReaderTest : public testing::Test { +public: + void SetUp() override { + _test_dir = std::filesystem::temp_directory_path() / "doris_format_v2_text_reader_test"; + std::filesystem::remove_all(_test_dir); + std::filesystem::create_directories(_test_dir); + _file_path = (_test_dir / "reader.text").string(); + std::ofstream output(_file_path, std::ios::binary); + output << "1,alice,10\n"; + output << "2,bob,20\n"; + output.close(); + _slots = build_slots(&_pool); + _params = text_scan_params(); + } + + void TearDown() override { std::filesystem::remove_all(_test_dir); } + +protected: + ObjectPool _pool; + MockRuntimeState _state; + RuntimeProfile _profile {"text_v2_reader_test"}; + std::filesystem::path _test_dir; + std::string _file_path; + std::vector _slots; + TFileScanRangeParams _params; +}; + +// Scenario: Text v2 exposes FE-provided file slots as nullable file-local schema using column_idxs +// as Hive text field ordinals. +TEST_F(TextV2ReaderTest, SchemaUsesSlotTypesAndColumnIdxs) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + EXPECT_EQ(schema[0].name, "id"); + EXPECT_EQ(schema[0].local_id, 0); + EXPECT_TRUE(schema[0].type->is_nullable()); + EXPECT_EQ(schema[1].name, "name"); + EXPECT_EQ(schema[1].local_id, 1); + EXPECT_TRUE(schema[1].type->is_nullable()); +} + +// Scenario: FE slot types for Hive text are table target types. CHAR/VARCHAR length is not stored +// in the text file, so the file schema must expose bounded strings as unbounded STRING. Otherwise +// TableReader believes the file value already satisfies the table length and skips truncation. +TEST_F(TextV2ReaderTest, SchemaTreatsCharVarcharSlotsAsUnboundedFileStrings) { + auto slots = build_char_varchar_slots(&_pool); + auto reader = create_reader(_file_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + + const auto city_type = remove_nullable(schema[1].type); + EXPECT_EQ(city_type->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(assert_cast(city_type.get())->len(), -1); + + const auto region_type = remove_nullable(schema[2].type); + ASSERT_EQ(region_type->get_primitive_type(), TYPE_STRUCT); + const auto* region_struct = assert_cast(region_type.get()); + ASSERT_EQ(region_struct->get_elements().size(), 2); + EXPECT_EQ(remove_nullable(region_struct->get_element(0))->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(remove_nullable(region_struct->get_element(1))->get_primitive_type(), TYPE_STRING); + ASSERT_EQ(schema[2].children.size(), 2); + EXPECT_EQ(remove_nullable(schema[2].children[0].type)->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(remove_nullable(schema[2].children[1].type)->get_primitive_type(), TYPE_STRING); +} + +// Scenario: Hive text is row-oriented and cannot lazy-read predicate columns separately. The +// reader declares that capability by choosing MaterializedColumnMapper itself. +TEST_F(TextV2ReaderTest, CreatesMaterializedColumnMapper) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto mapper = reader->create_column_mapper({.mode = TableColumnMappingMode::BY_NAME}); + + ASSERT_NE(dynamic_cast(mapper.get()), nullptr); +} + +// Scenario: Text v2 exposes delimited-text profile counters for read, parse, deserialize, and +// file-local conjunct filtering, so scanner profiles can explain where row-reader time is spent. +TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { + const auto profile_path = (_test_dir / "profile.text").string(); + std::ofstream output(profile_path, std::ios::binary); + output << "\n"; + output << "1,alice,10\n"; + output << "2,bob,20\n"; + output.close(); + + _state._query_options.__set_read_csv_empty_line_as_null(true); + auto io_ctx = std::make_shared(); + auto reader = create_reader(profile_path, &_params, _slots, &_state, &_profile, 0, -1, io_ctx); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(2), LocalIndex(1)); + request->conjuncts = { + prepared_conjunct(&_state, std::make_shared(1, 15))}; + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2); + + EXPECT_NE(_profile.get_counter("OpenFileTime"), nullptr); + EXPECT_NE(_profile.get_counter("CreateLineReaderTime"), nullptr); + EXPECT_NE(_profile.get_counter("ReadLineTime"), nullptr); + EXPECT_NE(_profile.get_counter("SplitLineTime"), nullptr); + EXPECT_NE(_profile.get_counter("DeserializeTime"), nullptr); + EXPECT_NE(_profile.get_counter("ConjunctFilterTime"), nullptr); + EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); + EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); + EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); + EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); + EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); + EXPECT_EQ(counter_value(&_profile, "RowsReturned"), 1); + EXPECT_EQ(counter_value(&_profile, "EmptyLinesRead"), 1); + EXPECT_EQ(counter_value(&_profile, "SkippedLines"), 0); + EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6); +} + +// Scenario: Hive text has no embedded nested schema, but TableColumnMapper still needs semantic +// children for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children from the +// slot type while keeping the top-level local id as the text field ordinal from column_idxs. +TEST_F(TextV2ReaderTest, SchemaSynthesizesComplexChildrenForColumnMapper) { + _params.__set_column_idxs({4, 7, 9}); + auto slots = build_nested_complex_slots(&_pool); + auto reader = create_reader(_file_path, &_params, slots, &_state, &_profile); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + + EXPECT_EQ(schema[1].name, "xs"); + EXPECT_EQ(schema[1].local_id, 7); + ASSERT_EQ(schema[1].children.size(), 1); + EXPECT_EQ(schema[1].children[0].name, "element"); + EXPECT_EQ(schema[1].children[0].local_id, 0); + ASSERT_EQ(schema[1].children[0].children.size(), 2); + EXPECT_EQ(schema[1].children[0].children[0].name, "a"); + EXPECT_EQ(schema[1].children[0].children[0].local_id, 0); + EXPECT_EQ(schema[1].children[0].children[1].name, "b"); + EXPECT_EQ(schema[1].children[0].children[1].local_id, 1); + + EXPECT_EQ(schema[2].name, "kv"); + EXPECT_EQ(schema[2].local_id, 9); + ASSERT_EQ(schema[2].children.size(), 2); + EXPECT_EQ(schema[2].children[0].name, "key"); + EXPECT_EQ(schema[2].children[0].local_id, 0); + EXPECT_EQ(schema[2].children[1].name, "value"); + EXPECT_EQ(schema[2].children[1].local_id, 1); + ASSERT_EQ(schema[2].children[1].children.size(), 2); + EXPECT_EQ(schema[2].children[1].children[0].name, "a"); + EXPECT_EQ(schema[2].children[1].children[1].name, "b"); +} + +// Scenario: Hive text escapes a field separator inside a string. The splitter keeps the escaped +// separator in the same field, and hive-text serde unescapes the final string value. +TEST_F(TextV2ReaderTest, EscapedSeparatorStaysInsideStringField) { + const auto escaped_path = (_test_dir / "escaped.text").string(); + std::ofstream output(escaped_path, std::ios::binary); + output << "1,alice\\,team,10\n"; + output.close(); + + auto reader = create_reader(escaped_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1)), + LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(2), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1, 2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "alice,team"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 0), 10); +} + +// Hive LazySimpleSerDe treats an even-length escape run as escaped escape characters, so the +// following delimiter remains structural. V1 and V2 must both split the row at that delimiter. +TEST_F(TextV2ReaderTest, DoubleEscapeBeforeSeparatorStillSplitsField) { + const auto escaped_path = (_test_dir / "double_escaped.text").string(); + std::ofstream output(escaped_path, std::ios::binary); + output << R"(1|alice\\|10)" << '\n'; + output.close(); + + _params.file_attributes.text_params.__set_column_separator("|"); + auto reader = create_reader(escaped_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1)), + LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(2), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1, 2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), R"(alice\)"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 0), 10); +} + +// Scenario: Hive text supports multi-character field separators. V2 must not split on partial +// matches and must still honor FileScanRequest output positions. +TEST_F(TextV2ReaderTest, MultiCharacterSeparatorReadsRequestedColumns) { + const auto multi_path = (_test_dir / "multi.text").string(); + std::ofstream output(multi_path, std::ios::binary); + output << "3||carol||30\n"; + output.close(); + + _params.file_attributes.text_params.__set_column_separator("||"); + auto reader = create_reader(multi_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1)), + LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(0), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1, 0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "carol"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 0), 3); +} + +// Scenario: column_idxs can map table slots to non-identity Hive text field ordinals. +TEST_F(TextV2ReaderTest, ColumnIdxsMapSlotsToTextOrdinals) { + const auto remap_path = (_test_dir / "remapped.text").string(); + std::ofstream output(remap_path, std::ios::binary); + output << "doris,40,4\n"; + output.close(); + + _params.__set_column_idxs({2, 0, 1}); + auto reader = create_reader(remap_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + EXPECT_EQ(schema[0].local_id, 2); + EXPECT_EQ(schema[1].local_id, 0); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(2)), + LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(2), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(0), LocalIndex(1)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {2, 0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 4); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "doris"); +} + +// Scenario: Hive text complex values are encoded inside one top-level text field. V2 reads the +// complete struct field first, then evaluates a file-local predicate on one child, covering +// `SELECT s.a WHERE s.b > 10` without pretending that Text has physical nested-column pruning. +TEST_F(TextV2ReaderTest, FullStructColumnSupportsChildConjunctFiltering) { + const auto complex_path = (_test_dir / "complex.text").string(); + std::ofstream output(complex_path, std::ios::binary); + output << "1|11,5|10\n"; + output << "2|22,20|20\n"; + output.close(); + + _params.file_attributes.text_params.__set_column_separator("|"); + _params.file_attributes.text_params.__set_collection_delimiter(","); + _params.__set_column_idxs({0, 1, 2}); + auto slots = build_struct_slots(&_pool); + auto reader = create_reader(complex_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + request->conjuncts = {prepared_conjunct( + &_state, std::make_shared( + /*block_position=*/0, /*child_index=*/1, /*value=*/10))}; + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_struct_int_child_at(*block.get_by_position(0).column, 0, 0), 22); + EXPECT_EQ(nullable_struct_int_child_at(*block.get_by_position(0).column, 1, 0), 20); +} + +// Scenario: missing Hive text fields are materialized as NULL rather than shifting later columns. +TEST_F(TextV2ReaderTest, MissingRequestedFieldUsesNullFormat) { + const auto missing_path = (_test_dir / "missing.text").string(); + std::ofstream output(missing_path, std::ios::binary); + output << "1,alice\n"; + output.close(); + + auto reader = create_reader(missing_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(2), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 0)); +} + +// Scenario: Text v2 can scan a request with no materialized columns. This is used by table-level +// COUNT-style paths where the reader must still return the number of logical rows read. +TEST_F(TextV2ReaderTest, EmptyFileLocalProjectionStillReportsRows) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + Block block; + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 2); + EXPECT_FALSE(eof); +} + +// Scenario: stream load/http_stream text input is not backed by a filesystem. If TableReader fails +// to preserve the stream load id, the v2 reader should report that directly instead of calling the +// generic FileFactory path and returning "unsupported file reader type: 2". +TEST_F(TextV2ReaderTest, StreamInputRequiresLoadIdBeforeOpeningPipe) { + _params.__set_file_type(TFileType::FILE_STREAM); + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + const auto status = reader->open(request); + + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("stream reader requires load id"), std::string::npos) + << status; +} + +// Scenario: explicit text null_format is honored by Hive-text serde. Unlike CSV +// empty_field_as_null, an empty text field is not NULL unless it equals null_format exactly. +TEST_F(TextV2ReaderTest, NullFormatProducesNullableValue) { + const auto null_path = (_test_dir / "null_format.text").string(); + std::ofstream output(null_path, std::ios::binary); + output << "1,NULL,10\n"; + output << "2,,20\n"; + output.close(); + + _params.file_attributes.text_params.__set_null_format("NULL"); + auto reader = create_reader(null_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_FALSE(is_null_at(*block.get_by_position(0).column, 1)); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 1), ""); +} + +// Scenario: Hive SerDe can define the empty string itself as NULL. The nullable string fast path +// must match the generic nullable serde behavior instead of treating empty null_format as +// "null format is not configured". +TEST_F(TextV2ReaderTest, EmptyNullFormatProducesNullableValue) { + const auto null_path = (_test_dir / "empty_null_format.text").string(); + std::ofstream output(null_path, std::ios::binary); + output << "1,alice,10\n"; + output << "2,,20\n"; + output << "3,NULL,30\n"; + output.close(); + + _params.file_attributes.text_params.__set_null_format(""); + auto reader = create_reader(null_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 3); + EXPECT_FALSE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 1)); + EXPECT_FALSE(is_null_at(*block.get_by_position(0).column, 2)); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 2), "NULL"); +} + +// Scenario: TEXT_WITH_NAMES_AND_TYPES-style headers share the delimited text base skip path with +// CSV. Both header records must be skipped before the first data row is read. +TEST_F(TextV2ReaderTest, HeaderNamesAndTypesSkipsTwoLines) { + const auto header_path = (_test_dir / "header_names_types.text").string(); + std::ofstream output(header_path, std::ios::binary); + output << "id,name,score\n"; + output << "INT,STRING,INT\n"; + output << "7,carol,70\n"; + output.close(); + + _params.file_attributes.__set_header_type(BeConsts::CSV_WITH_NAMES_AND_TYPES); + auto reader = create_reader(header_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 7); +} + +// Scenario: the shared delimited text base removes UTF-8 BOM from the first returned data line. +// This matters for headerless text files whose first column is numeric. +TEST_F(TextV2ReaderTest, BomIsRemovedFromFirstDataLineWithoutHeader) { + const auto bom_path = (_test_dir / "bom_data.text").string(); + std::ofstream output(bom_path, std::ios::binary); + output.write("\xEF\xBB\xBF", 3); + output << "5,bom,50\n"; + output.close(); + + auto reader = create_reader(bom_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 5); +} + +// Scenario: when FE does not set header_type, skip_lines should be honored by the shared +// delimited text base before TextReader starts splitting rows. +TEST_F(TextV2ReaderTest, SkipLinesUsedWhenHeaderTypeUnset) { + const auto skip_path = (_test_dir / "skip_lines.text").string(); + std::ofstream output(skip_path, std::ios::binary); + output << "skip me\n"; + output << "skip me too\n"; + output << "3,dan,30\n"; + output.close(); + + _params.file_attributes.__isset.header_type = false; + _params.file_attributes.__set_skip_lines(2); + auto reader = create_reader(skip_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 3); +} + +// Scenario: Hive TEXTFILE treats an empty physical line as a record. For the first field it +// deserializes an empty value; missing trailing fields are filled with null_format. +TEST_F(TextV2ReaderTest, EmptyLineAsRecordByDefault) { + const auto empty_line_path = (_test_dir / "empty_line.text").string(); + std::ofstream output(empty_line_path, std::ios::binary); + output << "\n"; + output << "4,erin,40\n"; + output.close(); + + auto reader = create_reader(empty_line_path, &_params, _slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1)), + LocalColumnIndex::top_level(LocalColumnId(2))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + request->local_positions.emplace(LocalColumnId(2), LocalIndex(2)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0, 1, 2}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_TRUE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_TRUE(is_null_at(*block.get_by_position(1).column, 0)); + EXPECT_TRUE(is_null_at(*block.get_by_position(2).column, 0)); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 1), 4); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 1), "erin"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(2).column, 1), 40); +} + +// Scenario: for a single-column Hive TEXTFILE table, an empty physical line is one empty string +// field rather than a skipped row. +TEST_F(TextV2ReaderTest, EmptyLineAsSingleEmptyStringField) { + const auto empty_line_path = (_test_dir / "empty_line_single_string.text").string(); + std::ofstream output(empty_line_path, std::ios::binary); + output << "\n"; + output << "erin\n"; + output.close(); + + _params.__set_column_idxs({0}); + const std::vector slots {make_test_slot( + &_pool, 0, 0, make_nullable(std::make_shared()), "value")}; + auto reader = create_reader(empty_line_path, &_params, slots, &_state, &_profile); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_FALSE(is_null_at(*block.get_by_position(0).column, 0)); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), ""); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 1), "erin"); +} + +// Scenario: text v2 COUNT pushdown counts empty physical lines as Hive TEXTFILE records. +TEST_F(TextV2ReaderTest, CountAggregatePreservesEmptyLines) { + const auto empty_line_path = (_test_dir / "empty_line_count.text").string(); + std::ofstream output(empty_line_path, std::ios::binary); + output << "\n"; + output << "4,erin,40\n"; + output.close(); + + auto reader = create_reader(empty_line_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + FileAggregateResult aggregate_result; + ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result).ok()); + EXPECT_EQ(aggregate_result.count, 2); +} + +// Scenario: Text v2 COUNT pushdown scans rows because text files do not expose row-count metadata. +TEST_F(TextV2ReaderTest, CountAggregateScansRows) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + FileAggregateResult aggregate_result; + ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result).ok()); + EXPECT_EQ(aggregate_result.count, 2); +} + +// Scenario: a non-first split starts inside a text record and must skip the partial first line. +TEST_F(TextV2ReaderTest, NonFirstSplitSkipsPartialFirstRecord) { + const auto split_path = (_test_dir / "split.text").string(); + std::ofstream output(split_path, std::ios::binary); + output << "1,skip,10\n"; + output << "2,bob,20\n"; + output.close(); + + auto reader = create_reader(split_path, &_params, _slots, &_state, &_profile, + /*range_start_offset=*/3); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2); +} + +// Scenario: compressed text cannot be split at arbitrary byte offsets because the decompressor +// needs the stream from the beginning. V2 should reject such a split before constructing the line +// reader. +TEST_F(TextV2ReaderTest, NonFirstCompressedSplitReturnsError) { + _params.__set_compress_type(TFileCompressType::GZ); + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile, + /*range_start_offset=*/1); + + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + EXPECT_FALSE(reader->open(request).ok()); +} + +// Scenario: FileScanRequest is a TableReader-to-FileReader contract. Unknown TEXT ordinals, +// out-of-range block positions, and sparse block-position maps must fail during reader open. +TEST_F(TextV2ReaderTest, InvalidScanRequestReturnsError) { + { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(99))}; + request->local_positions.emplace(LocalColumnId(99), LocalIndex(0)); + EXPECT_FALSE(reader->open(request).ok()); + } + { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(2)); + EXPECT_FALSE(reader->open(request).ok()); + } + { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), + LocalColumnIndex::top_level(LocalColumnId(1))}; + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(0)); + EXPECT_FALSE(reader->open(request).ok()); + } +} + +// Scenario: unsupported aggregate requests must fail explicitly instead of returning partial +// results from the scan path. +TEST_F(TextV2ReaderTest, UnsupportedAggregateReturnsNotSupported) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + FileAggregateResult aggregate_result; + EXPECT_FALSE(reader->get_aggregate_result(aggregate_request, &aggregate_result).ok()); +} + +} // namespace +} // namespace doris::format::text diff --git a/be/test/format_v2/expr/cast_test.cpp b/be/test/format_v2/expr/cast_test.cpp new file mode 100644 index 00000000000000..341b89433f0c08 --- /dev/null +++ b/be/test/format_v2/expr/cast_test.cpp @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/expr/cast.h" + +#include + +#include +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "runtime/descriptors.h" +#include "testutil/column_helper.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris::format { + +class CastTest : public testing::Test { +protected: + void SetUp() override { state.set_enable_strict_cast(true); } + + static VExprContextSPtr create_context(const DataTypePtr& return_type, + const DataTypePtr& child_type, int child_column_id = 0) { + auto cast = Cast::create_shared(return_type); + cast->add_child(VSlotRef::create_shared(child_column_id, child_column_id, -1, child_type, + "source_column")); + return VExprContext::create_shared(cast); + } + + Status prepare_open_execute(VExprContext* context, Block* block, int* result_column_id) { + RETURN_IF_ERROR(context->prepare(&state, RowDescriptor())); + RETURN_IF_ERROR(context->open(&state)); + return context->execute(block, result_column_id); + } + + MockRuntimeState state; +}; + +TEST_F(CastTest, CastIntSlotToBigInt) { + auto source_type = std::make_shared(); + auto return_type = std::make_shared(); + auto context = create_context(return_type, source_type); + Block block; + block.insert(ColumnHelper::create_column_with_name({1, -2, 3})); + + int result_column_id = -1; + auto status = prepare_open_execute(context.get(), &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + ASSERT_EQ(result_column_id, 1); + ASSERT_EQ(block.columns(), 2); + EXPECT_EQ(block.get_by_position(result_column_id).type, return_type); + const auto& result_column = + assert_cast(*block.get_by_position(result_column_id).column); + EXPECT_EQ(result_column.get_data()[0], 1); + EXPECT_EQ(result_column.get_data()[1], -2); + EXPECT_EQ(result_column.get_data()[2], 3); + + context->close(); +} + +TEST_F(CastTest, CastStringSlotToNullableInt) { + state.set_enable_strict_cast(false); + auto source_type = std::make_shared(); + auto return_type = std::make_shared(std::make_shared()); + auto context = create_context(return_type, source_type); + Block block; + block.insert(ColumnHelper::create_column_with_name({"10", "bad", "-3"})); + + int result_column_id = -1; + auto status = prepare_open_execute(context.get(), &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + const auto& nullable_column = + assert_cast(*block.get_by_position(result_column_id).column); + const auto& result_column = + assert_cast(nullable_column.get_nested_column()); + const auto& null_map = nullable_column.get_null_map_data(); + EXPECT_EQ(result_column.get_data()[0], 10); + EXPECT_EQ(result_column.get_data()[2], -3); + EXPECT_EQ(null_map[0], 0); + EXPECT_EQ(null_map[1], 1); + EXPECT_EQ(null_map[2], 0); + + context->close(); +} + +TEST_F(CastTest, CastLiteralToString) { + auto source_type = std::make_shared(); + auto return_type = std::make_shared(); + auto cast = Cast::create_shared(return_type); + cast->add_child(VLiteral::create_shared(source_type, Field::create_field(123))); + auto context = VExprContext::create_shared(cast); + Block block; + block.insert(ColumnHelper::create_column_with_name({1, 2, 3})); + + int result_column_id = -1; + auto status = prepare_open_execute(context.get(), &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + const auto& result = block.get_by_position(result_column_id); + EXPECT_EQ(result.type->to_string(*result.column, 0), "123"); + EXPECT_EQ(result.type->to_string(*result.column, 1), "123"); + EXPECT_EQ(result.type->to_string(*result.column, 2), "123"); + + context->close(); +} + +TEST_F(CastTest, EmptyBlockAppendsEmptyResultColumn) { + auto source_type = std::make_shared(); + auto return_type = std::make_shared(); + auto context = create_context(return_type, source_type); + Block block; + block.insert(ColumnHelper::create_column_with_name({})); + + int result_column_id = -1; + auto status = prepare_open_execute(context.get(), &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + ASSERT_EQ(result_column_id, 1); + EXPECT_EQ(block.get_by_position(result_column_id).column->size(), 0); + + context->close(); +} + +TEST_F(CastTest, PrepareRejectsMissingChild) { + auto cast = Cast::create_shared(std::make_shared()); + VExprContext context(cast); + + auto status = context.prepare(&state, RowDescriptor()); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("exactly 1 child expr"), std::string::npos); +} + +TEST_F(CastTest, PrepareRejectsMultipleChildren) { + auto child_type = std::make_shared(); + auto cast = Cast::create_shared(std::make_shared()); + cast->add_child(VSlotRef::create_shared(0, 0, -1, child_type, "c0")); + cast->add_child(VSlotRef::create_shared(1, 1, -1, child_type, "c1")); + VExprContext context(cast); + + auto status = context.prepare(&state, RowDescriptor()); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("exactly 1 child expr"), std::string::npos); +} + +} // namespace doris::format diff --git a/be/test/format_v2/expr/delete_predicate_test.cpp b/be/test/format_v2/expr/delete_predicate_test.cpp new file mode 100644 index 00000000000000..2f8d4af0f5b177 --- /dev/null +++ b/be/test/format_v2/expr/delete_predicate_test.cpp @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/expr/delete_predicate.h" + +#include + +#include +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "exprs/vexpr_context.h" +#include "runtime/descriptors.h" +#include "testutil/mock/mock_slot_ref.h" + +namespace doris::format { + +class DeletePredicateTest : public testing::Test { +protected: + static Block make_block(const std::vector& row_ids) { + auto column = ColumnInt64::create(); + for (auto row_id : row_ids) { + column->insert_value(row_id); + } + + Block block; + block.insert({std::move(column), std::make_shared(), "row_id"}); + return block; + } + + static std::vector result_column_data(const Block& block, int result_column_id) { + const auto& result_column = + assert_cast(*block.get_by_position(result_column_id).column); + return {result_column.get_data().begin(), result_column.get_data().end()}; + } + + static Status execute_delete_predicate(const std::vector& deleted_rows, Block* block, + int* result_column_id) { + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->_open_finished = true; + delete_predicate->add_child( + std::make_shared(0, std::make_shared())); + + VExprContext context(delete_predicate); + return delete_predicate->execute(&context, block, result_column_id); + } + + static Status execute_delete_predicate(const roaring::Roaring64Map& deletion_vector, + Block* block, int* result_column_id) { + auto delete_predicate = std::make_shared(deletion_vector); + delete_predicate->_open_finished = true; + delete_predicate->add_child( + std::make_shared(0, std::make_shared())); + + VExprContext context(delete_predicate); + return delete_predicate->execute(&context, block, result_column_id); + } +}; + +TEST_F(DeletePredicateTest, MatchDeletedRowsInInputRange) { + const std::vector deleted_rows {-3, 1, 4, 8, 12, 20}; + auto block = make_block({0, 1, 2, 3, 4, 5, 8, 12}); + + int result_column_id = -1; + auto status = execute_delete_predicate(deleted_rows, &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(result_column_id, 1); + EXPECT_EQ(result_column_data(block, result_column_id), + std::vector({0, 1, 0, 0, 1, 0, 1, 1})); +} + +TEST_F(DeletePredicateTest, MatchCompressedDeletionVectorWithoutExpansion) { + // The cached DV remains a Roaring bitmap all the way into the file-level predicate. Include + // values on both sides of the current batch to exercise the iterator range seek. + const roaring::Roaring64Map deletion_vector {1, 4, 8, 12, 20}; + auto block = make_block({0, 1, 2, 3, 4, 5, 8, 12}); + + int result_column_id = -1; + const auto status = execute_delete_predicate(deletion_vector, &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(result_column_data(block, result_column_id), + std::vector({0, 1, 0, 0, 1, 0, 1, 1})); +} + +TEST_F(DeletePredicateTest, EmptyDeletedRowsReturnAllFalse) { + const std::vector deleted_rows; + auto block = make_block({1, 2, 3}); + + int result_column_id = -1; + auto status = execute_delete_predicate(deleted_rows, &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(result_column_data(block, result_column_id), std::vector({0, 0, 0})); +} + +TEST_F(DeletePredicateTest, DeletedRowsOutsideInputRangeReturnAllFalse) { + const std::vector deleted_rows {-10, -1, 10, 11}; + auto block = make_block({1, 2, 3}); + + int result_column_id = -1; + auto status = execute_delete_predicate(deleted_rows, &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(result_column_data(block, result_column_id), std::vector({0, 0, 0})); +} + +TEST_F(DeletePredicateTest, EmptyRowIdColumnAppendsEmptyResultColumn) { + const std::vector deleted_rows {1, 2, 3}; + auto block = make_block({}); + + int result_column_id = -1; + auto status = execute_delete_predicate(deleted_rows, &block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(block.columns(), 2); + EXPECT_EQ(result_column_id, 1); + EXPECT_EQ(result_column_data(block, result_column_id), std::vector({})); +} + +TEST_F(DeletePredicateTest, MissingRowIdColumnReturnsError) { + const std::vector deleted_rows {1, 2, 3}; + Block block; + + int result_column_id = -1; + auto status = execute_delete_predicate(deleted_rows, &block, &result_column_id); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("invalid column id"), std::string::npos); + EXPECT_EQ(block.columns(), 0); + EXPECT_EQ(result_column_id, -1); +} + +TEST_F(DeletePredicateTest, MissingRowIdChildReturnsError) { + const std::vector deleted_rows {1}; + auto block = make_block({1}); + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->_open_finished = true; + VExprContext context(delete_predicate); + + int result_column_id = -1; + auto status = delete_predicate->execute(&context, &block, &result_column_id); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("exactly 1 child expr"), std::string::npos); +} + +TEST_F(DeletePredicateTest, ExecuteColumnImplReturnsError) { + const std::vector deleted_rows {1}; + DeletePredicate delete_predicate(deleted_rows); + VExprContext context(std::make_shared(deleted_rows)); + ColumnPtr result_column; + + auto status = + delete_predicate.execute_column_impl(&context, nullptr, nullptr, 0, result_column); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("DeletePredicate::execute_column_impl"), std::string::npos); +} + +TEST_F(DeletePredicateTest, LifecycleAndDebugString) { + const std::vector deleted_rows {1}; + DeletePredicate delete_predicate(deleted_rows); + VExprContext context(std::make_shared(deleted_rows)); + RowDescriptor row_desc; + + auto status = delete_predicate.prepare(nullptr, row_desc, &context); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(delete_predicate.expr_name(), "DeletePredicate"); + EXPECT_EQ(delete_predicate.debug_string(), "DeletePredicate"); + + status = delete_predicate.open(nullptr, &context, FunctionContext::THREAD_LOCAL); + ASSERT_TRUE(status.ok()) << status; + delete_predicate.close(&context, FunctionContext::THREAD_LOCAL); +} + +} // namespace doris::format diff --git a/be/test/format_v2/expr/equality_delete_predicate_test.cpp b/be/test/format_v2/expr/equality_delete_predicate_test.cpp new file mode 100644 index 00000000000000..0f63e915f28a53 --- /dev/null +++ b/be/test/format_v2/expr/equality_delete_predicate_test.cpp @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/expr/equality_delete_predicate.h" + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exprs/vexpr_context.h" +#include "format_v2/expr/cast.h" +#include "runtime/descriptors.h" +#include "testutil/column_helper.h" +#include "testutil/mock/mock_runtime_state.h" +#include "testutil/mock/mock_slot_ref.h" + +namespace doris::format { + +class EqualityDeletePredicateTest : public testing::Test { +protected: + static ColumnWithTypeAndName make_nullable_int_column( + const std::string& name, const std::vector>& values) { + auto data = ColumnInt32::create(); + auto null_map = ColumnUInt8::create(); + for (const auto& value : values) { + data->insert_value(value.value_or(0)); + null_map->insert_value(!value.has_value()); + } + auto type = make_nullable(std::make_shared()); + return {ColumnNullable::create(std::move(data), std::move(null_map)), type, name}; + } + + static ColumnWithTypeAndName make_nullable_string_column( + const std::string& name, const std::vector>& values) { + auto data = ColumnString::create(); + auto null_map = ColumnUInt8::create(); + for (const auto& value : values) { + const std::string data_value = value.value_or(""); + data->insert_data(data_value.data(), data_value.size()); + null_map->insert_value(!value.has_value()); + } + auto type = make_nullable(std::make_shared()); + return {ColumnNullable::create(std::move(data), std::move(null_map)), type, name}; + } + + static std::vector result_column_data(const Block& block, int result_column_id) { + const auto& result_column = + assert_cast(*block.get_by_position(result_column_id).column); + return {result_column.get_data().begin(), result_column.get_data().end()}; + } + + static Status execute_equality_delete_predicate(Block delete_block, std::vector field_ids, + Block* data_block, int* result_column_id) { + auto predicate = + std::make_shared(std::move(delete_block), field_ids); + predicate->_open_finished = true; + for (size_t idx = 0; idx < field_ids.size(); ++idx) { + predicate->add_child( + std::make_shared(idx, data_block->get_by_position(idx).type)); + } + + VExprContext context(predicate); + return predicate->execute(&context, data_block, result_column_id); + } + + static Status execute_prepared_equality_delete_predicate(const VExprContextSPtr& context, + MockRuntimeState* state, + Block* data_block, + int* result_column_id) { + RETURN_IF_ERROR(context->prepare(state, RowDescriptor())); + RETURN_IF_ERROR(context->open(state)); + return context->execute(data_block, result_column_id); + } +}; + +TEST_F(EqualityDeletePredicateTest, MatchSingleColumn) { + Block delete_block; + delete_block.insert(make_nullable_int_column("id", {1, 4})); + Block data_block; + data_block.insert(make_nullable_int_column("id", {1, 2, 3, 4})); + + int result_column_id = -1; + auto status = execute_equality_delete_predicate(std::move(delete_block), {1}, &data_block, + &result_column_id); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({1, 0, 0, 1})); +} + +TEST_F(EqualityDeletePredicateTest, UsesPopulatedPredicateColumnForLazyBatchSize) { + Block delete_block; + delete_block.insert(make_nullable_int_column("id", {1, 4})); + Block data_block; + // A lazy reader can leave the first projected column unread while decoding a later predicate + // column. Block::rows() is therefore zero even though the equality key has four rows. + data_block.insert(make_nullable_string_column("unread", {})); + data_block.insert(make_nullable_int_column("id", {1, 2, 3, 4})); + + auto predicate = std::make_shared(std::move(delete_block), + std::vector {1}); + predicate->_open_finished = true; + predicate->add_child(std::make_shared(1, data_block.get_by_position(1).type)); + VExprContext context(predicate); + + int result_column_id = -1; + auto status = predicate->execute(&context, &data_block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({1, 0, 0, 1})); +} + +TEST_F(EqualityDeletePredicateTest, MatchMultipleColumns) { + Block delete_block; + delete_block.insert(make_nullable_int_column("id", {1, 2})); + delete_block.insert(make_nullable_string_column("name", {"a", "b"})); + Block data_block; + data_block.insert(make_nullable_int_column("id", {1, 1, 2, 2})); + data_block.insert(make_nullable_string_column("name", {"a", "b", "a", "b"})); + + int result_column_id = -1; + auto status = execute_equality_delete_predicate(std::move(delete_block), {1, 2}, &data_block, + &result_column_id); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({1, 0, 0, 1})); +} + +TEST_F(EqualityDeletePredicateTest, MatchNullValues) { + Block delete_block; + delete_block.insert(make_nullable_int_column("id", {std::nullopt})); + Block data_block; + data_block.insert(make_nullable_int_column("id", {1, std::nullopt, 3})); + + int result_column_id = -1; + auto status = execute_equality_delete_predicate(std::move(delete_block), {1}, &data_block, + &result_column_id); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({0, 1, 0})); +} + +TEST_F(EqualityDeletePredicateTest, MatchAfterCastToDeleteKeyType) { + Block delete_block; + delete_block.insert(make_nullable_int_column("id", {1, 4})); + Block data_block; + data_block.insert(ColumnHelper::create_column_with_name({1, 2, 4})); + + auto predicate = std::make_shared(std::move(delete_block), + std::vector {1}); + auto cast_expr = Cast::create_shared(make_nullable(std::make_shared())); + cast_expr->add_child(std::make_shared(0, data_block.get_by_position(0).type)); + predicate->add_child(std::move(cast_expr)); + auto context = VExprContext::create_shared(predicate); + MockRuntimeState state; + + int result_column_id = -1; + auto status = execute_prepared_equality_delete_predicate(context, &state, &data_block, + &result_column_id); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({1, 0, 1})); + context->close(); +} + +TEST_F(EqualityDeletePredicateTest, ChildCountMismatchReturnsError) { + Block delete_block; + delete_block.insert(make_nullable_int_column("id", {1})); + auto predicate = std::make_shared(std::move(delete_block), + std::vector {1}); + predicate->_open_finished = true; + Block data_block; + data_block.insert(make_nullable_int_column("id", {1})); + VExprContext context(predicate); + + int result_column_id = -1; + auto status = predicate->execute(&context, &data_block, &result_column_id); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("should have 1 child exprs"), std::string::npos); +} + +} // namespace doris::format diff --git a/be/test/format_v2/jni/jni_table_reader_test.cpp b/be/test/format_v2/jni/jni_table_reader_test.cpp new file mode 100644 index 00000000000000..7510708e0178ca --- /dev/null +++ b/be/test/format_v2/jni/jni_table_reader_test.cpp @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/jni_table_reader.h" + +#include + +#include +#include +#include +#include + +#include "io/io_common.h" + +namespace doris::format { +namespace { + +class FakeJniTableReader final : public JniTableReader { +public: + int get_next_calls = 0; + int close_calls = 0; + int64_t close_profile_delta = 0; + RuntimeProfile::Counter* close_profile_counter = nullptr; + std::vector open_batch_sizes; + std::vector propagated_batch_sizes; + std::vector close_results; + bool next_eof = false; + +protected: + std::string connector_class() const override { return "test/FakeJniScanner"; } + + Status build_scanner_params(std::map* params) const override { + params->clear(); + return Status::OK(); + } + + Status _get_next_jni_block(size_t* rows, bool* eof) override { + ++get_next_calls; + *rows = 0; + *eof = next_eof; + return Status::OK(); + } + + Status _close_jni_scanner() override { + if (!TEST_scanner_opened()) { + return Status::OK(); + } + ++close_calls; + if (_reserve_split_profile_publication() && close_profile_counter != nullptr) { + COUNTER_UPDATE(close_profile_counter, close_profile_delta); + } + Status status = Status::OK(); + if (static_cast(close_calls) <= close_results.size()) { + status = close_results[close_calls - 1]; + } + if (status.ok()) { + TEST_set_split_state(false, TEST_eof()); + } + return status; + } + + Status _set_open_scanner_batch_size(size_t batch_size) override { + propagated_batch_sizes.push_back(batch_size); + return Status::OK(); + } + + Status _open_jni_scanner() override { + open_batch_sizes.push_back(TEST_batch_size()); + TEST_set_split_state(true, false); + return Status::OK(); + } +}; + +Status init_reader(FakeJniTableReader* reader, const std::shared_ptr& io_ctx, + RuntimeProfile* scanner_profile = nullptr) { + return reader->init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::JNI, + .scan_params = nullptr, + .io_ctx = io_ctx, + .runtime_state = nullptr, + .scanner_profile = scanner_profile, + }); +} + +TEST(JniTableReaderTest, CancellationStopsBeforeFetchingAnotherJavaBatch) { + auto io_ctx = std::make_shared(); + FakeJniTableReader reader; + ASSERT_TRUE(init_reader(&reader, io_ctx).ok()); + reader.TEST_set_split_state(true, false); + io_ctx->should_stop = true; + + Block block; + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(reader.get_next_calls, 0); + EXPECT_EQ(reader.close_calls, 1); +} + +TEST(JniTableReaderTest, EndOfSplitRemainsIdempotentAfterScannerClose) { + FakeJniTableReader reader; + ASSERT_TRUE(init_reader(&reader, nullptr).ok()); + reader.TEST_set_split_state(true, false); + reader.next_eof = true; + + Block block; + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_TRUE(reader.TEST_eof()); + EXPECT_FALSE(reader.TEST_scanner_opened()); + + eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(reader.get_next_calls, 1); + EXPECT_EQ(reader.close_calls, 1); +} + +TEST(JniTableReaderTest, FailedCloseCanBeRetried) { + RuntimeProfile profile("FailedCloseCanBeRetried"); + FakeJniTableReader reader; + reader.close_profile_counter = profile.add_counter("PublishedCloseProfile", TUnit::UNIT); + reader.close_profile_delta = 17; + ASSERT_TRUE(init_reader(&reader, nullptr, &profile).ok()); + reader.TEST_set_split_state(true, false); + reader.close_results = {Status::InternalError("injected close failure"), Status::OK()}; + + EXPECT_FALSE(reader.close().ok()); + EXPECT_FALSE(reader.TEST_closed()); + EXPECT_TRUE(reader.TEST_scanner_opened()); + EXPECT_EQ(reader.close_calls, 1); + EXPECT_EQ(reader.close_profile_counter->value(), 17); + + EXPECT_TRUE(reader.close().ok()); + EXPECT_TRUE(reader.TEST_closed()); + EXPECT_EQ(reader.close_calls, 2); + EXPECT_EQ(reader.close_profile_counter->value(), 17); +} + +TEST(JniTableReaderTest, AdaptiveBatchSizeUpdatesAnOpenJavaScanner) { + FakeJniTableReader reader; + ASSERT_TRUE(init_reader(&reader, nullptr).ok()); + + reader.set_batch_size(17); + EXPECT_EQ(reader.TEST_batch_size(), 17); + EXPECT_TRUE(reader.propagated_batch_sizes.empty()); + + reader.TEST_set_split_state(true, false); + reader.set_batch_size(33); + EXPECT_EQ(reader.TEST_batch_size(), 33); + EXPECT_EQ(reader.propagated_batch_sizes, std::vector({33})); +} + +TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { + FakeJniTableReader reader; + ASSERT_TRUE(init_reader(&reader, nullptr).ok()); + + reader.set_batch_size(32); + ASSERT_TRUE(reader.prepare_split({ + .partition_values = {}, + .conjuncts = std::nullopt, + .partition_prune_conjuncts = {}, + .cache = nullptr, + .current_range = {}, + .current_split_format = FileFormat::JNI, + .global_rowid_context = std::nullopt, + }) + .ok()); + + EXPECT_EQ(reader.open_batch_sizes, std::vector({32})); + EXPECT_TRUE(reader.TEST_scanner_opened()); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/jni/paimon_jni_reader_test.cpp b/be/test/format_v2/jni/paimon_jni_reader_test.cpp new file mode 100644 index 00000000000000..570977a5f7eb4e --- /dev/null +++ b/be/test/format_v2/jni/paimon_jni_reader_test.cpp @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/paimon_jni_reader.h" + +#include + +#include +#include +#include + +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::paimon { +namespace { + +TFileRangeDesc make_paimon_jni_range() { + TFileRangeDesc range; + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("paimon"); + TPaimonFileDesc paimon_params; + paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_JNI); + paimon_params.__set_paimon_split("serialized-split"); + table_format_params.__set_paimon_params(std::move(paimon_params)); + range.__set_table_format_params(std::move(table_format_params)); + return range; +} + +TFileScanRangeParams make_scan_params() { + TFileScanRangeParams scan_params; + scan_params.__set_serialized_table("serialized-table"); + return scan_params; +} + +Status init_reader(PaimonJniReader* reader, TFileScanRangeParams* scan_params) { + return reader->init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::JNI, + .scan_params = scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = nullptr, + }); +} + +Status build_params(PaimonJniReader* reader, const TFileRangeDesc& range, + std::map* params) { + reader->_current_range = range; + return reader->build_scanner_params(params); +} + +TEST(PaimonJniReaderTest, UsesScanLevelPredicateBeforeLegacySplitPredicate) { + auto range = make_paimon_jni_range(); + range.table_format_params.paimon_params.__set_paimon_predicate("legacy-predicate"); + + auto scan_params = make_scan_params(); + scan_params.__set_paimon_predicate("scan-predicate"); + + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + ASSERT_TRUE(reader.validate_scan_range(range).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["paimon_predicate"], "scan-predicate"); +} + +TEST(PaimonJniReaderTest, FallsBackToLegacySplitPredicateWhenScanPredicateIsMissing) { + auto range = make_paimon_jni_range(); + range.table_format_params.paimon_params.__set_paimon_predicate("legacy-predicate"); + + auto scan_params = make_scan_params(); + + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + ASSERT_TRUE(reader.validate_scan_range(range).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["paimon_predicate"], "legacy-predicate"); +} + +TEST(PaimonJniReaderTest, FallsBackToLegacySplitPredicateWhenScanPredicateIsEmpty) { + auto range = make_paimon_jni_range(); + range.table_format_params.paimon_params.__set_paimon_predicate("legacy-predicate"); + + auto scan_params = make_scan_params(); + scan_params.__set_paimon_predicate(""); + + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + ASSERT_TRUE(reader.validate_scan_range(range).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["paimon_predicate"], "legacy-predicate"); +} + +TEST(PaimonJniReaderTest, RejectsMissingPredicateFromBothProtocolLocations) { + const auto range = make_paimon_jni_range(); + auto scan_params = make_scan_params(); + + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + const auto status = reader.validate_scan_range(range); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("missing paimon_predicate"), std::string::npos); +} + +TEST(PaimonJniReaderTest, FallsBackToLegacySplitOptionsAndHadoopConf) { + auto range = make_paimon_jni_range(); + auto& paimon_params = range.table_format_params.paimon_params; + paimon_params.__set_paimon_predicate("legacy-predicate"); + paimon_params.__set_paimon_options({{"legacy-option", "legacy-value"}}); + paimon_params.__set_hadoop_conf({{"fs.defaultFS", "hdfs://legacy"}}); + + auto scan_params = make_scan_params(); + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["paimon.legacy-option"], "legacy-value"); + EXPECT_EQ(params["hadoop.fs.defaultFS"], "hdfs://legacy"); +} + +TEST(PaimonJniReaderTest, ScanLevelOptionsOverrideLegacySplitFallbacks) { + auto range = make_paimon_jni_range(); + auto& paimon_params = range.table_format_params.paimon_params; + paimon_params.__set_paimon_predicate("legacy-predicate"); + paimon_params.__set_paimon_options({{"source", "legacy"}}); + paimon_params.__set_hadoop_conf({{"source", "legacy"}}); + + auto scan_params = make_scan_params(); + scan_params.__set_paimon_options({{"source", "scan"}}); + scan_params.__set_properties({{"source", "scan"}}); + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["paimon.source"], "scan"); + EXPECT_EQ(params["hadoop.source"], "scan"); +} + +TEST(PaimonJniReaderTest, KeepsInitialPhysicalBatchSizeAfterOpen) { + PaimonJniReader reader; + reader.set_batch_size(32); + EXPECT_EQ(reader.TEST_batch_size(), 32); + + // Paimon copies the constructor size into the RecordReader during Java open. A later predictor + // result cannot resize that physical reader, so keep the initial probe size for the split. + reader.TEST_set_split_state(true, false); + reader.set_batch_size(1); + EXPECT_EQ(reader.TEST_batch_size(), 32); +} + +} // namespace +} // namespace doris::format::paimon diff --git a/be/test/format_v2/json/json_reader_test.cpp b/be/test/format_v2/json/json_reader_test.cpp new file mode 100644 index 00000000000000..683a47dee299cc --- /dev/null +++ b/be/test/format_v2/json/json_reader_test.cpp @@ -0,0 +1,608 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/json/json_reader.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/object_pool.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/column_data.h" +#include "io/io_common.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_profile.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris::format::json { +namespace { + +TFileScanRangeParams json_scan_params(bool read_json_by_line = true, bool strip_outer_array = false, + std::string jsonpaths = "", std::string json_root = "", + bool ignore_malformed = false) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_JSON); + params.__set_file_type(TFileType::FILE_LOCAL); + params.__set_compress_type(TFileCompressType::PLAIN); + TFileAttributes attributes; + TFileTextScanRangeParams text_params; + text_params.__set_line_delimiter("\n"); + attributes.__set_text_params(std::move(text_params)); + attributes.__set_read_json_by_line(read_json_by_line); + attributes.__set_strip_outer_array(strip_outer_array); + attributes.__set_num_as_string(false); + attributes.__set_fuzzy_parse(false); + if (!jsonpaths.empty()) { + attributes.__set_jsonpaths(std::move(jsonpaths)); + } + if (!json_root.empty()) { + attributes.__set_json_root(std::move(json_root)); + } + if (ignore_malformed) { + attributes.__set_openx_json_ignore_malformed(true); + } + params.__set_file_attributes(std::move(attributes)); + return params; +} + +SlotDescriptor* make_test_slot(ObjectPool* pool, int slot_id, int slot_idx, DataTypePtr type, + const std::string& name) { + TSlotDescriptor slot_desc; + slot_desc.__set_id(slot_id); + slot_desc.__set_parent(0); + slot_desc.__set_slotType(type->to_thrift()); + slot_desc.__set_columnPos(slot_idx); + slot_desc.__set_byteOffset(0); + if (type->is_nullable()) { + slot_desc.__set_nullIndicatorByte(slot_idx / 8); + slot_desc.__set_nullIndicatorBit(slot_idx % 8); + } else { + slot_desc.__set_nullIndicatorByte(0); + slot_desc.__set_nullIndicatorBit(-1); + } + slot_desc.__set_slotIdx(slot_idx); + slot_desc.__set_isMaterialized(true); + slot_desc.__set_colName(name); + return pool->add(new SlotDescriptor(slot_desc)); +} + +std::vector build_slots(ObjectPool* pool) { + return {make_test_slot(pool, 0, 0, make_nullable(std::make_shared()), "id"), + make_test_slot(pool, 1, 1, make_nullable(std::make_shared()), "name")}; +} + +std::vector build_slots_with_required_name(ObjectPool* pool) { + return {make_test_slot(pool, 0, 0, make_nullable(std::make_shared()), "id"), + make_test_slot(pool, 1, 1, std::make_shared(), "name")}; +} + +std::vector build_complex_slots(ObjectPool* pool) { + auto varchar_type = make_nullable(std::make_shared(8, TYPE_VARCHAR)); + auto array_type = make_nullable( + std::make_shared(make_nullable(std::make_shared()))); + auto map_type = make_nullable(std::make_shared( + std::make_shared(4, TYPE_CHAR), + make_nullable(std::make_shared(16, TYPE_VARCHAR)))); + auto struct_type = make_nullable(std::make_shared( + DataTypes {std::make_shared(8, TYPE_VARCHAR), + make_nullable(std::make_shared( + make_nullable(std::make_shared())))}, + Strings {"name", "scores"})); + return {make_test_slot(pool, 0, 0, varchar_type, "nickname"), + make_test_slot(pool, 1, 1, array_type, "tags"), + make_test_slot(pool, 2, 2, map_type, "props"), + make_test_slot(pool, 3, 3, struct_type, "profile")}; +} + +std::unique_ptr file_description(const std::string& path) { + auto desc = std::make_unique(); + desc->path = path; + desc->file_size = static_cast(std::filesystem::file_size(path)); + desc->range_start_offset = 0; + desc->range_size = desc->file_size; + return desc; +} + +std::filesystem::path write_json_file(const std::string& name, const std::string& content) { + const auto test_dir = std::filesystem::temp_directory_path() / "doris_format_v2_json_reader"; + std::filesystem::create_directories(test_dir); + const auto file_path = test_dir / name; + std::ofstream out(file_path); + out << content; + return file_path; +} + +TFileRangeDesc file_range(const std::filesystem::path& file_path) { + TFileRangeDesc range; + range.__set_path(file_path.string()); + range.__set_start_offset(0); + range.__set_size(static_cast(std::filesystem::file_size(file_path))); + range.__set_file_size(static_cast(std::filesystem::file_size(file_path))); + return range; +} + +Block make_block(const std::vector& schema, + const std::vector& local_ids) { + Block block; + for (const auto local_id : local_ids) { + const auto it = std::ranges::find_if( + schema, [&](const auto& column) { return column.local_id == local_id; }); + EXPECT_TRUE(it != schema.end()); + block.insert({it->type->create_column(), it->type, it->name}); + } + return block; +} + +struct ReadResult { + Status status; + Status second_status = Status::OK(); + Block block; + size_t rows = 0; + bool eof = false; + size_t second_rows = 0; + bool second_eof = false; + std::vector schema; +}; + +ReadResult read_once(const std::string& file_name, const std::string& content, + TFileScanRangeParams params, const std::vector& slots, + const std::vector& requested_local_ids, bool read_twice = false) { + const auto file_path = write_json_file(file_name, content); + auto range = file_range(file_path); + + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto desc = file_description(file_path.string()); + RuntimeProfile profile("json_v2_reader_test"); + MockRuntimeState state; + JsonReader reader(system_properties, desc, nullptr, &profile, ¶ms, range, slots); + + ReadResult result; + result.status = reader.init(&state); + if (!result.status.ok()) { + return result; + } + result.status = reader.get_schema(&result.schema); + if (!result.status.ok()) { + return result; + } + + auto request = std::make_shared(); + for (size_t i = 0; i < requested_local_ids.size(); ++i) { + request->local_positions.emplace(LocalColumnId(requested_local_ids[i]), LocalIndex(i)); + } + result.status = reader.open(request); + if (!result.status.ok()) { + return result; + } + + result.block = make_block(result.schema, requested_local_ids); + result.status = reader.get_block(&result.block, &result.rows, &result.eof); + if (result.status.ok() && read_twice) { + auto eof_block = make_block(result.schema, requested_local_ids); + result.second_status = + reader.get_block(&eof_block, &result.second_rows, &result.second_eof); + } + return result; +} + +std::string nullable_string_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data_at(row).to_string(); +} + +std::string string_at(const IColumn& column, size_t row) { + const auto& nested = assert_cast(column); + return nested.get_data_at(row).to_string(); +} + +int32_t nullable_int_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data()[row]; +} + +bool nullable_is_null_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + return nullable.is_null_at(row); +} + +class NullableIntGreaterThanExpr final : public VExpr { +public: + NullableIntGreaterThanExpr(size_t block_position, int32_t value) + : VExpr(std::make_shared(), false), + _block_position(block_position), + _value(value) {} + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_block_position).column); + const auto& data = assert_cast(nullable.get_nested_column()); + + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto source_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable.is_null_at(source_row) && data.get_element(source_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_block_position, _value); + return Status::OK(); + } + +private: + size_t _block_position; + int32_t _value; + const std::string _name = "NullableIntGreaterThanExpr"; +}; + +VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { + auto context = VExprContext::create_shared(expr); + auto status = context->prepare(state, RowDescriptor()); + EXPECT_TRUE(status.ok()) << status; + status = context->open(state); + EXPECT_TRUE(status.ok()) << status; + return context; +} + +} // namespace + +TEST(JsonReaderTest, ReadsRequestedColumnsInFileScanRequestOrder) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("order.jsonl", + R"({"id":1,"name":"alice"})" + "\n" + R"({"id":2,"name":"bob"})" + "\n", + json_scan_params(), slots, {1, 0}, true); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.schema.size(), 2); + EXPECT_EQ(result.schema[0].name, "id"); + EXPECT_EQ(result.schema[0].local_id, 0); + EXPECT_EQ(result.schema[1].name, "name"); + EXPECT_EQ(result.schema[1].local_id, 1); + ASSERT_EQ(result.rows, 2); + ASSERT_EQ(result.block.columns(), 2); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(0).column, 0), "alice"); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(0).column, 1), "bob"); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(1).column, 0), 1); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(1).column, 1), 2); + ASSERT_TRUE(result.second_status.ok()) << result.second_status.to_string(); + EXPECT_EQ(result.second_rows, 0); + EXPECT_TRUE(result.second_eof); +} + +TEST(JsonReaderTest, ReadsSingleDocumentOuterArray) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = + read_once("outer_array.json", R"([{"id":3,"name":"carol"},{"id":4,"name":"dave"}])", + json_scan_params(false, true), slots, {0, 1}); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 2); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 3); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 0), "carol"); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 1), 4); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 1), "dave"); +} + +TEST(JsonReaderTest, ReadsJsonRootByLine) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("json_root.jsonl", + R"({"payload":{"id":5,"name":"eve"}})" + "\n" + R"({"payload":{"id":6,"name":"frank"}})" + "\n", + json_scan_params(true, false, "", "$.payload"), slots, {0, 1}); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 2); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 5); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 0), "eve"); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 1), 6); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 1), "frank"); +} + +TEST(JsonReaderTest, ReadsJsonPathsBySourceSlotAndReturnsRequestedBlockOrder) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("jsonpaths.jsonl", + R"({"payload":{"id":7,"user":"grace"}})" + "\n" + R"({"payload":{"id":8,"user":"heidi"}})" + "\n", + json_scan_params(true, false, R"(["$.payload.id","$.payload.user"])"), + slots, {1, 0}); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 2); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(0).column, 0), "grace"); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(0).column, 1), "heidi"); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(1).column, 0), 7); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(1).column, 1), 8); +} + +TEST(JsonReaderTest, ReadsJsonPathsFromSingleDocumentOuterArray) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once( + "outer_array_jsonpaths.json", + R"([{"payload":{"id":12,"user":"kate"}},{"payload":{"id":13,"user":"leo"}}])", + json_scan_params(false, true, R"(["$.payload.id","$.payload.user"])"), slots, {0, 1}); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 2); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 12); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 0), "kate"); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 1), 13); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 1), "leo"); +} + +TEST(JsonReaderTest, FillsMissingNullableColumnWithNull) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("missing_nullable.jsonl", + R"({"id":9})" + "\n", + json_scan_params(), slots, {0, 1}); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 1); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 9); + EXPECT_TRUE(nullable_is_null_at(*result.block.get_by_position(1).column, 0)); +} + +TEST(JsonReaderTest, ReturnsErrorForMissingRequiredColumn) { + ObjectPool pool; + auto slots = build_slots_with_required_name(&pool); + auto result = read_once("missing_required.jsonl", + R"({"id":10})" + "\n", + json_scan_params(), slots, {0, 1}); + + EXPECT_FALSE(result.status.ok()); +} + +TEST(JsonReaderTest, ReadsPresentRequiredColumn) { + ObjectPool pool; + auto slots = build_slots_with_required_name(&pool); + auto result = read_once("present_required.jsonl", + R"({"id":14,"name":"mallory"})" + "\n", + json_scan_params(), slots, {0, 1}); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.schema.size(), 2); + EXPECT_TRUE(result.schema[0].type->is_nullable()); + EXPECT_FALSE(result.schema[1].type->is_nullable()); + ASSERT_EQ(result.rows, 1); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 14); + EXPECT_EQ(string_at(*result.block.get_by_position(1).column, 0), "mallory"); +} + +TEST(JsonReaderTest, SynthesizesComplexFileSchemaFromSlotTypes) { + ObjectPool pool; + auto slots = build_complex_slots(&pool); + const auto file_path = write_json_file("complex_schema.jsonl", "{}\n"); + auto params = json_scan_params(); + auto range = file_range(file_path); + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto desc = file_description(file_path.string()); + RuntimeProfile profile("json_v2_reader_complex_schema_test"); + MockRuntimeState state; + JsonReader reader(system_properties, desc, nullptr, &profile, ¶ms, range, slots); + + ASSERT_TRUE(reader.init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader.get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + EXPECT_EQ(schema[0].name, "nickname"); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_STRING); + + ASSERT_EQ(schema[1].children.size(), 1); + EXPECT_EQ(schema[1].children[0].name, "element"); + EXPECT_EQ(schema[1].children[0].local_id, 0); + EXPECT_EQ(remove_nullable(schema[1].children[0].type)->get_primitive_type(), TYPE_INT); + + ASSERT_EQ(schema[2].children.size(), 2); + EXPECT_EQ(schema[2].children[0].name, "key"); + EXPECT_EQ(schema[2].children[1].name, "value"); + EXPECT_EQ(remove_nullable(schema[2].children[0].type)->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(remove_nullable(schema[2].children[1].type)->get_primitive_type(), TYPE_STRING); + + ASSERT_EQ(schema[3].children.size(), 2); + EXPECT_EQ(schema[3].children[0].name, "name"); + EXPECT_EQ(schema[3].children[1].name, "scores"); + EXPECT_EQ(remove_nullable(schema[3].children[0].type)->get_primitive_type(), TYPE_STRING); + ASSERT_EQ(schema[3].children[1].children.size(), 1); + EXPECT_EQ(schema[3].children[1].children[0].name, "element"); + EXPECT_EQ(remove_nullable(schema[3].children[1].children[0].type)->get_primitive_type(), + TYPE_INT); +} + +TEST(JsonReaderTest, RejectsInvalidFileScanRequestsBeforeOpeningFile) { + ObjectPool pool; + auto slots = build_slots(&pool); + const auto file_path = write_json_file("invalid_request.jsonl", "{}\n"); + auto params = json_scan_params(); + auto range = file_range(file_path); + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto desc = file_description(file_path.string()); + RuntimeProfile profile("json_v2_reader_invalid_request_test"); + MockRuntimeState state; + JsonReader reader(system_properties, desc, nullptr, &profile, ¶ms, range, slots); + ASSERT_TRUE(reader.init(&state).ok()); + + auto unknown_column_request = std::make_shared(); + unknown_column_request->local_positions.emplace(LocalColumnId(9), LocalIndex(0)); + auto status = reader.open(unknown_column_request); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("unknown local column id 9"), std::string::npos); + + auto invalid_position_request = std::make_shared(); + invalid_position_request->local_positions.emplace(LocalColumnId(0), LocalIndex(2)); + status = reader.open(invalid_position_request); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("invalid block position 2"), std::string::npos); + + auto missing_position_request = std::make_shared(); + missing_position_request->local_positions.emplace(LocalColumnId(0), LocalIndex(1)); + missing_position_request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + status = reader.open(missing_position_request); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("misses block position 0"), std::string::npos); + + std::vector schema; + ASSERT_TRUE(reader.get_schema(&schema).ok()); + auto block = make_block(schema, {0}); + size_t rows = 0; + bool eof = false; + status = reader.get_block(&block, &rows, &eof); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("reader is not open"), std::string::npos); +} + +TEST(JsonReaderTest, ReturnsErrorForMalformedJsonByDefault) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("malformed_strict.jsonl", + "not-json\n" + R"({"id":11,"name":"judy"})" + "\n", + json_scan_params(), slots, {0, 1}); + + EXPECT_FALSE(result.status.ok()); +} + +TEST(JsonReaderTest, IgnoresMalformedJsonAsNullRowsWhenConfigured) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("ignore_malformed.jsonl", + "not-json\n" + R"({"id":11,"name":"judy"})" + "\n", + json_scan_params(true, false, "", "", true), slots, {0, 1}); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 2); + EXPECT_TRUE(nullable_is_null_at(*result.block.get_by_position(0).column, 0)); + EXPECT_TRUE(nullable_is_null_at(*result.block.get_by_position(1).column, 0)); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 1), 11); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 1), "judy"); +} + +TEST(JsonReaderTest, SkipsEmptyJsonLine) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("empty_line.jsonl", + "\n" + R"({"id":15,"name":"nancy"})" + "\n", + json_scan_params(), slots, {0, 1}); + + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 1); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 15); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 0), "nancy"); +} + +// Scenario: JSON, Native, CSV, and Hive text all share the same file-local filter order: +// delete conjuncts run first, ordinary conjuncts run second, and only ordinary conjuncts contribute +// to IOContext::predicate_filtered_rows. This guards the JSON caller of the shared helper because +// CSV/Text already assert the optional profile-counter path. +TEST(JsonReaderTest, AppliesDeleteAndNormalConjunctsWithPredicateFilterAccounting) { + ObjectPool pool; + auto slots = build_slots(&pool); + const auto file_path = write_json_file("filters.jsonl", R"({"id":1,"name":"alice"})" + "\n" + R"({"id":2,"name":"bob"})" + "\n" + R"({"id":3,"name":"carol"})" + "\n"); + auto params = json_scan_params(); + auto range = file_range(file_path); + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto desc = file_description(file_path.string()); + RuntimeProfile profile("json_v2_reader_filter_test"); + MockRuntimeState state; + auto io_ctx = std::make_shared(); + JsonReader reader(system_properties, desc, io_ctx, &profile, ¶ms, range, slots); + + ASSERT_TRUE(reader.init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader.get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); + request->delete_conjuncts = { + prepared_conjunct(&state, std::make_shared(0, 2))}; + request->conjuncts = { + prepared_conjunct(&state, std::make_shared(0, 1))}; + ASSERT_TRUE(reader.open(request).ok()); + + auto block = make_block(schema, {0, 1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader.get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "bob"); + EXPECT_EQ(io_ctx->predicate_filtered_rows, 1); +} + +} // namespace doris::format::json diff --git a/be/test/format_v2/native/native_reader_test.cpp b/be/test/format_v2/native/native_reader_test.cpp new file mode 100644 index 00000000000000..aaa7aa90e0681e --- /dev/null +++ b/be/test/format_v2/native/native_reader_test.cpp @@ -0,0 +1,419 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/native/native_reader.h" + +#include + +#include +#include +#include +#include +#include + +#include "agent/be_exec_version_manager.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format/native/native_format.h" +#include "format_v2/column_mapper.h" +#include "io/fs/local_file_system.h" +#include "io/io_common.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "util/coding.h" +#include "util/uid_util.h" + +namespace doris::format::native { +namespace { + +std::unique_ptr file_description(const std::string& path) { + auto desc = std::make_unique(); + desc->path = path; + desc->file_size = static_cast(std::filesystem::file_size(path)); + desc->range_start_offset = 0; + desc->range_size = desc->file_size; + return desc; +} + +Status write_file(const std::string& path, std::string_view content) { + io::FileWriterPtr writer; + RETURN_IF_ERROR(io::global_local_filesystem()->create_file(path, &writer)); + if (!content.empty()) { + RETURN_IF_ERROR(writer->append({content.data(), content.size()})); + } + return writer->close(); +} + +std::unique_ptr create_reader(const std::string& path, RuntimeState* state, + RuntimeProfile* profile, + std::shared_ptr io_ctx = nullptr) { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto desc = file_description(path); + return std::make_unique(system_properties, desc, std::move(io_ctx), profile); +} + +Block make_source_block() { + auto id_column = ColumnInt32::create(); + id_column->insert_value(10); + id_column->insert_value(20); + + auto name_column = ColumnString::create(); + name_column->insert_data("alice", 5); + name_column->insert_data("bob", 3); + + Block block; + block.insert({id_column->get_ptr(), std::make_shared(), "id"}); + block.insert({name_column->get_ptr(), std::make_shared(), "name"}); + return block; +} + +Status write_native_file(const std::string& path, const Block& block) { + io::FileWriterPtr writer; + RETURN_IF_ERROR(io::global_local_filesystem()->create_file(path, &writer)); + RETURN_IF_ERROR(writer->append({DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)})); + + uint8_t version_buffer[sizeof(uint32_t)]; + encode_fixed32_le(version_buffer, DORIS_NATIVE_FORMAT_VERSION); + RETURN_IF_ERROR(writer->append({version_buffer, sizeof(version_buffer)})); + + PBlock pblock; + size_t uncompressed_bytes = 0; + size_t compressed_bytes = 0; + int64_t compressed_time = 0; + RETURN_IF_ERROR(block.serialize(BeExecVersionManager::get_newest_version(), &pblock, + &uncompressed_bytes, &compressed_bytes, &compressed_time, + segment_v2::CompressionTypePB::SNAPPY)); + + const std::string payload = pblock.SerializeAsString(); + uint8_t len_buffer[sizeof(uint64_t)]; + encode_fixed64_le(len_buffer, payload.size()); + RETURN_IF_ERROR(writer->append({len_buffer, sizeof(len_buffer)})); + RETURN_IF_ERROR(writer->append(payload)); + return writer->close(); +} + +Block make_request_block(const std::vector& schema, + const std::vector& local_ids) { + Block block; + for (const auto local_id : local_ids) { + const auto it = std::find_if(schema.begin(), schema.end(), [&](const auto& column) { + return column.local_id == local_id; + }); + DORIS_CHECK(it != schema.end()); + block.insert({it->type->create_column(), it->type, it->name}); + } + return block; +} + +int32_t nullable_int_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data()[row]; +} + +std::string nullable_string_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data_at(row).to_string(); +} + +class NullableIntGreaterThanExpr final : public VExpr { +public: + NullableIntGreaterThanExpr(size_t block_position, int32_t value) + : VExpr(std::make_shared(), false), + _block_position(block_position), + _value(value) {} + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_block_position).column); + const auto& data = assert_cast(nullable.get_nested_column()); + + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto source_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable.is_null_at(source_row) && data.get_element(source_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_block_position, _value); + return Status::OK(); + } + +private: + size_t _block_position; + int32_t _value; + const std::string _name = "NullableIntGreaterThanExpr"; +}; + +VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { + auto context = VExprContext::create_shared(expr); + auto status = context->prepare(state, RowDescriptor()); + EXPECT_TRUE(status.ok()) << status; + status = context->open(state); + EXPECT_TRUE(status.ok()) << status; + return context; +} + +} // namespace + +TEST(NativeV2ReaderTest, SchemaProbeReplaysFirstBlockAndProjectsColumns) { + const auto path = "./log/native_v2_reader_" + UniqueId::gen_uid().to_string() + ".native"; + std::filesystem::create_directories("./log"); + ASSERT_TRUE(write_native_file(path, make_source_block()).ok()); + + RuntimeState state; + RuntimeProfile profile("native_v2_reader_test"); + auto reader = create_reader(path, &state, &profile); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(schema[0].name, "id"); + EXPECT_EQ(schema[0].local_id, 0); + EXPECT_EQ(schema[1].name, "name"); + EXPECT_EQ(schema[1].local_id, 1); + EXPECT_TRUE(schema[0].type->is_nullable()); + EXPECT_TRUE(schema[1].type->is_nullable()); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(1)).ok()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {1, 0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_FALSE(eof); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "alice"); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 1), "bob"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 0), 10); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 1), 20); + + block.clear_column_data(2); + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + ASSERT_TRUE(reader->close().ok()); + static_cast(io::global_local_filesystem()->delete_file(path)); +} + +TEST(NativeV2ReaderTest, AppliesConjunctsAndTracksPredicateFilteredRows) { + const auto path = + "./log/native_v2_reader_filter_" + UniqueId::gen_uid().to_string() + ".native"; + std::filesystem::create_directories("./log"); + ASSERT_TRUE(write_native_file(path, make_source_block()).ok()); + + RuntimeState state; + RuntimeProfile profile("native_v2_reader_filter_test"); + auto io_ctx = std::make_shared(); + auto reader = create_reader(path, &state, &profile, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(1)).ok()); + request->conjuncts = { + prepared_conjunct(&state, std::make_shared(0, 10))}; + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {0, 1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 20); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "bob"); + EXPECT_EQ(io_ctx->predicate_filtered_rows, 1); + ASSERT_TRUE(reader->close().ok()); + static_cast(io::global_local_filesystem()->delete_file(path)); +} + +TEST(NativeV2ReaderTest, RejectsInvalidHeaderAndEmptyFile) { + std::filesystem::create_directories("./log"); + RuntimeState state; + RuntimeProfile profile("native_v2_reader_bad_header_test"); + + const auto bad_magic_path = + "./log/native_v2_bad_magic_" + UniqueId::gen_uid().to_string() + ".native"; + std::string bad_magic(sizeof(DORIS_NATIVE_MAGIC) + sizeof(uint32_t), '\0'); + bad_magic.replace(0, 4, "BAD!"); + ASSERT_TRUE(write_file(bad_magic_path, bad_magic).ok()); + auto bad_magic_reader = create_reader(bad_magic_path, &state, &profile); + EXPECT_FALSE(bad_magic_reader->init(&state).ok()); + static_cast(io::global_local_filesystem()->delete_file(bad_magic_path)); + + const auto empty_path = "./log/native_v2_empty_" + UniqueId::gen_uid().to_string() + ".native"; + ASSERT_TRUE(write_file(empty_path, "").ok()); + auto empty_reader = create_reader(empty_path, &state, &profile); + EXPECT_FALSE(empty_reader->init(&state).ok()); + static_cast(io::global_local_filesystem()->delete_file(empty_path)); +} + +TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndHeaderOnlyFile) { + std::filesystem::create_directories("./log"); + RuntimeState state; + RuntimeProfile profile("native_v2_reader_header_boundary_test"); + + const auto bad_version_path = + "./log/native_v2_bad_version_" + UniqueId::gen_uid().to_string() + ".native"; + std::string bad_version; + bad_version.append(DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)); + uint8_t version_buffer[sizeof(uint32_t)]; + encode_fixed32_le(version_buffer, DORIS_NATIVE_FORMAT_VERSION + 1); + bad_version.append(reinterpret_cast(version_buffer), sizeof(version_buffer)); + ASSERT_TRUE(write_file(bad_version_path, bad_version).ok()); + auto bad_version_reader = create_reader(bad_version_path, &state, &profile); + EXPECT_FALSE(bad_version_reader->init(&state).ok()); + static_cast(io::global_local_filesystem()->delete_file(bad_version_path)); + + const auto header_only_path = + "./log/native_v2_header_only_" + UniqueId::gen_uid().to_string() + ".native"; + std::string header_only; + header_only.append(DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)); + encode_fixed32_le(version_buffer, DORIS_NATIVE_FORMAT_VERSION); + header_only.append(reinterpret_cast(version_buffer), sizeof(version_buffer)); + ASSERT_TRUE(write_file(header_only_path, header_only).ok()); + auto header_only_reader = create_reader(header_only_path, &state, &profile); + ASSERT_TRUE(header_only_reader->init(&state).ok()); + std::vector schema; + EXPECT_FALSE(header_only_reader->get_schema(&schema).ok()); + static_cast(io::global_local_filesystem()->delete_file(header_only_path)); +} + +TEST(NativeV2ReaderTest, RejectsTruncatedBlockDuringSchemaProbe) { + const auto path = "./log/native_v2_truncated_" + UniqueId::gen_uid().to_string() + ".native"; + std::filesystem::create_directories("./log"); + + std::string content; + content.append(DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)); + uint8_t version_buffer[sizeof(uint32_t)]; + encode_fixed32_le(version_buffer, DORIS_NATIVE_FORMAT_VERSION); + content.append(reinterpret_cast(version_buffer), sizeof(version_buffer)); + uint8_t len_buffer[sizeof(uint64_t)]; + encode_fixed64_le(len_buffer, 8); + content.append(reinterpret_cast(len_buffer), sizeof(len_buffer)); + content.append("x"); + ASSERT_TRUE(write_file(path, content).ok()); + + RuntimeState state; + RuntimeProfile profile("native_v2_reader_truncated_test"); + auto reader = create_reader(path, &state, &profile); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + EXPECT_FALSE(reader->get_schema(&schema).ok()); + static_cast(io::global_local_filesystem()->delete_file(path)); +} + +TEST(NativeV2ReaderTest, RejectsZeroLengthBlockAndInvalidPBlock) { + std::filesystem::create_directories("./log"); + RuntimeState state; + RuntimeProfile profile("native_v2_reader_bad_block_test"); + + auto build_header = [] { + std::string content; + content.append(DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)); + uint8_t version_buffer[sizeof(uint32_t)]; + encode_fixed32_le(version_buffer, DORIS_NATIVE_FORMAT_VERSION); + content.append(reinterpret_cast(version_buffer), sizeof(version_buffer)); + return content; + }; + + const auto zero_len_path = + "./log/native_v2_zero_len_" + UniqueId::gen_uid().to_string() + ".native"; + auto zero_len_content = build_header(); + uint8_t len_buffer[sizeof(uint64_t)]; + encode_fixed64_le(len_buffer, 0); + zero_len_content.append(reinterpret_cast(len_buffer), sizeof(len_buffer)); + ASSERT_TRUE(write_file(zero_len_path, zero_len_content).ok()); + auto zero_len_reader = create_reader(zero_len_path, &state, &profile); + ASSERT_TRUE(zero_len_reader->init(&state).ok()); + std::vector schema; + EXPECT_FALSE(zero_len_reader->get_schema(&schema).ok()); + static_cast(io::global_local_filesystem()->delete_file(zero_len_path)); + + const auto invalid_pblock_path = + "./log/native_v2_invalid_pblock_" + UniqueId::gen_uid().to_string() + ".native"; + auto invalid_pblock_content = build_header(); + encode_fixed64_le(len_buffer, 1); + invalid_pblock_content.append(reinterpret_cast(len_buffer), sizeof(len_buffer)); + invalid_pblock_content.append("x"); + ASSERT_TRUE(write_file(invalid_pblock_path, invalid_pblock_content).ok()); + auto invalid_pblock_reader = create_reader(invalid_pblock_path, &state, &profile); + ASSERT_TRUE(invalid_pblock_reader->init(&state).ok()); + schema.clear(); + EXPECT_FALSE(invalid_pblock_reader->get_schema(&schema).ok()); + static_cast(io::global_local_filesystem()->delete_file(invalid_pblock_path)); +} + +TEST(NativeV2ReaderTest, RejectsUnknownRequestedLocalColumn) { + const auto path = + "./log/native_v2_unknown_column_" + UniqueId::gen_uid().to_string() + ".native"; + std::filesystem::create_directories("./log"); + ASSERT_TRUE(write_native_file(path, make_source_block()).ok()); + + RuntimeState state; + RuntimeProfile profile("native_v2_reader_unknown_column_test"); + auto reader = create_reader(path, &state, &profile); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(42)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + Block block; + block.insert({schema[0].type->create_column(), schema[0].type, schema[0].name}); + size_t rows = 0; + bool eof = false; + EXPECT_FALSE(reader->get_block(&block, &rows, &eof).ok()); + static_cast(io::global_local_filesystem()->delete_file(path)); +} + +} // namespace doris::format::native diff --git a/be/test/format_v2/orc/orc_file_input_stream_test.cpp b/be/test/format_v2/orc/orc_file_input_stream_test.cpp new file mode 100644 index 00000000000000..54c89505edc3f5 --- /dev/null +++ b/be/test/format_v2/orc/orc_file_input_stream_test.cpp @@ -0,0 +1,343 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/orc/orc_file_input_stream.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "runtime/runtime_profile.h" + +namespace doris::format::orc { +namespace { + +struct TestStream { + uint64_t column_id; + ::orc::StreamKind kind; + uint64_t length; +}; + +class RecordingFileReader final : public io::FileReader { +public: + explicit RecordingFileReader(size_t size) : _data(size) { + for (size_t offset = 0; offset < size; ++offset) { + _data[offset] = static_cast(offset % 251); + } + } + + Status close() override { + _closed = true; + return Status::OK(); + } + const io::Path& path() const override { return _path; } + size_t size() const override { return _data.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 0; } + + const std::vector& reads() const { return _reads; } + char value_at(size_t offset) const { return _data[offset]; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext* io_ctx) override { + _reads.emplace_back(offset, std::min(offset + result.size, _data.size())); + *bytes_read = std::min(result.size, _data.size() - offset); + std::memcpy(result.data, _data.data() + offset, *bytes_read); + return Status::OK(); + } + +private: + std::vector _data; + std::vector _reads; + bool _closed = false; + io::Path _path = "/tmp/orc_v2_input_stream"; +}; + +class TestStreamInformation final : public ::orc::StreamInformation { +public: + TestStreamInformation(TestStream stream, uint64_t offset) : _stream(stream), _offset(offset) {} + + ::orc::StreamKind getKind() const override { return _stream.kind; } + uint64_t getColumnId() const override { return _stream.column_id; } + uint64_t getOffset() const override { return _offset; } + uint64_t getLength() const override { return _stream.length; } + +private: + TestStream _stream; + uint64_t _offset; +}; + +class TestStripeInformation final : public ::orc::StripeInformation { +public: + TestStripeInformation(uint64_t offset, std::vector streams) + : _offset(offset), _streams(std::move(streams)) {} + + uint64_t getOffset() const override { return _offset; } + uint64_t getLength() const override { + uint64_t length = 0; + for (const auto& stream : _streams) { + length += stream.length; + } + return length; + } + uint64_t getIndexLength() const override { return 0; } + uint64_t getDataLength() const override { return getLength(); } + uint64_t getFooterLength() const override { return 0; } + uint64_t getNumberOfRows() const override { return 1; } + uint64_t getNumberOfStreams() const override { return _streams.size(); } + + std::unique_ptr<::orc::StreamInformation> getStreamInformation( + uint64_t stream_id) const override { + uint64_t offset = _offset; + for (uint64_t index = 0; index < stream_id; ++index) { + offset += _streams[index].length; + } + return std::make_unique(_streams[stream_id], offset); + } + + ::orc::ColumnEncodingKind getColumnEncoding(uint64_t col_id) const override { + return ::orc::ColumnEncodingKind_DIRECT; + } + uint64_t getDictionarySize(uint64_t col_id) const override { return 0; } + const std::string& getWriterTimezone() const override { return _timezone; } + +private: + uint64_t _offset; + std::vector _streams; + std::string _timezone = "UTC"; +}; + +using StripeStreamMap = std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>; + +std::shared_ptr<::orc::InputStream> find_stream(const StripeStreamMap& streams, uint64_t column_id, + ::orc::StreamKind kind) { + auto it = streams.find(::orc::StreamId(column_id, kind)); + EXPECT_NE(it, streams.end()); + return it == streams.end() ? nullptr : it->second; +} + +std::vector selected_columns(std::initializer_list columns) { + std::vector selected(8, false); + for (uint64_t column : columns) { + selected[column] = true; + } + return selected; +} + +TEST(OrcFileInputStreamTest, MergesAdjacentSelectedStreams) { + auto reader = std::make_shared(256); + OrcFileInputStream input("test.orc", reader, nullptr, nullptr, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 0}); + StripeStreamMap streams; + input.beforeReadStripe(std::make_unique( + 100, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + + auto second = find_stream(streams, 2, ::orc::StreamKind_DATA); + ASSERT_NE(second, nullptr); + std::array second_data {}; + second->read(second_data.data(), second_data.size(), 104); + + auto first = find_stream(streams, 1, ::orc::StreamKind_DATA); + ASSERT_NE(first, nullptr); + std::array first_data {}; + first->read(first_data.data(), first_data.size(), 100); + + ASSERT_EQ(reader->reads().size(), 1); + EXPECT_EQ(reader->reads().front(), io::PrefetchRange(100, 108)); + EXPECT_EQ(second_data[0], reader->value_at(104)); + EXPECT_EQ(first_data[0], reader->value_at(100)); +} + +TEST(OrcFileInputStreamTest, SupportsRepeatedAndBackwardClusterReads) { + auto reader = std::make_shared(256); + OrcFileInputStream input("test.orc", reader, nullptr, nullptr, + {.once_max_read_bytes = 32, .max_merge_distance_bytes = 8}); + StripeStreamMap streams; + input.beforeReadStripe(std::make_unique( + 40, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {0, ::orc::StreamKind_DATA, 3}, + {2, ::orc::StreamKind_LENGTH, 5}}), + selected_columns({1, 2}), streams); + + auto later = find_stream(streams, 2, ::orc::StreamKind_LENGTH); + auto earlier = find_stream(streams, 1, ::orc::StreamKind_DATA); + ASSERT_NE(later, nullptr); + ASSERT_NE(earlier, nullptr); + + std::array later_data {}; + later->read(later_data.data(), later_data.size(), 47); + std::array repeated_data {}; + later->read(repeated_data.data(), repeated_data.size(), 48); + std::array earlier_data {}; + earlier->read(earlier_data.data(), earlier_data.size(), 40); + + ASSERT_EQ(reader->reads().size(), 1); + EXPECT_EQ(reader->reads().front(), io::PrefetchRange(40, 52)); + EXPECT_EQ(repeated_data[0], reader->value_at(48)); + EXPECT_EQ(earlier_data[0], reader->value_at(40)); +} + +TEST(OrcFileInputStreamTest, RespectsMergeDistanceAndMaximumSpan) { + auto distance_reader = std::make_shared(256); + OrcFileInputStream distance_input("test.orc", distance_reader, nullptr, nullptr, + {.once_max_read_bytes = 32, .max_merge_distance_bytes = 2}); + StripeStreamMap distance_streams; + distance_input.beforeReadStripe( + std::make_unique( + 0, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {0, ::orc::StreamKind_DATA, 3}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), distance_streams); + std::array buffer {}; + find_stream(distance_streams, 1, ::orc::StreamKind_DATA)->read(buffer.data(), buffer.size(), 0); + find_stream(distance_streams, 2, ::orc::StreamKind_DATA)->read(buffer.data(), buffer.size(), 7); + ASSERT_EQ(distance_reader->reads().size(), 2); + EXPECT_EQ(distance_reader->reads()[0], io::PrefetchRange(0, 4)); + EXPECT_EQ(distance_reader->reads()[1], io::PrefetchRange(7, 11)); + + auto span_reader = std::make_shared(256); + OrcFileInputStream span_input("test.orc", span_reader, nullptr, nullptr, + {.once_max_read_bytes = 8, .max_merge_distance_bytes = 0}); + StripeStreamMap span_streams; + span_input.beforeReadStripe( + std::make_unique( + 20, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 5}}), + selected_columns({1, 2}), span_streams); + find_stream(span_streams, 1, ::orc::StreamKind_DATA)->read(buffer.data(), buffer.size(), 20); + std::array second_buffer {}; + find_stream(span_streams, 2, ::orc::StreamKind_DATA) + ->read(second_buffer.data(), second_buffer.size(), 24); + ASSERT_EQ(span_reader->reads().size(), 2); + EXPECT_EQ(span_reader->reads()[0], io::PrefetchRange(20, 24)); + EXPECT_EQ(span_reader->reads()[1], io::PrefetchRange(24, 29)); +} + +TEST(OrcFileInputStreamTest, KeepsLargeAndSingleStreamsDirect) { + auto reader = std::make_shared(256); + OrcFileInputStream input("test.orc", reader, nullptr, nullptr, + {.once_max_read_bytes = 8, .max_merge_distance_bytes = 8}); + StripeStreamMap streams; + input.beforeReadStripe(std::make_unique( + 60, std::vector {{1, ::orc::StreamKind_DATA, 9}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + + std::array large_data {}; + find_stream(streams, 1, ::orc::StreamKind_DATA)->read(large_data.data(), large_data.size(), 62); + std::array single_data {}; + find_stream(streams, 2, ::orc::StreamKind_DATA) + ->read(single_data.data(), single_data.size(), 69); + + ASSERT_EQ(reader->reads().size(), 2); + EXPECT_EQ(reader->reads()[0], io::PrefetchRange(62, 65)); + EXPECT_EQ(reader->reads()[1], io::PrefetchRange(69, 73)); +} + +TEST(OrcFileInputStreamTest, ZeroOnceMaxReadBytesKeepsStreamsDirect) { + auto reader = std::make_shared(256); + OrcFileInputStream input("test.orc", reader, nullptr, nullptr, + {.once_max_read_bytes = 0, .max_merge_distance_bytes = 16}); + StripeStreamMap streams; + input.beforeReadStripe(std::make_unique( + 120, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + + std::array first_data {}; + find_stream(streams, 1, ::orc::StreamKind_DATA) + ->read(first_data.data(), first_data.size(), 120); + std::array second_data {}; + find_stream(streams, 2, ::orc::StreamKind_DATA) + ->read(second_data.data(), second_data.size(), 124); + + ASSERT_EQ(reader->reads().size(), 2); + EXPECT_EQ(reader->reads()[0], io::PrefetchRange(120, 124)); + EXPECT_EQ(reader->reads()[1], io::PrefetchRange(124, 128)); +} + +TEST(OrcFileInputStreamTest, SkipsUnselectedStreams) { + auto reader = std::make_shared(256); + OrcFileInputStream input("test.orc", reader, nullptr, nullptr, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 16}); + StripeStreamMap streams; + input.beforeReadStripe(std::make_unique( + 80, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({2}), streams); + + EXPECT_EQ(streams.size(), 1); + EXPECT_EQ(streams.count(::orc::StreamId(1, ::orc::StreamKind_DATA)), 0); + EXPECT_EQ(streams.count(::orc::StreamId(2, ::orc::StreamKind_DATA)), 1); +} + +TEST(OrcFileInputStreamTest, RejectsInvalidStreamColumnId) { + auto reader = std::make_shared(256); + OrcFileInputStream input("test.orc", reader, nullptr, nullptr, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 16}); + StripeStreamMap streams; + + EXPECT_THROW(input.beforeReadStripe( + std::make_unique( + 80, std::vector {{9, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams), + ::orc::ParseError); +} + +TEST(OrcFileInputStreamTest, PublishesClusterProfileExactlyOnce) { + auto reader = std::make_shared(256); + RuntimeProfile profile("orc_v2_input_stream"); + { + OrcFileInputStream input("test.orc", reader, nullptr, &profile, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 0}); + StripeStreamMap streams; + input.beforeReadStripe( + std::make_unique( + 100, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + + std::array data {}; + find_stream(streams, 2, ::orc::StreamKind_DATA)->read(data.data(), data.size(), 104); + find_stream(streams, 1, ::orc::StreamKind_DATA)->read(data.data(), data.size(), 100); + + StripeStreamMap next_streams; + input.beforeReadStripe( + std::make_unique(200, std::vector {}), + selected_columns({}), next_streams); + ASSERT_NE(profile.get_counter("RequestIO"), nullptr); + ASSERT_NE(profile.get_counter("MergedIO"), nullptr); + EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + } + + EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); +} + +} // namespace +} // namespace doris::format::orc diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp new file mode 100644 index 00000000000000..0b79239750d28b --- /dev/null +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -0,0 +1,10020 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/orc/orc_reader.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date.h" +#include "core/data_type/data_type_date_time.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type/primitive_type.h" +#include "core/value/timestamptz_value.h" +#include "exprs/create_predicate_function.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vruntimefilter_wrapper.h" +#include "exprs/vslot_ref.h" +#include "format/orc/orc_memory_stream_test.h" +#include "format_v2/expr/cast.h" +#include "format_v2/expr/delete_predicate.h" +#include "format_v2/file_reader.h" +#include "gen_cpp/Types_types.h" +#include "io/fs/buffered_reader.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/segment/condition_cache.h" +#include "storage/utils.h" +#include "util/timezone_utils.h" + +namespace doris { +namespace { + +std::filesystem::path unique_test_dir(std::string_view prefix) { + static std::atomic next_dir_id {0}; + std::string name(prefix); + if (const auto* test_info = testing::UnitTest::GetInstance()->current_test_info(); + test_info != nullptr) { + name += "_"; + name += test_info->test_suite_name(); + name += "_"; + name += test_info->name(); + } + name += "_"; + name += std::to_string(getpid()); + name += "_"; + name += std::to_string(next_dir_id.fetch_add(1)); + for (auto& ch : name) { + if (!std::isalnum(static_cast(ch)) && ch != '_' && ch != '-' && ch != '.') { + ch = '_'; + } + } + return std::filesystem::temp_directory_path() / name; +} + +std::filesystem::path find_repo_file(std::string_view relative_path) { + auto dir = std::filesystem::current_path(); + while (true) { + auto candidate = dir / relative_path; + if (std::filesystem::exists(candidate)) { + return candidate; + } + if (!dir.has_parent_path() || dir.parent_path() == dir) { + return candidate; + } + dir = dir.parent_path(); + } +} + +DateV2Value make_date_v2(uint16_t year, uint8_t month, uint8_t day) { + DateV2Value value; + value.unchecked_set_time(year, month, day, 0, 0, 0, 0); + return value; +} + +int64_t orc_date_offset(uint16_t year, uint8_t month, uint8_t day) { + static constexpr int32_t DATE_THRESHOLD = 719528; + return make_date_v2(year, month, day).daynr() - DATE_THRESHOLD; +} + +DateV2Value make_datetime_v2(uint16_t year, uint8_t month, uint8_t day, + uint8_t hour = 0, uint8_t minute = 0, + uint8_t second = 0, uint32_t microsecond = 0) { + DateV2Value value; + value.unchecked_set_time(year, month, day, hour, minute, second, microsecond); + return value; +} + +TEST(OrcStatisticsBoundsTest, RejectsInvertedAndNanDecodedBounds) { + EXPECT_TRUE(format::orc::detail::valid_statistics_bounds(Field::create_field(1), + Field::create_field(10))); + EXPECT_FALSE(format::orc::detail::valid_statistics_bounds(Field::create_field(10), + Field::create_field(1))); + EXPECT_FALSE(format::orc::detail::valid_statistics_bounds( + Field::create_field(std::numeric_limits::quiet_NaN()), + Field::create_field(10))); + EXPECT_FALSE(format::orc::detail::valid_statistics_bounds( + Field::create_field("z"), Field::create_field("a"))); + EXPECT_FALSE(format::orc::detail::valid_statistics_bounds( + Field::create_field(make_date_v2(2026, 7, 13)), + Field::create_field(make_date_v2(2026, 7, 12)))); + EXPECT_FALSE(format::orc::detail::valid_statistics_bounds( + Field::create_field(Decimal128V3(10)), + Field::create_field(Decimal128V3(1)))); +} + +class TableLiteral : public VLiteral { +public: + template + static std::shared_ptr create_shared(Args&&... args) { + return std::make_shared(std::forward(args)...); + } + + TableLiteral(const DataTypePtr& type, const Field& field) : VLiteral(type) { + _node_type = node_type_for_field(field); + _data_type = type; + _column_ptr = _data_type->create_column_const(1, field); + _expr_name = _data_type->get_name(); + } + +private: + static TExprNodeType::type node_type_for_field(const Field& field) { + switch (field.get_type()) { + case TYPE_BOOLEAN: + return TExprNodeType::BOOL_LITERAL; + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + return TExprNodeType::INT_LITERAL; + case TYPE_LARGEINT: + return TExprNodeType::LARGE_INT_LITERAL; + case TYPE_FLOAT: + case TYPE_DOUBLE: + return TExprNodeType::FLOAT_LITERAL; + case TYPE_DATE: + case TYPE_DATETIME: + case TYPE_DATEV2: + case TYPE_DATETIMEV2: + case TYPE_TIMESTAMPTZ: + return TExprNodeType::DATE_LITERAL; + case TYPE_DECIMALV2: + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMAL128I: + case TYPE_DECIMAL256: + return TExprNodeType::DECIMAL_LITERAL; + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + return TExprNodeType::STRING_LITERAL; + case TYPE_IPV4: + return TExprNodeType::IPV4_LITERAL; + case TYPE_IPV6: + return TExprNodeType::IPV6_LITERAL; + case TYPE_TIMEV2: + return TExprNodeType::TIMEV2_LITERAL; + case TYPE_VARBINARY: + return TExprNodeType::VARBINARY_LITERAL; + case TYPE_ARRAY: + return TExprNodeType::ARRAY_LITERAL; + case TYPE_MAP: + return TExprNodeType::MAP_LITERAL; + case TYPE_STRUCT: + return TExprNodeType::STRUCT_LITERAL; + case TYPE_NULL: + return TExprNodeType::NULL_LITERAL; + default: + return TExprNodeType::LITERAL_PRED; + } + } +}; + +class TableSlotRef : public VSlotRef { +public: + template + static std::shared_ptr create_shared(Args&&... args) { + return std::make_shared(std::forward(args)...); + } + + TableSlotRef(int slot_id, int column_id, int column_uniq_id, const DataTypePtr& type, + const std::string& column_name) + : VSlotRef(slot_id, column_id, column_uniq_id), _cname(column_name) { + _data_type = type; + } + + Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override { + if (_prepared) { + return Status::OK(); + } + _prepared = true; + _prepare_finished = true; + return Status::OK(); + } + + const std::string& expr_name() const override { return _cname; } + const std::string& column_name() const override { return _cname; } + +private: + const std::string _cname; +}; + +constexpr int64_t ROW_COUNT = 5; +constexpr int64_t PRIMITIVE_ROW_COUNT = 3; +constexpr int64_t COMPLEX_ROW_COUNT = 3; +constexpr int64_t DEEP_NESTED_ROW_COUNT = 4; +constexpr int64_t DEEP_NESTED_BATCH_CAPACITY = 16; +constexpr int64_t NULL_ROW = 1; +constexpr int64_t PREFETCH_ROW_COUNT = 256; + +VExprSPtr function_expr(const std::string& function_name, const DataTypePtr& return_type, + const std::vector& arg_types, + TExprNodeType::type node_type = TExprNodeType::FUNCTION_CALL, + TExprOpcode::type opcode = TExprOpcode::INVALID_OPCODE) { + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + std::vector thrift_arg_types; + thrift_arg_types.reserve(arg_types.size()); + for (const auto& arg_type : arg_types) { + thrift_arg_types.push_back(arg_type->to_thrift()); + } + fn.__set_arg_types(thrift_arg_types); + fn.__set_ret_type(return_type->to_thrift()); + fn.__set_has_var_args(false); + + TExprNode node; + node.__set_node_type(node_type); + node.__set_opcode(opcode); + node.__set_type(return_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(static_cast(arg_types.size())); + node.__set_is_nullable(return_type->is_nullable()); + return VectorizedFnCall::create_shared(node); +} + +TExprNode make_filter_in_node(TExprNodeType::type node_type) { + TExprNode node; + node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); + node.__set_node_type(node_type); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_num_children(1); + node.__set_is_nullable(false); + node.in_predicate.__set_is_not_in(false); + return node; +} + +class NullableInt32GreaterThanExpr final : public VExpr { +public: + NullableInt32GreaterThanExpr(int column_id, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int_type = std::make_shared(); + add_child(TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(int_type), + "id")); + add_child(TableLiteral::create_shared(int_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable_column.is_null_at(input_row) && input.get_element(input_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableInt32GreaterThanExpr"; +}; + +class NullableInt32LessThanExpr final : public VExpr { +public: + NullableInt32LessThanExpr(int column_id, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::LT; + const auto int_type = std::make_shared(); + add_child(TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(int_type), + "id")); + add_child(TableLiteral::create_shared(int_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable_column.is_null_at(input_row) && input.get_element(input_row) < _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableInt32LessThanExpr"; +}; + +class NullableInt32EqualsExpr final : public VExpr { +public: + NullableInt32EqualsExpr(int column_id, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableInt32EqualsExpr"; +}; + +class NullableInt32NullSafeEqualsNullExpr final : public VExpr { +public: + NullableInt32NullSafeEqualsNullExpr(int column_id, bool literal_on_left) + : VExpr(std::make_shared(), false), _column_id(column_id) { + _node_type = TExprNodeType::NULL_AWARE_BINARY_PRED; + _opcode = TExprOpcode::EQ_FOR_NULL; + const auto int32_type = std::make_shared(); + auto slot = TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(int32_type), + "id"); + auto null_literal = TableLiteral::create_shared(make_nullable(int32_type), + Field::create_field(Null())); + if (literal_on_left) { + add_child(std::move(null_literal)); + add_child(std::move(slot)); + } else { + add_child(std::move(slot)); + add_child(std::move(null_literal)); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = nullable_column.is_null_at(input_row); + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _expr_name = "NullableInt32NullSafeEqualsNullExpr"; +}; + +class NullableInt32NullSafeEqualsLiteralExpr final : public VExpr { +public: + NullableInt32NullSafeEqualsLiteralExpr(int column_id, int32_t value, bool literal_on_left) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::NULL_AWARE_BINARY_PRED; + _opcode = TExprOpcode::EQ_FOR_NULL; + const auto int32_type = std::make_shared(); + auto slot = TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(int32_type), + "id"); + auto literal = + TableLiteral::create_shared(int32_type, Field::create_field(value)); + if (literal_on_left) { + add_child(std::move(literal)); + add_child(std::move(slot)); + } else { + add_child(std::move(slot)); + add_child(std::move(literal)); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableInt32NullSafeEqualsLiteralExpr"; +}; + +class NullableInt32NullSafeEqualsSlotExpr final : public VExpr { +public: + NullableInt32NullSafeEqualsSlotExpr(int left_column_id, int right_column_id) + : VExpr(std::make_shared(), false), + _left_column_id(left_column_id), + _right_column_id(right_column_id) { + _node_type = TExprNodeType::NULL_AWARE_BINARY_PRED; + _opcode = TExprOpcode::EQ_FOR_NULL; + const auto int32_type = std::make_shared(); + add_child(TableSlotRef::create_shared(left_column_id, left_column_id, -1, + make_nullable(int32_type), "lhs")); + add_child(TableSlotRef::create_shared(right_column_id, right_column_id, -1, + make_nullable(int32_type), "rhs")); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& left_nullable = + assert_cast(*block->get_by_position(_left_column_id).column); + const auto& right_nullable = assert_cast( + *block->get_by_position(_right_column_id).column); + const auto& left = assert_cast(left_nullable.get_nested_column()); + const auto& right = assert_cast(right_nullable.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + const bool left_is_null = left_nullable.is_null_at(input_row); + const bool right_is_null = right_nullable.is_null_at(input_row); + result_data[row] = + left_is_null == right_is_null && + (left_is_null || left.get_element(input_row) == right.get_element(input_row)); + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _left_column_id; + const int _right_column_id; + const std::string _expr_name = "NullableInt32NullSafeEqualsSlotExpr"; +}; + +class NullableInt32IsNullExpr final : public VExpr { +public: + explicit NullableInt32IsNullExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) { + _node_type = TExprNodeType::FUNCTION_CALL; + _opcode = TExprOpcode::INVALID_OPCODE; + TFunctionName fn_name; + fn_name.__set_function_name("is_null_pred"); + _fn.__set_name(fn_name); + const auto int32_type = std::make_shared(); + add_child(TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(int32_type), + "id")); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = nullable_column.is_null_at(input_row); + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _expr_name = "NullableInt32IsNullExpr"; +}; + +class CompoundPredicateExpr final : public VExpr { +public: + CompoundPredicateExpr(TExprOpcode::type opcode, VExprSPtrs children) + : VExpr(std::make_shared(), false), + _expr_name(opcode == TExprOpcode::COMPOUND_OR ? "CompoundOrPredicateExpr" + : "CompoundNotPredicateExpr") { + _node_type = TExprNodeType::COMPOUND_PRED; + _opcode = opcode; + set_children(std::move(children)); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + if (_opcode == TExprOpcode::COMPOUND_NOT) { + DORIS_CHECK(_children.size() == 1); + ColumnPtr child_column; + RETURN_IF_ERROR(_children[0]->execute_column_impl(context, block, selector, count, + child_column)); + for (size_t row = 0; row < count; ++row) { + result_data[row] = !bool_value(*child_column, row); + } + result_column = std::move(result); + return Status::OK(); + } + + DORIS_CHECK(_opcode == TExprOpcode::COMPOUND_OR); + DORIS_CHECK(!_children.empty()); + std::vector child_columns; + child_columns.reserve(_children.size()); + for (const auto& child : _children) { + ColumnPtr child_column; + RETURN_IF_ERROR( + child->execute_column_impl(context, block, selector, count, child_column)); + child_columns.push_back(std::move(child_column)); + } + for (size_t row = 0; row < count; ++row) { + result_data[row] = 0; + for (const auto& child_column : child_columns) { + if (bool_value(*child_column, row)) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + static bool bool_value(const IColumn& column, size_t row) { + if (const auto* nullable = check_and_get_column(column)) { + if (nullable->is_null_at(row)) { + return false; + } + const auto& nested = assert_cast(nullable->get_nested_column()); + return nested.get_element(row) != 0; + } + const auto& data = assert_cast(column); + return data.get_element(row) != 0; + } + + std::string _expr_name; +}; + +class RuntimeFilterWrapperExpr final : public VExpr { +public: + explicit RuntimeFilterWrapperExpr(VExprSPtr impl) + : VExpr(std::make_shared(), false), + _impl(std::move(impl)), + _expr_name("RuntimeFilterWrapperExpr") { + DORIS_CHECK(_impl != nullptr); + _node_type = _impl->node_type(); + _opcode = _impl->op(); + } + + bool is_rf_wrapper() const override { return true; } + + VExprSPtr get_impl() const override { return _impl; } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + return _impl->execute_column_impl(context, block, selector, count, result_column); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + VExprSPtr _impl; + const std::string _expr_name; +}; + +class NullableInt32CastToInt64GreaterThanExpr final : public VExpr { +public: + NullableInt32CastToInt64GreaterThanExpr(int column_id, int64_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int32_type = std::make_shared(); + const auto int64_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(int64_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int32_type), "id")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(int64_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int64_t _value; + const std::string _expr_name = "NullableInt32CastToInt64GreaterThanExpr"; +}; + +class NullableInt32CastToInt64NullSafeEqualsNullExpr final : public VExpr { +public: + explicit NullableInt32CastToInt64NullSafeEqualsNullExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) { + _node_type = TExprNodeType::NULL_AWARE_BINARY_PRED; + _opcode = TExprOpcode::EQ_FOR_NULL; + const auto int32_type = std::make_shared(); + const auto int64_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(int64_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int32_type), "id")); + auto null_literal = TableLiteral::create_shared(make_nullable(int64_type), + Field::create_field(Null())); + add_child(std::move(cast_expr)); + add_child(std::move(null_literal)); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = nullable_column.is_null_at(input_row); + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _expr_name = "NullableInt32CastToInt64NullSafeEqualsNullExpr"; +}; + +class NullableInt32CastToInt64NullSafeEqualsLiteralExpr final : public VExpr { +public: + NullableInt32CastToInt64NullSafeEqualsLiteralExpr(int column_id, int64_t value, + bool literal_on_left) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::NULL_AWARE_BINARY_PRED; + _opcode = TExprOpcode::EQ_FOR_NULL; + const auto int32_type = std::make_shared(); + const auto int64_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(int64_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int32_type), "id")); + auto literal = + TableLiteral::create_shared(int64_type, Field::create_field(value)); + if (literal_on_left) { + add_child(std::move(literal)); + add_child(std::move(cast_expr)); + } else { + add_child(std::move(cast_expr)); + add_child(std::move(literal)); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int64_t _value; + const std::string _expr_name = "NullableInt32CastToInt64NullSafeEqualsLiteralExpr"; +}; + +class NullableInt32CastToDoubleGreaterThanExpr final : public VExpr { +public: + NullableInt32CastToDoubleGreaterThanExpr(int column_id, double value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int32_type = std::make_shared(); + const auto double_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(double_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int32_type), "id")); + add_child(std::move(cast_expr)); + add_child( + TableLiteral::create_shared(double_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const double _value; + const std::string _expr_name = "NullableInt32CastToDoubleGreaterThanExpr"; +}; + +class NullableInt32CastToFloatGreaterThanExpr final : public VExpr { +public: + NullableInt32CastToFloatGreaterThanExpr(int column_id, float value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int32_type = std::make_shared(); + const auto float_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(float_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int32_type), "id")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(float_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const float _value; + const std::string _expr_name = "NullableInt32CastToFloatGreaterThanExpr"; +}; + +class NullableInt64CastToDoubleGreaterThanExpr final : public VExpr { +public: + NullableInt64CastToDoubleGreaterThanExpr(int column_id, double value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int64_type = std::make_shared(); + const auto double_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(double_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int64_type), "id")); + add_child(std::move(cast_expr)); + add_child( + TableLiteral::create_shared(double_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const double _value; + const std::string _expr_name = "NullableInt64CastToDoubleGreaterThanExpr"; +}; + +class NullableInt32CastToDoubleLessThanExpr final : public VExpr { +public: + NullableInt32CastToDoubleLessThanExpr(int column_id, double value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::LT; + const auto int32_type = std::make_shared(); + const auto double_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(double_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int32_type), "id")); + add_child(std::move(cast_expr)); + add_child( + TableLiteral::create_shared(double_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) < _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const double _value; + const std::string _expr_name = "NullableInt32CastToDoubleLessThanExpr"; +}; + +class NullableInt32CastToDoubleInExpr final : public VExpr { +public: + NullableInt32CastToDoubleInExpr(int column_id, std::vector values) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _values(std::move(values)) { + _node_type = TExprNodeType::IN_PRED; + _opcode = TExprOpcode::FILTER_IN; + const auto int32_type = std::make_shared(); + const auto double_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(double_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int32_type), "id")); + add_child(std::move(cast_expr)); + for (const auto value : _values) { + add_child(TableLiteral::create_shared(double_type, + Field::create_field(value))); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (nullable_column.is_null_at(input_row)) { + continue; + } + const auto value = static_cast(input.get_element(input_row)); + for (const auto literal : _values) { + if (value == literal) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::vector _values; + const std::string _expr_name = "NullableInt32CastToDoubleInExpr"; +}; + +class NullableInt32CastToFloatInExpr final : public VExpr { +public: + NullableInt32CastToFloatInExpr(int column_id, std::vector values) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _values(std::move(values)) { + _node_type = TExprNodeType::IN_PRED; + _opcode = TExprOpcode::FILTER_IN; + const auto int32_type = std::make_shared(); + const auto float_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(float_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int32_type), "id")); + add_child(std::move(cast_expr)); + for (const auto value : _values) { + add_child(TableLiteral::create_shared(float_type, + Field::create_field(value))); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (nullable_column.is_null_at(input_row)) { + continue; + } + const auto value = static_cast(input.get_element(input_row)); + for (const auto literal : _values) { + if (value == literal) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::vector _values; + const std::string _expr_name = "NullableInt32CastToFloatInExpr"; +}; + +class NullableInt64CastToDoubleInExpr final : public VExpr { +public: + NullableInt64CastToDoubleInExpr(int column_id, std::vector values) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _values(std::move(values)) { + _node_type = TExprNodeType::IN_PRED; + _opcode = TExprOpcode::FILTER_IN; + const auto int64_type = std::make_shared(); + const auto double_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(double_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int64_type), "id")); + add_child(std::move(cast_expr)); + for (const auto value : _values) { + add_child(TableLiteral::create_shared(double_type, + Field::create_field(value))); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (nullable_column.is_null_at(input_row)) { + continue; + } + const auto value = static_cast(input.get_element(input_row)); + for (const auto literal : _values) { + if (value == literal) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::vector _values; + const std::string _expr_name = "NullableInt64CastToDoubleInExpr"; +}; + +class NullableInt64CastToInt32GreaterThanExpr final : public VExpr { +public: + NullableInt64CastToInt32GreaterThanExpr(int column_id, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int64_type = std::make_shared(); + const auto int32_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(int32_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int64_type), "id")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(int32_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableInt64CastToInt32GreaterThanExpr"; +}; + +class NullableInt64CastToDoubleNullSafeEqualsLiteralExpr final : public VExpr { +public: + NullableInt64CastToDoubleNullSafeEqualsLiteralExpr(int column_id, double value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::NULL_AWARE_BINARY_PRED; + _opcode = TExprOpcode::EQ_FOR_NULL; + const auto int64_type = std::make_shared(); + const auto double_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(double_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int64_type), "id")); + add_child(std::move(cast_expr)); + add_child( + TableLiteral::create_shared(double_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const double _value; + const std::string _expr_name = "NullableInt64CastToDoubleNullSafeEqualsLiteralExpr"; +}; + +class NullableInt16CastToFloatGreaterThanExpr final : public VExpr { +public: + NullableInt16CastToFloatGreaterThanExpr(int column_id, float value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int16_type = std::make_shared(); + const auto float_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(float_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(int16_type), "id")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(float_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const float _value; + const std::string _expr_name = "NullableInt16CastToFloatGreaterThanExpr"; +}; + +class NullableFloatCastToDoubleGreaterThanExpr final : public VExpr { +public: + NullableFloatCastToDoubleGreaterThanExpr(int column_id, double value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto float_type = std::make_shared(); + const auto double_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(double_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(float_type), "float_col")); + add_child(std::move(cast_expr)); + add_child( + TableLiteral::create_shared(double_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + static_cast(input.get_element(input_row)) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const double _value; + const std::string _expr_name = "NullableFloatCastToDoubleGreaterThanExpr"; +}; + +class NullableDateV2CastToDateGreaterThanExpr final : public VExpr { +public: + NullableDateV2CastToDateGreaterThanExpr(int column_id, DateV2Value file_value, + const VecDateTimeValue& literal) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _file_value(file_value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto date_v2_type = std::make_shared(); + const auto date_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(date_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(date_v2_type), "date_col")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(date_type, Field::create_field(literal))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) > _file_value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const DateV2Value _file_value; + const std::string _expr_name = "NullableDateV2CastToDateGreaterThanExpr"; +}; + +class NullableDateV2CastToDateTimeV2GreaterThanExpr final : public VExpr { +public: + NullableDateV2CastToDateTimeV2GreaterThanExpr(int column_id, + DateV2Value file_value, + DateV2Value literal) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _file_value(file_value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto date_v2_type = std::make_shared(); + const auto datetime_v2_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(datetime_v2_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(date_v2_type), "date_col")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(datetime_v2_type, + Field::create_field(literal))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) > _file_value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const DateV2Value _file_value; + const std::string _expr_name = "NullableDateV2CastToDateTimeV2GreaterThanExpr"; +}; + +class NullableDateV2CastToDateTimeV2ComparisonExpr final : public VExpr { +public: + NullableDateV2CastToDateTimeV2ComparisonExpr(int column_id, TExprOpcode::type opcode, + DateV2Value literal) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _literal(literal) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = opcode; + const auto date_v2_type = std::make_shared(); + const auto datetime_v2_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(datetime_v2_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(date_v2_type), "date_col")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(datetime_v2_type, + Field::create_field(literal))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + compare(date_to_midnight(input.get_element(input_row))); + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + DateV2Value date_to_midnight( + const DateV2Value& date) const { + return make_datetime_v2(date.year(), date.month(), date.day()); + } + + bool compare(const DateV2Value& value) const { + switch (_opcode) { + case TExprOpcode::GE: + return value >= _literal; + case TExprOpcode::GT: + return value > _literal; + case TExprOpcode::LE: + return value <= _literal; + case TExprOpcode::LT: + return value < _literal; + default: + return false; + } + } + + const int _column_id; + const DateV2Value _literal; + const std::string _expr_name = "NullableDateV2CastToDateTimeV2ComparisonExpr"; +}; + +class NullableDateV2CastToDateTimeV2InExpr final : public VExpr { +public: + NullableDateV2CastToDateTimeV2InExpr(int column_id, + std::vector> values) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _values(std::move(values)) { + _node_type = TExprNodeType::IN_PRED; + _opcode = TExprOpcode::FILTER_IN; + const auto date_v2_type = std::make_shared(); + const auto datetime_v2_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(datetime_v2_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(date_v2_type), "date_col")); + add_child(std::move(cast_expr)); + for (const auto& value : _values) { + add_child(TableLiteral::create_shared(datetime_v2_type, + Field::create_field(value))); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (nullable_column.is_null_at(input_row)) { + continue; + } + const auto value = date_to_midnight(input.get_element(input_row)); + for (const auto& literal : _values) { + if (value == literal) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + DateV2Value date_to_midnight( + const DateV2Value& date) const { + return make_datetime_v2(date.year(), date.month(), date.day()); + } + + const int _column_id; + const std::vector> _values; + const std::string _expr_name = "NullableDateV2CastToDateTimeV2InExpr"; +}; + +class NullableDateV2CastToDateTimeGreaterThanExpr final : public VExpr { +public: + NullableDateV2CastToDateTimeGreaterThanExpr(int column_id, + DateV2Value file_value, + const VecDateTimeValue& literal) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _file_value(file_value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto date_v2_type = std::make_shared(); + const auto datetime_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(make_nullable(datetime_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(date_v2_type), "date_col")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(datetime_type, + Field::create_field(literal))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) > _file_value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const DateV2Value _file_value; + const std::string _expr_name = "NullableDateV2CastToDateTimeGreaterThanExpr"; +}; + +class NullableStringCastToStringGreaterThanExpr final : public VExpr { +public: + NullableStringCastToStringGreaterThanExpr(int column_id, DataTypePtr source_type, + std::string value, std::string column_name) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(std::move(value)) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto string_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(string_type); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(std::move(source_type)), + std::move(column_name))); + add_child(std::move(cast_expr)); + add_child( + TableLiteral::create_shared(string_type, Field::create_field(_value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_data_at(input_row).to_string_view().compare(_value) > 0; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _value; + const std::string _expr_name = "NullableStringCastToStringGreaterThanExpr"; +}; + +class NullableStringEqualsExpr final : public VExpr { +public: + NullableStringEqualsExpr(int column_id, DataTypePtr source_type, std::string value, + std::string column_name) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(std::move(value)) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::EQ; + const auto string_type = std::make_shared(); + add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(std::move(source_type)), + std::move(column_name))); + add_child( + TableLiteral::create_shared(string_type, Field::create_field(_value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_data_at(input_row).to_string_view().compare(_value) == 0; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _value; + const std::string _expr_name = "NullableStringEqualsExpr"; +}; + +class SargOnlyStringEqualsVarbinaryLiteralExpr final : public VExpr { +public: + SargOnlyStringEqualsVarbinaryLiteralExpr(int column_id, DataTypePtr source_type, + std::string value, std::string column_name) + : VExpr(std::make_shared(), false), _value(std::move(value)) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::EQ; + add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(std::move(source_type)), + std::move(column_name))); + add_child(TableLiteral::create_shared( + std::make_shared(), + Field::create_field(StringView(_value)))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + result_data[row] = 1; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const std::string _value; + const std::string _expr_name = "SargOnlyStringEqualsVarbinaryLiteralExpr"; +}; + +class SargOnlyStringCastToStringEqualsExpr final : public VExpr { +public: + SargOnlyStringCastToStringEqualsExpr(int column_id, DataTypePtr source_type, std::string value, + std::string column_name) + : VExpr(std::make_shared(), false), _value(std::move(value)) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::EQ; + const auto string_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(string_type); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(std::move(source_type)), + std::move(column_name))); + add_child(std::move(cast_expr)); + add_child( + TableLiteral::create_shared(string_type, Field::create_field(_value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + result_data[row] = 1; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const std::string _value; + const std::string _expr_name = "SargOnlyStringCastToStringEqualsExpr"; +}; + +class NullableDateV2CastToStringGreaterThanExpr final : public VExpr { +public: + NullableDateV2CastToStringGreaterThanExpr(int column_id, std::string value, + DateV2Value file_value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _file_value(file_value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto date_type = std::make_shared(); + const auto string_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(string_type); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(date_type), "date_col")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(string_type, + Field::create_field(std::move(value)))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) > _file_value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const DateV2Value _file_value; + const std::string _expr_name = "NullableDateV2CastToStringGreaterThanExpr"; +}; + +class NullableDateTimeV2LowerPrecisionCastGreaterThanExpr final : public VExpr { +public: + NullableDateTimeV2LowerPrecisionCastGreaterThanExpr(int column_id, DataTypePtr source_type, + DateV2Value literal) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _literal(literal) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto lower_precision_type = std::make_shared(0); + auto cast_expr = format::Cast::create_shared(make_nullable(lower_precision_type)); + cast_expr->add_child(TableSlotRef::create_shared( + column_id, column_id, -1, make_nullable(std::move(source_type)), "timestamp_col")); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(lower_precision_type, + Field::create_field(literal))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = + assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) > _literal; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const DateV2Value _literal; + const std::string _expr_name = "NullableDateTimeV2LowerPrecisionCastGreaterThanExpr"; +}; + +class NullableStructChildInt32GreaterThanExpr final : public VExpr { +public: + NullableStructChildInt32GreaterThanExpr(int column_id, DataTypePtr struct_type, + std::string child_name, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int_type = std::make_shared(); + auto child_expr = function_expr("struct_element", make_nullable(int_type), + {struct_type, std::make_shared()}); + child_expr->add_child( + TableSlotRef::create_shared(column_id, column_id, -1, struct_type, "struct_col")); + child_expr->add_child(TableLiteral::create_shared( + std::make_shared(), + Field::create_field(std::move(child_name)))); + add_child(std::move(child_expr)); + add_child(TableLiteral::create_shared(int_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_struct = + assert_cast(*block->get_by_position(_column_id).column); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& nullable_child = + assert_cast(struct_column.get_column(0)); + const auto& child = assert_cast(nullable_child.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_struct.is_null_at(input_row) && + !nullable_child.is_null_at(input_row) && + child.get_element(input_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableStructChildInt32GreaterThanExpr"; +}; + +class NullableStructChildInt32CastToInt64GreaterThanExpr final : public VExpr { +public: + NullableStructChildInt32CastToInt64GreaterThanExpr(int column_id, DataTypePtr struct_type, + std::string child_name, int64_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int_type = std::make_shared(); + const auto int64_type = std::make_shared(); + auto child_expr = function_expr("struct_element", make_nullable(int_type), + {struct_type, std::make_shared()}); + child_expr->add_child( + TableSlotRef::create_shared(column_id, column_id, -1, struct_type, "struct_col")); + child_expr->add_child(TableLiteral::create_shared( + std::make_shared(), + Field::create_field(std::move(child_name)))); + auto cast_expr = format::Cast::create_shared(make_nullable(int64_type)); + cast_expr->add_child(std::move(child_expr)); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(int64_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_struct = + assert_cast(*block->get_by_position(_column_id).column); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& nullable_child = + assert_cast(struct_column.get_column(0)); + const auto& child = assert_cast(nullable_child.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_struct.is_null_at(input_row) && + !nullable_child.is_null_at(input_row) && + static_cast(child.get_element(input_row)) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int64_t _value; + const std::string _expr_name = "NullableStructChildInt32CastToInt64GreaterThanExpr"; +}; + +class NullableStructChildInt32NullSafeEqualsLiteralExpr final : public VExpr { +public: + NullableStructChildInt32NullSafeEqualsLiteralExpr(int column_id, DataTypePtr struct_type, + std::string child_name, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::NULL_AWARE_BINARY_PRED; + _opcode = TExprOpcode::EQ_FOR_NULL; + const auto int_type = std::make_shared(); + auto child_expr = function_expr("struct_element", make_nullable(int_type), + {struct_type, std::make_shared()}); + child_expr->add_child( + TableSlotRef::create_shared(column_id, column_id, -1, struct_type, "struct_col")); + child_expr->add_child(TableLiteral::create_shared( + std::make_shared(), + Field::create_field(std::move(child_name)))); + add_child(std::move(child_expr)); + add_child(TableLiteral::create_shared(int_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_struct = + assert_cast(*block->get_by_position(_column_id).column); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& nullable_child = + assert_cast(struct_column.get_column(0)); + const auto& child = assert_cast(nullable_child.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_struct.is_null_at(input_row) && + !nullable_child.is_null_at(input_row) && + child.get_element(input_row) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableStructChildInt32NullSafeEqualsLiteralExpr"; +}; + +class NullableStructChildArrayNullSafeEqualsLiteralExpr final : public VExpr { +public: + NullableStructChildArrayNullSafeEqualsLiteralExpr(int column_id, DataTypePtr struct_type, + DataTypePtr array_type, + std::string child_name, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::NULL_AWARE_BINARY_PRED; + _opcode = TExprOpcode::EQ_FOR_NULL; + const auto string_type = std::make_shared(); + auto child_expr = function_expr("struct_element", array_type, {struct_type, string_type}); + child_expr->add_child( + TableSlotRef::create_shared(column_id, column_id, -1, struct_type, "struct_col")); + child_expr->add_child(TableLiteral::create_shared( + string_type, Field::create_field(std::move(child_name)))); + + Array literal_values {Field::create_field(value)}; + add_child(std::move(child_expr)); + add_child(TableLiteral::create_shared( + std::move(array_type), Field::create_field(std::move(literal_values)))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_struct = + assert_cast(*block->get_by_position(_column_id).column); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& nullable_array = + assert_cast(struct_column.get_column(0)); + const auto& array_column = + assert_cast(nullable_array.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + const auto& values_nullable = assert_cast(array_column.get_data()); + const auto& values = assert_cast(values_nullable.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (nullable_struct.is_null_at(input_row) || nullable_array.is_null_at(input_row)) { + continue; + } + const auto begin = input_row == 0 ? 0 : offsets[input_row - 1]; + const auto end = offsets[input_row]; + result_data[row] = end == begin + 1 && !values_nullable.is_null_at(begin) && + values.get_element(begin) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableStructChildArrayNullSafeEqualsLiteralExpr"; +}; + +class NullableNestedStructChildInt32GreaterThanExpr final : public VExpr { +public: + NullableNestedStructChildInt32GreaterThanExpr(int column_id, DataTypePtr struct_type, + DataTypePtr nested_struct_type, + std::string nested_child_name, + std::string child_name, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int_type = std::make_shared(); + const auto string_type = std::make_shared(); + auto nested_expr = + function_expr("struct_element", nested_struct_type, {struct_type, string_type}); + nested_expr->add_child( + TableSlotRef::create_shared(column_id, column_id, -1, struct_type, "struct_col")); + nested_expr->add_child(TableLiteral::create_shared( + string_type, Field::create_field(std::move(nested_child_name)))); + auto child_expr = function_expr("struct_element", make_nullable(int_type), + {nested_struct_type, std::make_shared()}); + child_expr->add_child(std::move(nested_expr)); + child_expr->add_child(TableLiteral::create_shared( + std::make_shared(), + Field::create_field(std::move(child_name)))); + add_child(std::move(child_expr)); + add_child(TableLiteral::create_shared(int_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_struct = + assert_cast(*block->get_by_position(_column_id).column); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& nullable_nested = + assert_cast(struct_column.get_column(0)); + const auto& nested_struct = + assert_cast(nullable_nested.get_nested_column()); + const auto& nullable_child = + assert_cast(nested_struct.get_column(0)); + const auto& child = assert_cast(nullable_child.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_struct.is_null_at(input_row) && + !nullable_nested.is_null_at(input_row) && + !nullable_child.is_null_at(input_row) && + child.get_element(input_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableNestedStructChildInt32GreaterThanExpr"; +}; + +class NullableArrayContainsInt32Expr final : public VExpr { +public: + NullableArrayContainsInt32Expr(int column_id, DataTypePtr array_type, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::FUNCTION_CALL; + const auto int32_type = std::make_shared(); + add_child(TableSlotRef::create_shared(column_id, column_id, -1, std::move(array_type), + "array_col")); + add_child(TableLiteral::create_shared(int32_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& array_nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& array_column = + assert_cast(array_nullable.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + const auto& values_nullable = assert_cast(array_column.get_data()); + const auto& values = assert_cast(values_nullable.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (array_nullable.is_null_at(input_row)) { + continue; + } + const auto begin = input_row == 0 ? 0 : offsets[input_row - 1]; + const auto end = offsets[input_row]; + for (auto element = begin; element < end; ++element) { + if (!values_nullable.is_null_at(element) && values.get_element(element) == _value) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableArrayContainsInt32Expr"; +}; + +class NullableMapContainsKeyExpr final : public VExpr { +public: + NullableMapContainsKeyExpr(int column_id, DataTypePtr map_type, std::string key) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _key(std::move(key)) { + _node_type = TExprNodeType::FUNCTION_CALL; + const auto string_type = std::make_shared(); + add_child(TableSlotRef::create_shared(column_id, column_id, -1, std::move(map_type), + "map_col")); + add_child(TableLiteral::create_shared(string_type, Field::create_field(_key))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& map_nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + const auto& keys_nullable = assert_cast(map_column.get_keys()); + const auto& keys = assert_cast(keys_nullable.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (map_nullable.is_null_at(input_row)) { + continue; + } + const auto begin = input_row == 0 ? 0 : offsets[input_row - 1]; + const auto end = offsets[input_row]; + for (auto entry = begin; entry < end; ++entry) { + if (!keys_nullable.is_null_at(entry) && + keys.get_data_at(entry).to_string_view() == _key) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _key; + const std::string _expr_name = "NullableMapContainsKeyExpr"; +}; + +class NullableMapElementInt32GreaterThanExpr final : public VExpr { +public: + NullableMapElementInt32GreaterThanExpr(int column_id, DataTypePtr map_type, std::string key, + int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _key(std::move(key)), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int32_type = std::make_shared(); + const auto string_type = std::make_shared(); + auto element_expr = + function_expr("element_at", make_nullable(int32_type), {map_type, string_type}); + element_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + std::move(map_type), "map_col")); + element_expr->add_child( + TableLiteral::create_shared(string_type, Field::create_field(_key))); + add_child(std::move(element_expr)); + add_child(TableLiteral::create_shared(int32_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& map_nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + const auto& keys_nullable = assert_cast(map_column.get_keys()); + const auto& keys = assert_cast(keys_nullable.get_nested_column()); + const auto& values_nullable = assert_cast(map_column.get_values()); + const auto& values = assert_cast(values_nullable.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (map_nullable.is_null_at(input_row)) { + continue; + } + const auto begin = input_row == 0 ? 0 : offsets[input_row - 1]; + const auto end = offsets[input_row]; + for (auto entry = begin; entry < end; ++entry) { + if (!keys_nullable.is_null_at(entry) && + keys.get_data_at(entry).to_string_view() == _key && + !values_nullable.is_null_at(entry) && values.get_element(entry) > _value) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _key; + const int32_t _value; + const std::string _expr_name = "NullableMapElementInt32GreaterThanExpr"; +}; + +class NullableMapKeysInExpr final : public VExpr { +public: + NullableMapKeysInExpr(int column_id, DataTypePtr map_type, std::string key) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _key(std::move(key)) { + _node_type = TExprNodeType::IN_PRED; + _opcode = TExprOpcode::FILTER_IN; + const auto string_type = std::make_shared(); + auto keys_expr = function_expr("map_keys", + std::make_shared(make_nullable(string_type)), + {map_type}); + keys_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + std::move(map_type), "map_col")); + add_child(std::move(keys_expr)); + add_child(TableLiteral::create_shared(string_type, Field::create_field(_key))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& map_nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + const auto& keys_nullable = assert_cast(map_column.get_keys()); + const auto& keys = assert_cast(keys_nullable.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = 0; + if (map_nullable.is_null_at(input_row)) { + continue; + } + const auto begin = input_row == 0 ? 0 : offsets[input_row - 1]; + const auto end = offsets[input_row]; + for (auto entry = begin; entry < end; ++entry) { + if (!keys_nullable.is_null_at(entry) && + keys.get_data_at(entry).to_string_view() == _key) { + result_data[row] = 1; + break; + } + } + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _key; + const std::string _expr_name = "NullableMapKeysInExpr"; +}; + +class NullableArraySizeGreaterThanExpr final : public VExpr { +public: + NullableArraySizeGreaterThanExpr(int column_id, DataTypePtr array_type, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int32_type = std::make_shared(); + auto size_expr = function_expr("size", int32_type, {array_type}); + size_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + std::move(array_type), "array_col")); + add_child(std::move(size_expr)); + add_child(TableLiteral::create_shared(int32_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& array_nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& array_column = + assert_cast(array_nullable.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + const auto begin = input_row == 0 ? 0 : offsets[input_row - 1]; + const auto end = offsets[input_row]; + result_data[row] = !array_nullable.is_null_at(input_row) && + cast_set(end - begin) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableArraySizeGreaterThanExpr"; +}; + +class NullableArrayStructChildInt32GreaterThanExpr final : public VExpr { +public: + NullableArrayStructChildInt32GreaterThanExpr(int column_id, DataTypePtr array_type, + DataTypePtr struct_type, std::string child_name, + int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto int32_type = std::make_shared(); + const auto string_type = std::make_shared(); + auto element_expr = function_expr("element_at", struct_type, {array_type, int32_type}); + element_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + std::move(array_type), "array_col")); + element_expr->add_child( + TableLiteral::create_shared(int32_type, Field::create_field(1))); + auto child_expr = function_expr("struct_element", make_nullable(int32_type), + {struct_type, string_type}); + child_expr->add_child(std::move(element_expr)); + child_expr->add_child(TableLiteral::create_shared( + string_type, Field::create_field(std::move(child_name)))); + add_child(std::move(child_expr)); + add_child(TableLiteral::create_shared(int32_type, Field::create_field(value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& array_nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& array_column = + assert_cast(array_nullable.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + const auto& values_nullable = assert_cast(array_column.get_data()); + const auto& struct_values = + assert_cast(values_nullable.get_nested_column()); + const auto& child_nullable = + assert_cast(struct_values.get_column(0)); + const auto& child = assert_cast(child_nullable.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + const auto begin = input_row == 0 ? 0 : offsets[input_row - 1]; + const auto end = offsets[input_row]; + result_data[row] = 0; + if (array_nullable.is_null_at(input_row) || begin == end || + values_nullable.is_null_at(begin) || child_nullable.is_null_at(begin)) { + continue; + } + result_data[row] = child.get_element(begin) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "NullableArrayStructChildInt32GreaterThanExpr"; +}; + +template +class NullableGreaterThanExpr final : public VExpr { +public: + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + using ValueType = typename PrimitiveTypeTraits::CppType; + + NullableGreaterThanExpr(int column_id, DataTypePtr type, const Field& value, + std::string column_name, bool literal_on_left = false) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value.get()), + _expr_name("NullableGreaterThanExpr") { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = literal_on_left ? TExprOpcode::LT : TExprOpcode::GT; + auto slot = TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(type), + column_name); + auto literal = TableLiteral::create_shared(std::move(type), value); + if (literal_on_left) { + add_child(literal); + add_child(slot); + } else { + add_child(slot); + add_child(literal); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable_column.is_null_at(input_row) && input.get_element(input_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const ValueType _value; + const std::string _expr_name; +}; + +template +class NullableEqualsExpr final : public VExpr { +public: + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + using ValueType = typename PrimitiveTypeTraits::CppType; + + NullableEqualsExpr(int column_id, DataTypePtr type, const Field& value, std::string column_name) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value.get()), + _expr_name("NullableEqualsExpr") { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::EQ; + add_child(TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(type), + std::move(column_name))); + add_child(TableLiteral::create_shared(std::move(type), value)); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const ValueType _value; + const std::string _expr_name; +}; + +template +class NullableInExpr final : public VExpr { +public: + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + using ValueType = typename PrimitiveTypeTraits::CppType; + + NullableInExpr(int column_id, DataTypePtr type, std::vector values, + std::string column_name, bool not_in = false) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _not_in(not_in) { + _node_type = TExprNodeType::IN_PRED; + _opcode = not_in ? TExprOpcode::FILTER_NOT_IN : TExprOpcode::FILTER_IN; + add_child(TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(type), + column_name)); + _values.reserve(values.size()); + for (const auto& value : values) { + _values.push_back(value.get()); + add_child(TableLiteral::create_shared(type, value)); + } + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + if (nullable_column.is_null_at(input_row)) { + result_data[row] = 0; + continue; + } + const auto value = input.get_element(input_row); + const auto contains = std::find(_values.begin(), _values.end(), value) != _values.end(); + result_data[row] = _not_in ? !contains : contains; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const bool _not_in; + std::vector _values; + const std::string _expr_name = "NullableInExpr"; +}; + +class NullableDecimalCastToStringGreaterThanExpr final : public VExpr { +public: + NullableDecimalCastToStringGreaterThanExpr(int column_id, DataTypePtr slot_type, + std::string value, std::string column_name, + uint32_t scale) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(std::move(value)), + _scale(scale) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + const auto string_type = std::make_shared(); + auto cast_expr = format::Cast::create_shared(string_type); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(std::move(slot_type)), + std::move(column_name))); + add_child(std::move(cast_expr)); + add_child( + TableLiteral::create_shared(string_type, Field::create_field(_value))); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = + assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row).to_string(_scale).compare(_value) > 0; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const std::string _value; + const uint32_t _scale; + const std::string _expr_name = "NullableDecimalCastToStringGreaterThanExpr"; +}; + +class NullableDecimalGreaterThanExpr final : public VExpr { +public: + NullableDecimalGreaterThanExpr(int column_id, DataTypePtr slot_type, DataTypePtr literal_type, + const Field& literal, Decimal128V3 file_scale_value, + std::string column_name) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _file_scale_value(file_scale_value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + add_child(TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(slot_type), + column_name)); + add_child(TableLiteral::create_shared(std::move(literal_type), literal)); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = + assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) > _file_scale_value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const Decimal128V3 _file_scale_value; + const std::string _expr_name = "NullableDecimalGreaterThanExpr"; +}; + +class NullableDecimalCastGreaterThanExpr final : public VExpr { +public: + NullableDecimalCastGreaterThanExpr(int column_id, DataTypePtr slot_type, DataTypePtr cast_type, + const Field& literal, Decimal128V3 file_scale_value, + std::string column_name) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _file_scale_value(file_scale_value) { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::GT; + auto cast_expr = format::Cast::create_shared(make_nullable(cast_type)); + cast_expr->add_child(TableSlotRef::create_shared(column_id, column_id, -1, + make_nullable(std::move(slot_type)), + std::move(column_name))); + add_child(std::move(cast_expr)); + add_child(TableLiteral::create_shared(std::move(cast_type), literal)); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = + assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) > _file_scale_value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const Decimal128V3 _file_scale_value; + const std::string _expr_name = "NullableDecimalCastGreaterThanExpr"; +}; + +void set_nullable_batch(::orc::ColumnVectorBatch& batch) { + batch.hasNulls = true; + batch.numElements = PRIMITIVE_ROW_COUNT; + for (int64_t row = 0; row < PRIMITIVE_ROW_COUNT; ++row) { + batch.notNull[row] = 1; + } + batch.notNull[NULL_ROW] = 0; +} + +void set_string_value(::orc::StringVectorBatch& batch, int64_t row, std::string_view value) { + static constexpr uint64_t MAX_TEST_STRING_BYTES_PER_ROW = 4096; + // Reserve once per batch so previously assigned data pointers remain stable. + const auto reserve_bytes = std::max(batch.capacity * MAX_TEST_STRING_BYTES_PER_ROW, + MAX_TEST_STRING_BYTES_PER_ROW); + batch.blob.reserve(reserve_bytes); + const auto offset = batch.blob.size(); + ASSERT_LE(offset + value.size(), reserve_bytes); + batch.blob.resize(offset + value.size()); + auto* stored_value = batch.blob.data() + offset; + if (!value.empty()) { + memcpy(stored_value, value.data(), value.size()); + } + batch.data[row] = stored_value; + batch.length[row] = static_cast(value.size()); +} + +template +void expect_offsets(const Offsets& offsets, std::initializer_list expected) { + ASSERT_EQ(offsets.size(), expected.size()); + size_t idx = 0; + for (const auto value : expected) { + EXPECT_EQ(offsets[idx++], value); + } +} + +void expect_string_values(const ColumnString& column, + std::initializer_list expected) { + ASSERT_EQ(column.size(), expected.size()); + size_t idx = 0; + for (const auto value : expected) { + EXPECT_EQ(column.get_data_at(idx++).to_string(), value); + } +} + +void expect_int32_values(const ColumnInt32& column, std::initializer_list expected) { + ASSERT_EQ(column.size(), expected.size()); + size_t idx = 0; + for (const auto value : expected) { + EXPECT_EQ(column.get_element(idx++), value); + } +} + +void expect_deep_nested_column(const IColumn& column, std::initializer_list expected_nulls, + std::initializer_list expected_array_offsets, + std::initializer_list expected_names, + std::initializer_list expected_map_offsets, + std::initializer_list expected_keys, + std::initializer_list expected_value_array_offsets, + std::initializer_list expected_values, + std::initializer_list expected_labels) { + const auto& deep_nullable = assert_cast(column); + ASSERT_EQ(deep_nullable.size(), expected_nulls.size()); + size_t row = 0; + for (const auto expected_null : expected_nulls) { + EXPECT_EQ(deep_nullable.is_null_at(row++), expected_null); + } + + const auto& deep_array = assert_cast(deep_nullable.get_nested_column()); + expect_offsets(deep_array.get_offsets(), expected_array_offsets); + + const auto& element_struct = assert_cast( + assert_cast(deep_array.get_data()).get_nested_column()); + ASSERT_EQ(element_struct.tuple_size(), 2); + const auto& names = assert_cast( + assert_cast(element_struct.get_column(0)).get_nested_column()); + expect_string_values(names, expected_names); + + const auto& nested_map = assert_cast( + assert_cast(element_struct.get_column(1)).get_nested_column()); + expect_offsets(nested_map.get_offsets(), expected_map_offsets); + const auto& keys = assert_cast( + assert_cast(nested_map.get_keys()).get_nested_column()); + expect_string_values(keys, expected_keys); + + const auto& value_array = assert_cast( + assert_cast(nested_map.get_values()).get_nested_column()); + expect_offsets(value_array.get_offsets(), expected_value_array_offsets); + const auto& leaf_struct = assert_cast( + assert_cast(value_array.get_data()).get_nested_column()); + ASSERT_EQ(leaf_struct.tuple_size(), 2); + const auto& values = assert_cast( + assert_cast(leaf_struct.get_column(0)).get_nested_column()); + expect_int32_values(values, expected_values); + const auto& labels = assert_cast( + assert_cast(leaf_struct.get_column(1)).get_nested_column()); + expect_string_values(labels, expected_labels); +} + +void write_orc_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(ROW_COUNT); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& value_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + + std::vector values {"one", "two", "three", "four", "five"}; + for (int64_t row = 0; row < ROW_COUNT; ++row) { + id_batch.data[row] = row + 1; + set_string_value(value_batch, row, values[row]); + } + struct_batch.numElements = ROW_COUNT; + id_batch.numElements = ROW_COUNT; + value_batch.numElements = ROW_COUNT; + + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_orc_prefetch_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setDictionaryKeySizeThreshold(0); + options.setStripeSize(64 * 1024 * 1024); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(PREFETCH_ROW_COUNT); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& a_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& b_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[1]); + auto& c_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[2]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[3]); + + std::vector payloads; + payloads.reserve(PREFETCH_ROW_COUNT); + for (int64_t row = 0; row < PREFETCH_ROW_COUNT; ++row) { + a_batch.data[row] = row; + b_batch.data[row] = row * 2; + c_batch.data[row] = row * 3; + payloads.push_back("payload-" + std::to_string(row)); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = PREFETCH_ROW_COUNT; + a_batch.numElements = PREFETCH_ROW_COUNT; + b_batch.numElements = PREFETCH_ROW_COUNT; + c_batch.numElements = PREFETCH_ROW_COUNT; + payload_batch.numElements = PREFETCH_ROW_COUNT; + + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +bool has_mergeable_orc_stream_cluster(const std::string& file_path, + int64_t max_merge_distance_bytes, + int64_t once_max_read_bytes) { + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + const auto file_size = in.tellg(); + in.seekg(0); + std::vector data(static_cast(file_size)); + in.read(data.data(), file_size); + + ::orc::ReaderOptions options; + options.setMemoryPool(*::orc::getDefaultPool()); + auto input_stream = std::make_unique(data.data(), data.size()); + auto reader = ::orc::createReader(std::move(input_stream), options); + if (reader->getNumberOfStripes() != 1) { + return false; + } + + auto stripe = reader->getStripe(0); + std::vector small_ranges; + for (uint64_t stream_id = 0; stream_id < stripe->getNumberOfStreams(); ++stream_id) { + const auto stream = stripe->getStreamInformation(stream_id); + if (stream->getLength() > 0 && + stream->getLength() <= static_cast(once_max_read_bytes)) { + small_ranges.emplace_back(stream->getOffset(), + stream->getOffset() + stream->getLength()); + } + } + if (small_ranges.size() < 2) { + return false; + } + + const auto merged_ranges = io::PrefetchRange::merge_adjacent_seq_ranges( + small_ranges, max_merge_distance_bytes, once_max_read_bytes); + for (const auto& merged_range : merged_ranges) { + size_t member_count = 0; + for (const auto& range : small_ranges) { + if (range.start_offset >= merged_range.start_offset && + range.end_offset <= merged_range.end_offset) { + ++member_count; + } + } + if (member_count > 1) { + return true; + } + } + return false; +} + +void write_primitive_orc_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setTimezoneName("UTC"); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(PRIMITIVE_ROW_COUNT); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + + auto& bool_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& byte_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[1]); + auto& short_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[2]); + auto& int_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[3]); + auto& long_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[4]); + auto& float_batch = dynamic_cast<::orc::DoubleVectorBatch&>(*struct_batch.fields[5]); + auto& double_batch = dynamic_cast<::orc::DoubleVectorBatch&>(*struct_batch.fields[6]); + auto& string_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[7]); + auto& binary_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[8]); + auto& varchar_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[9]); + auto& char_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[10]); + auto& date_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[11]); + auto& timestamp_batch = dynamic_cast<::orc::TimestampVectorBatch&>(*struct_batch.fields[12]); + auto& timestamp_instant_batch = + dynamic_cast<::orc::TimestampVectorBatch&>(*struct_batch.fields[13]); + auto& decimal64_batch = dynamic_cast<::orc::Decimal64VectorBatch&>(*struct_batch.fields[14]); + auto& decimal128_batch = dynamic_cast<::orc::Decimal128VectorBatch&>(*struct_batch.fields[15]); + + std::array<::orc::ColumnVectorBatch*, 16> fields { + &bool_batch, &byte_batch, + &short_batch, &int_batch, + &long_batch, &float_batch, + &double_batch, &string_batch, + &binary_batch, &varchar_batch, + &char_batch, &date_batch, + ×tamp_batch, ×tamp_instant_batch, + &decimal64_batch, &decimal128_batch, + }; + for (auto* field : fields) { + set_nullable_batch(*field); + } + + bool_batch.data[0] = 1; + bool_batch.data[2] = 0; + byte_batch.data[0] = -7; + byte_batch.data[2] = 8; + short_batch.data[0] = -300; + short_batch.data[2] = 301; + int_batch.data[0] = -70000; + int_batch.data[2] = 70001; + long_batch.data[0] = -9000000000L; + long_batch.data[2] = 9000000001L; + float_batch.data[0] = 1.25; + float_batch.data[2] = -2.5; + double_batch.data[0] = 10.5; + double_batch.data[2] = -20.25; + + static constexpr std::array string_values {"alpha", "", + "gamma"}; + static constexpr std::array binary_values {"bin_a", "", + "bin_c"}; + static constexpr std::array varchar_values {"varchar", + "", "tail"}; + static constexpr std::array char_values {"ab ", "", + "xy "}; + set_string_value(string_batch, 0, string_values[0]); + set_string_value(string_batch, 2, string_values[2]); + set_string_value(binary_batch, 0, binary_values[0]); + set_string_value(binary_batch, 2, binary_values[2]); + set_string_value(varchar_batch, 0, varchar_values[0]); + set_string_value(varchar_batch, 2, varchar_values[2]); + set_string_value(char_batch, 0, char_values[0]); + set_string_value(char_batch, 2, char_values[2]); + + date_batch.data[0] = 0; + date_batch.data[2] = 18628; + timestamp_batch.data[0] = 86401; + timestamp_batch.nanoseconds[0] = 234567000; + timestamp_batch.data[2] = 1609459200; + timestamp_batch.nanoseconds[2] = 123456000; + timestamp_instant_batch.data[0] = 2; + timestamp_instant_batch.nanoseconds[0] = 345678000; + timestamp_instant_batch.data[2] = 1609459201; + timestamp_instant_batch.nanoseconds[2] = 654321000; + decimal64_batch.values[0] = 12345; + decimal64_batch.values[2] = -6700; + decimal128_batch.values[0] = ::orc::Int128("123456789012345678"); + decimal128_batch.values[2] = ::orc::Int128("-987654321000000"); + + struct_batch.numElements = PRIMITIVE_ROW_COUNT; + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void fill_complex_struct_column(::orc::StructVectorBatch& struct_col) { + auto& struct_a = dynamic_cast<::orc::LongVectorBatch&>(*struct_col.fields[0]); + auto& struct_b = dynamic_cast<::orc::StringVectorBatch&>(*struct_col.fields[1]); + std::array struct_a_values {10, 20, 30}; + static constexpr std::array struct_b_values { + "ten", "twenty", "thirty"}; + for (int64_t row = 0; row < COMPLEX_ROW_COUNT; ++row) { + struct_a.data[row] = struct_a_values[row]; + set_string_value(struct_b, row, struct_b_values[row]); + } + struct_col.numElements = COMPLEX_ROW_COUNT; + struct_a.numElements = COMPLEX_ROW_COUNT; + struct_b.numElements = COMPLEX_ROW_COUNT; +} + +void fill_complex_array_column(::orc::ListVectorBatch& array_col) { + auto& array_values = dynamic_cast<::orc::LongVectorBatch&>(*array_col.elements); + std::array array_offsets {0, 1, 1, 3}; + std::array array_data {1, 2, 3}; + for (int64_t row = 0; row <= COMPLEX_ROW_COUNT; ++row) { + array_col.offsets[row] = array_offsets[row]; + } + for (size_t idx = 0; idx < array_data.size(); ++idx) { + array_values.data[idx] = array_data[idx]; + } + array_col.numElements = COMPLEX_ROW_COUNT; + array_values.numElements = array_data.size(); +} + +void fill_complex_map_column(::orc::MapVectorBatch& map_col) { + auto& map_keys = dynamic_cast<::orc::StringVectorBatch&>(*map_col.keys); + auto& map_values = dynamic_cast<::orc::LongVectorBatch&>(*map_col.elements); + std::array map_offsets {0, 2, 2, 3}; + static constexpr std::array map_key_values {"a", "b", "c"}; + std::array map_data {100, 200, 300}; + for (int64_t row = 0; row <= COMPLEX_ROW_COUNT; ++row) { + map_col.offsets[row] = map_offsets[row]; + } + for (size_t idx = 0; idx < map_key_values.size(); ++idx) { + set_string_value(map_keys, static_cast(idx), map_key_values[idx]); + map_values.data[idx] = map_data[idx]; + } + map_col.numElements = COMPLEX_ROW_COUNT; + map_keys.numElements = map_key_values.size(); + map_values.numElements = map_data.size(); +} + +void fill_complex_array_struct_column(::orc::ListVectorBatch& array_struct_col) { + auto& array_struct_values = dynamic_cast<::orc::StructVectorBatch&>(*array_struct_col.elements); + auto& array_struct_a = dynamic_cast<::orc::LongVectorBatch&>(*array_struct_values.fields[0]); + auto& array_struct_b = dynamic_cast<::orc::StringVectorBatch&>(*array_struct_values.fields[1]); + std::array array_struct_offsets {0, 1, 3, 3}; + std::array array_struct_a_values {1, 2, 3}; + static constexpr std::array array_struct_b_values {"one", "two", "three"}; + for (int64_t row = 0; row <= COMPLEX_ROW_COUNT; ++row) { + array_struct_col.offsets[row] = array_struct_offsets[row]; + } + for (size_t idx = 0; idx < array_struct_a_values.size(); ++idx) { + array_struct_a.data[idx] = array_struct_a_values[idx]; + set_string_value(array_struct_b, static_cast(idx), array_struct_b_values[idx]); + } + array_struct_col.numElements = COMPLEX_ROW_COUNT; + array_struct_values.numElements = array_struct_a_values.size(); + array_struct_a.numElements = array_struct_a_values.size(); + array_struct_b.numElements = array_struct_b_values.size(); +} + +void fill_complex_map_struct_column(::orc::MapVectorBatch& map_struct_col) { + auto& map_struct_keys = dynamic_cast<::orc::StringVectorBatch&>(*map_struct_col.keys); + auto& map_struct_values = dynamic_cast<::orc::StructVectorBatch&>(*map_struct_col.elements); + auto& map_struct_a = dynamic_cast<::orc::LongVectorBatch&>(*map_struct_values.fields[0]); + auto& map_struct_b = dynamic_cast<::orc::StringVectorBatch&>(*map_struct_values.fields[1]); + std::array map_struct_offsets {0, 1, 1, 2}; + static constexpr std::array map_struct_key_values {"first", "second"}; + std::array map_struct_a_values {7, 8}; + static constexpr std::array map_struct_b_values {"seven", "eight"}; + for (int64_t row = 0; row <= COMPLEX_ROW_COUNT; ++row) { + map_struct_col.offsets[row] = map_struct_offsets[row]; + } + for (size_t idx = 0; idx < map_struct_key_values.size(); ++idx) { + set_string_value(map_struct_keys, static_cast(idx), map_struct_key_values[idx]); + map_struct_a.data[idx] = map_struct_a_values[idx]; + set_string_value(map_struct_b, static_cast(idx), map_struct_b_values[idx]); + } + map_struct_col.numElements = COMPLEX_ROW_COUNT; + map_struct_keys.numElements = map_struct_key_values.size(); + map_struct_values.numElements = map_struct_a_values.size(); + map_struct_a.numElements = map_struct_a_values.size(); + map_struct_b.numElements = map_struct_b_values.size(); +} + +void fill_nullable_struct_column(::orc::StructVectorBatch& struct_col) { + auto& struct_a = dynamic_cast<::orc::LongVectorBatch&>(*struct_col.fields[0]); + auto& struct_b = dynamic_cast<::orc::StringVectorBatch&>(*struct_col.fields[1]); + set_nullable_batch(struct_col); + std::array struct_a_values {10, 20, 30}; + static constexpr std::array struct_b_values { + "ten", "ignored", ""}; + for (int64_t row = 0; row < COMPLEX_ROW_COUNT; ++row) { + struct_a.data[row] = struct_a_values[row]; + set_string_value(struct_b, row, struct_b_values[row]); + } + struct_b.hasNulls = true; + struct_b.notNull[0] = 1; + struct_b.notNull[1] = 1; + struct_b.notNull[2] = 0; + struct_a.numElements = COMPLEX_ROW_COUNT; + struct_b.numElements = COMPLEX_ROW_COUNT; +} + +void fill_nullable_array_struct_column(::orc::ListVectorBatch& array_struct_col) { + auto& array_struct_values = dynamic_cast<::orc::StructVectorBatch&>(*array_struct_col.elements); + auto& array_struct_a = dynamic_cast<::orc::LongVectorBatch&>(*array_struct_values.fields[0]); + auto& array_struct_b = dynamic_cast<::orc::StringVectorBatch&>(*array_struct_values.fields[1]); + std::array array_struct_offsets {0, 1, 1, 2}; + std::array array_struct_a_values {11, 22}; + static constexpr std::array array_struct_b_values {"eleven", "twenty_two"}; + set_nullable_batch(array_struct_col); + for (int64_t row = 0; row <= COMPLEX_ROW_COUNT; ++row) { + array_struct_col.offsets[row] = array_struct_offsets[row]; + } + for (size_t idx = 0; idx < array_struct_a_values.size(); ++idx) { + array_struct_a.data[idx] = array_struct_a_values[idx]; + set_string_value(array_struct_b, static_cast(idx), array_struct_b_values[idx]); + } + array_struct_values.numElements = array_struct_a_values.size(); + array_struct_a.numElements = array_struct_a_values.size(); + array_struct_b.numElements = array_struct_b_values.size(); +} + +void fill_nullable_map_struct_column(::orc::MapVectorBatch& map_struct_col) { + auto& map_struct_keys = dynamic_cast<::orc::StringVectorBatch&>(*map_struct_col.keys); + auto& map_struct_values = dynamic_cast<::orc::StructVectorBatch&>(*map_struct_col.elements); + auto& map_struct_a = dynamic_cast<::orc::LongVectorBatch&>(*map_struct_values.fields[0]); + auto& map_struct_b = dynamic_cast<::orc::StringVectorBatch&>(*map_struct_values.fields[1]); + std::array map_struct_offsets {0, 1, 1, 2}; + static constexpr std::array map_struct_key_values {"left", "right"}; + std::array map_struct_a_values {101, 202}; + static constexpr std::array map_struct_b_values {"one_zero_one", + "two_zero_two"}; + set_nullable_batch(map_struct_col); + for (int64_t row = 0; row <= COMPLEX_ROW_COUNT; ++row) { + map_struct_col.offsets[row] = map_struct_offsets[row]; + } + for (size_t idx = 0; idx < map_struct_key_values.size(); ++idx) { + set_string_value(map_struct_keys, static_cast(idx), map_struct_key_values[idx]); + map_struct_a.data[idx] = map_struct_a_values[idx]; + set_string_value(map_struct_b, static_cast(idx), map_struct_b_values[idx]); + } + map_struct_keys.numElements = map_struct_key_values.size(); + map_struct_values.numElements = map_struct_a_values.size(); + map_struct_a.numElements = map_struct_a_values.size(); + map_struct_b.numElements = map_struct_b_values.size(); +} + +void write_complex_orc_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct,array_col:array," + "map_col:map,array_struct_col:array>," + "map_struct_col:map>>")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(COMPLEX_ROW_COUNT); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + + fill_complex_struct_column(dynamic_cast<::orc::StructVectorBatch&>(*struct_batch.fields[0])); + fill_complex_array_column(dynamic_cast<::orc::ListVectorBatch&>(*struct_batch.fields[1])); + fill_complex_map_column(dynamic_cast<::orc::MapVectorBatch&>(*struct_batch.fields[2])); + fill_complex_array_struct_column( + dynamic_cast<::orc::ListVectorBatch&>(*struct_batch.fields[3])); + fill_complex_map_struct_column(dynamic_cast<::orc::MapVectorBatch&>(*struct_batch.fields[4])); + + struct_batch.numElements = COMPLEX_ROW_COUNT; + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_map_decimal_date_orc_file(const std::string& file_path) { + constexpr size_t ROWS = 4; + constexpr int64_t HIVE_012_1900_DAY_OFFSET = -719530; + constexpr int64_t YEAR_0000_12_29_DAY_OFFSET = -719165; + constexpr int64_t YEAR_1000_10_16_DAY_OFFSET = -353997; + + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct>")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_LZ4); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(ROWS); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& map_batch = dynamic_cast<::orc::MapVectorBatch&>(*struct_batch.fields[1]); + auto& key_batch = dynamic_cast<::orc::Decimal64VectorBatch&>(*map_batch.keys); + auto& value_batch = dynamic_cast<::orc::LongVectorBatch&>(*map_batch.elements); + + id_batch.data[0] = 1; + id_batch.data[1] = 2; + id_batch.data[2] = 3; + id_batch.data[3] = 4; + map_batch.offsets[0] = 0; + map_batch.offsets[1] = 1; + map_batch.offsets[2] = 2; + map_batch.offsets[3] = 3; + map_batch.offsets[4] = 4; + key_batch.values[0] = -9999999999L; + key_batch.values[1] = 9999999999L; + key_batch.values[2] = 0; + key_batch.values[3] = 1; + value_batch.data[0] = HIVE_012_1900_DAY_OFFSET; + value_batch.data[1] = orc_date_offset(9999, 12, 31); + value_batch.data[2] = YEAR_0000_12_29_DAY_OFFSET; + value_batch.data[3] = YEAR_1000_10_16_DAY_OFFSET; + + struct_batch.numElements = ROWS; + id_batch.numElements = ROWS; + map_batch.numElements = ROWS; + key_batch.numElements = ROWS; + value_batch.numElements = ROWS; + + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_two_stripe_orc_array_map_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct,map_col:map,payload:string>")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int32_t array_value, std::string_view map_key, int32_t map_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& array_batch = dynamic_cast<::orc::ListVectorBatch&>(*struct_batch.fields[0]); + auto& array_values = dynamic_cast<::orc::LongVectorBatch&>(*array_batch.elements); + auto& map_batch = dynamic_cast<::orc::MapVectorBatch&>(*struct_batch.fields[1]); + auto& map_keys = dynamic_cast<::orc::StringVectorBatch&>(*map_batch.keys); + auto& map_values = dynamic_cast<::orc::LongVectorBatch&>(*map_batch.elements); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[2]); + std::vector keys; + std::vector payloads; + keys.reserve(ROWS_PER_STRIPE); + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row <= ROWS_PER_STRIPE; ++row) { + array_batch.offsets[row] = row; + map_batch.offsets[row] = row; + } + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + array_values.data[row] = array_value; + keys.emplace_back(map_key); + set_string_value(map_keys, row, keys.back()); + map_values.data[row] = map_value; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + array_batch.numElements = ROWS_PER_STRIPE; + array_values.numElements = ROWS_PER_STRIPE; + map_batch.numElements = ROWS_PER_STRIPE; + map_keys.numElements = ROWS_PER_STRIPE; + map_values.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(1, "a", 100); + add_batch(2, "c", 300); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_two_stripe_orc_array_size_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct,payload:string>")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](bool has_element) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& array_batch = dynamic_cast<::orc::ListVectorBatch&>(*struct_batch.fields[0]); + auto& array_values = dynamic_cast<::orc::LongVectorBatch&>(*array_batch.elements); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row <= ROWS_PER_STRIPE; ++row) { + array_batch.offsets[row] = has_element ? row : 0; + } + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + if (has_element) { + array_values.data[row] = 7; + } + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + array_batch.numElements = ROWS_PER_STRIPE; + array_values.numElements = has_element ? ROWS_PER_STRIPE : 0; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(false); + add_batch(true); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_two_stripe_orc_array_struct_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>,payload:string>")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t child_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& array_batch = dynamic_cast<::orc::ListVectorBatch&>(*struct_batch.fields[0]); + auto& array_values = dynamic_cast<::orc::StructVectorBatch&>(*array_batch.elements); + auto& child_a = dynamic_cast<::orc::LongVectorBatch&>(*array_values.fields[0]); + auto& child_b = dynamic_cast<::orc::StringVectorBatch&>(*array_values.fields[1]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector child_values; + std::vector payloads; + child_values.reserve(ROWS_PER_STRIPE); + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row <= ROWS_PER_STRIPE; ++row) { + array_batch.offsets[row] = row; + } + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + child_a.data[row] = child_value; + child_values.push_back("child-" + std::to_string(child_value)); + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(child_b, row, child_values.back()); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + array_batch.numElements = ROWS_PER_STRIPE; + array_values.numElements = ROWS_PER_STRIPE; + child_a.numElements = ROWS_PER_STRIPE; + child_b.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(1); + add_batch(1000); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_nullable_complex_orc_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct," + "nullable_array_struct_col:array>," + "nullable_map_struct_col:map>>")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(COMPLEX_ROW_COUNT); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + + fill_nullable_struct_column(dynamic_cast<::orc::StructVectorBatch&>(*struct_batch.fields[0])); + fill_nullable_array_struct_column( + dynamic_cast<::orc::ListVectorBatch&>(*struct_batch.fields[1])); + fill_nullable_map_struct_column(dynamic_cast<::orc::MapVectorBatch&>(*struct_batch.fields[2])); + + struct_batch.numElements = COMPLEX_ROW_COUNT; + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void fill_deep_nested_complex_column(::orc::ListVectorBatch& deep_col) { + auto& element_struct = dynamic_cast<::orc::StructVectorBatch&>(*deep_col.elements); + auto& name_batch = dynamic_cast<::orc::StringVectorBatch&>(*element_struct.fields[0]); + auto& nested_map = dynamic_cast<::orc::MapVectorBatch&>(*element_struct.fields[1]); + auto& map_keys = dynamic_cast<::orc::StringVectorBatch&>(*nested_map.keys); + auto& value_array = dynamic_cast<::orc::ListVectorBatch&>(*nested_map.elements); + auto& leaf_struct = dynamic_cast<::orc::StructVectorBatch&>(*value_array.elements); + auto& value_batch = dynamic_cast<::orc::LongVectorBatch&>(*leaf_struct.fields[0]); + auto& label_batch = dynamic_cast<::orc::StringVectorBatch&>(*leaf_struct.fields[1]); + + deep_col.hasNulls = true; + for (int64_t row = 0; row < DEEP_NESTED_ROW_COUNT; ++row) { + deep_col.notNull[row] = 1; + } + deep_col.notNull[1] = 0; + + std::array array_offsets {0, 1, 1, 3, 4}; + static constexpr std::array names {"row1", "row3_left", "row3_right", + "row4"}; + for (int64_t row = 0; row <= DEEP_NESTED_ROW_COUNT; ++row) { + deep_col.offsets[row] = array_offsets[row]; + } + for (size_t idx = 0; idx < names.size(); ++idx) { + set_string_value(name_batch, static_cast(idx), names[idx]); + } + + std::array map_offsets {0, 1, 2, 4, 5}; + static constexpr std::array keys { + "row1_key", "row3_left_key", "row3_right_a", "row3_right_empty", "row4_key"}; + for (size_t idx = 0; idx < map_offsets.size(); ++idx) { + nested_map.offsets[idx] = map_offsets[idx]; + } + for (size_t idx = 0; idx < keys.size(); ++idx) { + set_string_value(map_keys, static_cast(idx), keys[idx]); + } + + std::array value_array_offsets {0, 2, 3, 5, 5, 6}; + std::array values {10, 11, 30, 31, 32, 40}; + static constexpr std::array labels {"ten", "eleven", "thirty", + "thirty_one", "thirty_two", "forty"}; + for (size_t idx = 0; idx < value_array_offsets.size(); ++idx) { + value_array.offsets[idx] = value_array_offsets[idx]; + } + for (size_t idx = 0; idx < values.size(); ++idx) { + value_batch.data[idx] = values[idx]; + set_string_value(label_batch, static_cast(idx), labels[idx]); + } + + deep_col.numElements = DEEP_NESTED_ROW_COUNT; + element_struct.numElements = names.size(); + name_batch.numElements = names.size(); + nested_map.numElements = names.size(); + map_keys.numElements = keys.size(); + value_array.numElements = keys.size(); + leaf_struct.numElements = values.size(); + value_batch.numElements = values.size(); + label_batch.numElements = labels.size(); +} + +void write_deep_nested_complex_orc_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>>>>>")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(DEEP_NESTED_BATCH_CAPACITY); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + + for (int64_t row = 0; row < DEEP_NESTED_ROW_COUNT; ++row) { + id_batch.data[row] = row + 1; + } + id_batch.numElements = DEEP_NESTED_ROW_COUNT; + fill_deep_nested_complex_column(dynamic_cast<::orc::ListVectorBatch&>(*struct_batch.fields[1])); + + struct_batch.numElements = DEEP_NESTED_ROW_COUNT; + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_int_file(const std::string& file_path, + std::vector first_values = {1, 1000}) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + id_batch.data[row] = first_value + row; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + id_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + for (const auto first_value : first_values) { + add_batch(first_value); + } + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_int_only_file(const std::string& file_path, + const std::vector& first_values, + int64_t rows_per_stripe = 200) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_value) { + auto batch = writer->createRowBatch(rows_per_stripe); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + for (int64_t row = 0; row < rows_per_stripe; ++row) { + id_batch.data[row] = first_value + row; + } + struct_batch.numElements = rows_per_stripe; + id_batch.numElements = rows_per_stripe; + writer->add(*batch); + }; + + for (const auto first_value : first_values) { + add_batch(first_value); + } + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_large_orc_int_file(const std::string& file_path, int64_t row_count) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto batch = writer->createRowBatch(row_count); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + for (int64_t row = 0; row < row_count; ++row) { + id_batch.data[row] = row + 1; + } + struct_batch.numElements = row_count; + id_batch.numElements = row_count; + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_sarg_skipped_row_group_orc_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(64 * 1024 * 1024); + options.setRowIndexStride(ConditionCacheContext::GRANULE_SIZE); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + constexpr int64_t ROWS_PER_GROUP = ConditionCacheContext::GRANULE_SIZE; + constexpr int64_t ROW_GROUPS = 3; + auto batch = writer->createRowBatch(ROWS_PER_GROUP * ROW_GROUPS); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_GROUP * ROW_GROUPS); + for (int64_t group = 0; group < ROW_GROUPS; ++group) { + for (int64_t row = 0; row < ROWS_PER_GROUP; ++row) { + const auto batch_row = group * ROWS_PER_GROUP + row; + id_batch.data[batch_row] = group == 1 ? 10000 + row : row + 1; + payloads.push_back("payload_" + std::to_string(batch_row)); + set_string_value(payload_batch, batch_row, payloads.back()); + } + } + struct_batch.numElements = ROWS_PER_GROUP * ROW_GROUPS; + id_batch.numElements = ROWS_PER_GROUP * ROW_GROUPS; + payload_batch.numElements = ROWS_PER_GROUP * ROW_GROUPS; + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_pair_int_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& lhs_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& rhs_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[1]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[2]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + lhs_batch.data[row] = first_value + row; + rhs_batch.data[row] = first_value + row; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + lhs_batch.numElements = ROWS_PER_STRIPE; + rhs_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(1); + add_batch(1000); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_two_stripe_orc_nullable_int_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](bool null_ids) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + id_batch.hasNulls = null_ids; + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + id_batch.notNull[row] = null_ids ? 0 : 1; + id_batch.data[row] = null_ids ? 0 : row + 1; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + id_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(false); + add_batch(true); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_short_file(const std::string& file_path, + std::vector first_values = {1, 1000}) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + id_batch.data[row] = first_value + row; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + id_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + for (const auto first_value : first_values) { + add_batch(first_value); + } + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_long_file(const std::string& file_path, + std::vector first_values = {1, 1000}) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + id_batch.data[row] = first_value + row; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + id_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + for (const auto first_value : first_values) { + add_batch(first_value); + } + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_varchar_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](std::string_view prefix) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& value_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector values; + std::vector payloads; + values.reserve(ROWS_PER_STRIPE); + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + values.push_back(std::string(prefix) + "_" + std::to_string(1000 + row)); + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(value_batch, row, values.back()); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + value_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch("aaa"); + add_batch("zzz"); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_binary_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setRowIndexStride(50); + options.setDictionaryKeySizeThreshold(0); + options.setColumnsUseBloomFilter(std::set {1}); + options.setBloomFilterFPP(0.000001); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](std::string_view prefix) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& value_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector values; + std::vector payloads; + values.reserve(ROWS_PER_STRIPE); + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + values.push_back(std::string(prefix) + "_" + std::to_string(1000 + row)); + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(value_batch, row, values.back()); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + value_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch("aaa"); + add_batch("zzz"); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_char_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](std::string_view prefix) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& value_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector values; + std::vector payloads; + values.reserve(ROWS_PER_STRIPE); + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + values.push_back(std::string(prefix) + "_" + std::to_string(1000 + row)); + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(value_batch, row, values.back()); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + value_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch("aaa"); + add_batch("zzz"); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_struct_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct,payload:string>")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& nested_struct = dynamic_cast<::orc::StructVectorBatch&>(*struct_batch.fields[0]); + auto& child_a = dynamic_cast<::orc::LongVectorBatch&>(*nested_struct.fields[0]); + auto& child_b = dynamic_cast<::orc::StringVectorBatch&>(*nested_struct.fields[1]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector child_values; + std::vector payloads; + child_values.reserve(ROWS_PER_STRIPE); + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + child_a.data[row] = first_value + row; + child_values.push_back("child-" + std::to_string(first_value + row)); + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(child_b, row, child_values.back()); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + nested_struct.numElements = ROWS_PER_STRIPE; + child_a.numElements = ROWS_PER_STRIPE; + child_b.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(1); + add_batch(1000); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_nested_struct_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct,b:string>,payload:string>")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& struct_col = dynamic_cast<::orc::StructVectorBatch&>(*struct_batch.fields[0]); + auto& nested_struct = dynamic_cast<::orc::StructVectorBatch&>(*struct_col.fields[0]); + auto& child_a = dynamic_cast<::orc::LongVectorBatch&>(*nested_struct.fields[0]); + auto& child_b = dynamic_cast<::orc::StringVectorBatch&>(*struct_col.fields[1]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector child_values; + std::vector payloads; + child_values.reserve(ROWS_PER_STRIPE); + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + child_a.data[row] = first_value + row; + child_values.push_back("child-" + std::to_string(first_value + row)); + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(child_b, row, child_values.back()); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + struct_col.numElements = ROWS_PER_STRIPE; + nested_struct.numElements = ROWS_PER_STRIPE; + child_a.numElements = ROWS_PER_STRIPE; + child_b.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(1); + add_batch(1000); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_struct_array_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>,payload:string>")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& struct_col = dynamic_cast<::orc::StructVectorBatch&>(*struct_batch.fields[0]); + auto& array_child = dynamic_cast<::orc::ListVectorBatch&>(*struct_col.fields[0]); + auto& array_values = dynamic_cast<::orc::LongVectorBatch&>(*array_child.elements); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + array_child.offsets[row] = row; + array_values.data[row] = value; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + array_child.offsets[ROWS_PER_STRIPE] = ROWS_PER_STRIPE; + struct_batch.numElements = ROWS_PER_STRIPE; + struct_col.numElements = ROWS_PER_STRIPE; + array_child.numElements = ROWS_PER_STRIPE; + array_values.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(1); + add_batch(1000); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_float_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](double first_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& float_batch = dynamic_cast<::orc::DoubleVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + float_batch.data[row] = first_value + static_cast(row); + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + float_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(1); + add_batch(1000); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_sarg_types_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_date_day, int64_t first_timestamp_second, + int64_t first_decimal_value) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& date_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& timestamp_batch = dynamic_cast<::orc::TimestampVectorBatch&>(*struct_batch.fields[1]); + auto& decimal_batch = dynamic_cast<::orc::Decimal64VectorBatch&>(*struct_batch.fields[2]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[3]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + date_batch.data[row] = first_date_day + row; + timestamp_batch.data[row] = first_timestamp_second + row; + timestamp_batch.nanoseconds[row] = 123000000; + decimal_batch.values[row] = first_decimal_value + row; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + date_batch.numElements = ROWS_PER_STRIPE; + timestamp_batch.numElements = ROWS_PER_STRIPE; + decimal_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(0, 0, 0); + add_batch(18628, 1609459200, 100000); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_multi_stripe_orc_timestamp_instant_sarg_file( + const std::string& file_path, int64_t first_timestamp_second = 0, + int64_t second_timestamp_second = 1609459200) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t first_timestamp_second) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& timestamp_batch = dynamic_cast<::orc::TimestampVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + timestamp_batch.data[row] = first_timestamp_second + row; + timestamp_batch.nanoseconds[row] = 123000000; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + timestamp_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(first_timestamp_second); + add_batch(second_timestamp_second); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_two_stripe_constant_date_file(const std::string& file_path, int64_t first_date_day, + int64_t second_date_day) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t date_day) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& date_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + date_batch.data[row] = date_day; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + date_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(first_date_day); + add_batch(second_date_day); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_two_stripe_constant_timestamp_file(const std::string& file_path, + int64_t first_timestamp_second, + int64_t second_timestamp_second) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(4 * 1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + options.setStripeSize(1); + options.setDictionaryKeySizeThreshold(0); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + + auto add_batch = [&](int64_t timestamp_second) { + constexpr int64_t ROWS_PER_STRIPE = 200; + auto batch = writer->createRowBatch(ROWS_PER_STRIPE); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& timestamp_batch = dynamic_cast<::orc::TimestampVectorBatch&>(*struct_batch.fields[0]); + auto& payload_batch = dynamic_cast<::orc::StringVectorBatch&>(*struct_batch.fields[1]); + std::vector payloads; + payloads.reserve(ROWS_PER_STRIPE); + for (int64_t row = 0; row < ROWS_PER_STRIPE; ++row) { + timestamp_batch.data[row] = timestamp_second; + timestamp_batch.nanoseconds[row] = 123000000; + payloads.push_back(std::string(2048, static_cast('a' + row % 26))); + set_string_value(payload_batch, row, payloads.back()); + } + struct_batch.numElements = ROWS_PER_STRIPE; + timestamp_batch.numElements = ROWS_PER_STRIPE; + payload_batch.numElements = ROWS_PER_STRIPE; + writer->add(*batch); + }; + + add_batch(first_timestamp_second); + add_batch(second_timestamp_second); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +uint64_t get_orc_stripe_count(const std::string& file_path) { + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + const auto file_size = in.tellg(); + in.seekg(0); + std::vector data(static_cast(file_size)); + in.read(data.data(), file_size); + + ::orc::ReaderOptions options; + options.setMemoryPool(*::orc::getDefaultPool()); + auto input_stream = std::make_unique(data.data(), data.size()); + auto reader = ::orc::createReader(std::move(input_stream), options); + return reader->getNumberOfStripes(); +} + +struct OrcStripeLayout { + uint64_t offset = 0; + uint64_t length = 0; + uint64_t rows = 0; +}; + +std::vector get_orc_stripe_layout(const std::string& file_path) { + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + const auto file_size = in.tellg(); + in.seekg(0); + std::vector data(static_cast(file_size)); + in.read(data.data(), file_size); + + ::orc::ReaderOptions options; + options.setMemoryPool(*::orc::getDefaultPool()); + auto input_stream = std::make_unique(data.data(), data.size()); + auto reader = ::orc::createReader(std::move(input_stream), options); + + std::vector layout; + const auto stripe_count = reader->getNumberOfStripes(); + layout.reserve(stripe_count); + for (uint64_t i = 0; i < stripe_count; ++i) { + const auto stripe = reader->getStripe(i); + layout.push_back({.offset = stripe->getOffset(), + .length = stripe->getLength(), + .rows = stripe->getNumberOfRows()}); + } + return layout; +} + +Block build_file_block(const std::vector& schema) { + Block block; + for (const auto& field : schema) { + block.insert({field.type->create_column(), field.type, field.name}); + } + return block; +} + +const ColumnInt64& int64_data_column(const IColumn& column) { + if (column.is_nullable()) { + return assert_cast( + assert_cast(column).get_nested_column()); + } + return assert_cast(column); +} + +format::LocalColumnIndex field_projection(int32_t field_id, bool project_all_children = true) { + auto projection = format::LocalColumnIndex::top_level(format::LocalColumnId(field_id)); + projection.project_all_children = project_all_children; + return projection; +} + +format::LocalColumnIndex make_projection(const format::ColumnDefinition& field, + bool project_all_children = true) { + return field_projection(field.file_local_id(), project_all_children); +} + +format::LocalColumnIndex struct_child_projection(int32_t root_field_id, int32_t child_field_id) { + auto projection = format::LocalColumnIndex::partial_local(root_field_id); + projection.children.push_back(format::LocalColumnIndex::local(child_field_id)); + return projection; +} + +class NewOrcReaderTest : public testing::Test { +protected: + static void SetUpTestSuite() { TimezoneUtils::load_timezones_to_cache(); } + + void SetUp() override { + _test_dir = unique_test_dir("doris_new_orc_reader_test"); + std::filesystem::remove_all(_test_dir); + std::filesystem::create_directories(_test_dir); + _file_path = (_test_dir / "reader.orc").string(); + write_orc_file(_file_path); + } + + void TearDown() override { std::filesystem::remove_all(_test_dir); } + + std::unique_ptr create_reader( + RuntimeProfile* profile = nullptr, + std::optional global_rowid_context = std::nullopt) const { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto file_description = std::make_unique(); + file_description->path = _file_path; + file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); + return std::make_unique(system_properties, file_description, + nullptr, profile, global_rowid_context); + } + + std::unique_ptr create_reader_for_path( + const std::string& file_path, RuntimeProfile* profile = nullptr, + std::shared_ptr io_ctx = nullptr, + std::optional global_rowid_context = std::nullopt, + bool enable_mapping_timestamp_tz = false) const { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto file_description = std::make_unique(); + file_description->path = file_path; + file_description->file_size = static_cast(std::filesystem::file_size(file_path)); + return std::make_unique(system_properties, file_description, io_ctx, + profile, global_rowid_context, + enable_mapping_timestamp_tz); + } + + // Builds a reader whose FileDescription carries the split byte window + // [range_start_offset, range_start_offset + range_size). A negative range_size means the + // whole file (unset sentinel), matching how FE leaves the range unspecified. + std::unique_ptr create_reader_with_range( + const std::string& file_path, int64_t range_start_offset, int64_t range_size, + RuntimeProfile* profile = nullptr) const { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto file_description = std::make_unique(); + file_description->path = file_path; + file_description->file_size = static_cast(std::filesystem::file_size(file_path)); + file_description->range_start_offset = range_start_offset; + file_description->range_size = range_size; + return std::make_unique(system_properties, file_description, + nullptr, profile, std::nullopt); + } + + std::filesystem::path _test_dir; + std::string _file_path; +}; + +TEST_F(NewOrcReaderTest, AggregatePushdownReturnsCountFromFileMetadata) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + auto status = reader->open(request); + ASSERT_TRUE(status.ok()) << status; + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + format::FileAggregateResult aggregate_result; + status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(aggregate_result.count, ROW_COUNT); + EXPECT_TRUE(aggregate_result.columns.empty()); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownCountUsesOnlySplitStripes) { + const auto multi_stripe_file_path = (_test_dir / "aggregate_count_split.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + const auto layout = get_orc_stripe_layout(multi_stripe_file_path); + ASSERT_EQ(layout.size(), 2); + + auto count_split_rows = [&](uint64_t start, uint64_t size) { + auto reader = create_reader_with_range(multi_stripe_file_path, static_cast(start), + static_cast(size)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + EXPECT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + EXPECT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + format::FileAggregateResult aggregate_result; + auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + EXPECT_TRUE(status.ok()) << status; + return aggregate_result.count; + }; + + const int64_t first_split_count = + count_split_rows(layout[0].offset, layout[1].offset - layout[0].offset); + const int64_t second_split_count = count_split_rows(layout[1].offset, layout[1].length); + + EXPECT_EQ(first_split_count, layout[0].rows); + EXPECT_EQ(second_split_count, layout[1].rows); + EXPECT_EQ(first_split_count + second_split_count, layout[0].rows + layout[1].rows); +} + +TEST_F(NewOrcReaderTest, OpenAcceptsDorisOffsetTimezone) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("+08:00"); + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + auto status = reader->open(request); + ASSERT_TRUE(status.ok()) << status; +} + +TEST_F(NewOrcReaderTest, OpenAcceptsDorisCstTimezone) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("CST"); + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + auto status = reader->open(request); + ASSERT_TRUE(status.ok()) << status; +} + +TEST_F(NewOrcReaderTest, OpenDelegatesNonHourOffsetTimezoneToOrc) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("+05:30"); + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + auto status = reader->open(request); + EXPECT_FALSE(status.is()) << status.to_string(); + EXPECT_EQ(status.to_string().find("non-hour offset"), std::string::npos) << status.to_string(); +} + +TEST_F(NewOrcReaderTest, InitReturnsEndOfFileWhenIoContextShouldStop) { + auto io_ctx = std::make_shared(); + io_ctx->should_stop = true; + auto reader = create_reader_for_path(_file_path, nullptr, io_ctx); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto status = reader->init(&state); + EXPECT_TRUE(status.is()) << status; +} + +TEST_F(NewOrcReaderTest, AggregatePushdownReturnsMinMaxForPrimitiveColumn) { + const auto multi_stripe_file_path = (_test_dir / "aggregate_minmax_int.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))}); + format::FileAggregateResult aggregate_result; + auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(aggregate_result.columns.size(), 1); + EXPECT_EQ(aggregate_result.count, 400); + EXPECT_TRUE(aggregate_result.columns[0].has_min); + EXPECT_TRUE(aggregate_result.columns[0].has_max); + EXPECT_EQ(aggregate_result.columns[0].min_value.get(), 1); + EXPECT_EQ(aggregate_result.columns[0].max_value.get(), 1199); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownReturnsMinMaxForStructLeaf) { + const auto multi_stripe_file_path = (_test_dir / "aggregate_minmax_struct.orc").string(); + write_multi_stripe_orc_struct_file(multi_stripe_file_path); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {struct_child_projection(0, 0)}; + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back({.projection = struct_child_projection(0, 0)}); + format::FileAggregateResult aggregate_result; + auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(aggregate_result.columns.size(), 1); + EXPECT_EQ(aggregate_result.count, 400); + EXPECT_TRUE(aggregate_result.columns[0].has_min); + EXPECT_TRUE(aggregate_result.columns[0].has_max); + EXPECT_EQ(aggregate_result.columns[0].min_value.get(), 1); + EXPECT_EQ(aggregate_result.columns[0].max_value.get(), 1199); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownUsesPrunedStripes) { + const auto multi_stripe_file_path = (_test_dir / "aggregate_pruned_stripes.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest count_request; + count_request.agg_type = TPushAggOp::type::COUNT; + format::FileAggregateResult count_result; + auto status = reader->get_aggregate_result(count_request, &count_result); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(count_result.count, 200); + + format::FileAggregateRequest minmax_request; + minmax_request.agg_type = TPushAggOp::type::MINMAX; + minmax_request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))}); + format::FileAggregateResult minmax_result; + status = reader->get_aggregate_result(minmax_request, &minmax_result); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(minmax_result.columns.size(), 1); + EXPECT_EQ(minmax_result.count, 200); + EXPECT_TRUE(minmax_result.columns[0].has_min); + EXPECT_TRUE(minmax_result.columns[0].has_max); + EXPECT_EQ(minmax_result.columns[0].min_value.get(), 1000); + EXPECT_EQ(minmax_result.columns[0].max_value.get(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownTimestampMinMaxUsesSessionTimezone) { + const auto multi_stripe_file_path = (_test_dir / "aggregate_timestamp_timezone.orc").string(); + write_two_stripe_constant_timestamp_file(multi_stripe_file_path, 0, 3600); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("Asia/Shanghai"); + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))}); + format::FileAggregateResult aggregate_result; + auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(aggregate_result.columns.size(), 1); + EXPECT_EQ(aggregate_result.count, 400); + EXPECT_TRUE(aggregate_result.columns[0].has_min); + EXPECT_TRUE(aggregate_result.columns[0].has_max); + EXPECT_EQ(aggregate_result.columns[0].min_value.get(), + make_datetime_v2(1970, 1, 1, 8, 0, 0, 123000)); + EXPECT_EQ(aggregate_result.columns[0].max_value.get(), + make_datetime_v2(1970, 1, 1, 9, 0, 0, 123000)); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownTimestampInstantMinMaxUsesTimestampTzWhenMapped) { + const auto multi_stripe_file_path = (_test_dir / "aggregate_timestamp_instant_tz.orc").string(); + write_multi_stripe_orc_timestamp_instant_sarg_file(multi_stripe_file_path); + + auto reader = + create_reader_for_path(multi_stripe_file_path, nullptr, nullptr, std::nullopt, true); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("Asia/Shanghai"); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + ASSERT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_TIMESTAMPTZ); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))}); + format::FileAggregateResult aggregate_result; + auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(aggregate_result.columns.size(), 1); + EXPECT_EQ(aggregate_result.count, 400); + EXPECT_TRUE(aggregate_result.columns[0].has_min); + EXPECT_TRUE(aggregate_result.columns[0].has_max); + ASSERT_EQ(aggregate_result.columns[0].min_value.get_type(), TYPE_TIMESTAMPTZ); + ASSERT_EQ(aggregate_result.columns[0].max_value.get_type(), TYPE_TIMESTAMPTZ); + EXPECT_EQ(aggregate_result.columns[0].min_value.get().to_string( + state.timezone_obj(), 6), + "1970-01-01 08:00:00.123000+08:00"); + EXPECT_EQ(aggregate_result.columns[0].max_value.get().to_string( + state.timezone_obj(), 6), + "2021-01-01 08:03:19.123000+08:00"); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownCharMinMaxTrimsTrailingSpaces) { + const auto multi_stripe_file_path = (_test_dir / "aggregate_char_minmax.orc").string(); + write_multi_stripe_orc_char_file(multi_stripe_file_path); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult aggregate_result; + auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(aggregate_result.columns.size(), 1); + EXPECT_TRUE(aggregate_result.columns[0].has_min); + EXPECT_TRUE(aggregate_result.columns[0].has_max); + EXPECT_EQ(aggregate_result.columns[0].min_value.get(), "aaa_1000"); + EXPECT_EQ(aggregate_result.columns[0].max_value.get(), "zzz_1199"); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownCountColumnUsesNonNullValueCount) { + const auto multi_stripe_file_path = (_test_dir / "aggregate_count_nullable_int.orc").string(); + write_two_stripe_orc_nullable_int_file(multi_stripe_file_path); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + ASSERT_TRUE(reader->open(std::make_shared()).ok()); + + format::FileAggregateRequest count_star_request; + count_star_request.agg_type = TPushAggOp::type::COUNT; + format::FileAggregateResult count_star_result; + auto status = reader->get_aggregate_result(count_star_request, &count_star_result); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(count_star_result.count, 400); + + format::FileAggregateRequest count_column_request; + count_column_request.agg_type = TPushAggOp::type::COUNT; + count_column_request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult count_column_result; + status = reader->get_aggregate_result(count_column_request, &count_column_result); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(count_column_result.count, 200); +} + +TEST_F(NewOrcReaderTest, GetSchemaReturnsFileLocalColumns) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(schema[0].local_id, 0); + EXPECT_EQ(schema[0].name, "id"); + EXPECT_EQ(schema[0].type->get_primitive_type(), TYPE_INT); + EXPECT_TRUE(schema[0].type->is_nullable()); + EXPECT_EQ(schema[1].local_id, 1); + EXPECT_EQ(schema[1].name, "value"); + EXPECT_EQ(schema[1].type->get_primitive_type(), TYPE_STRING); + EXPECT_TRUE(schema[1].type->is_nullable()); +} + +TEST_F(NewOrcReaderTest, GetSchemaReturnsExpectedVirtualColumnNullability) { + const format::GlobalRowIdContext context {.version = 7, .backend_id = 12345, .file_id = 678}; + auto reader = create_reader(nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + const auto row_position_field = format::orc::OrcReader::row_position_column_definition(); + EXPECT_EQ(row_position_field.local_id, format::ROW_POSITION_COLUMN_ID); + EXPECT_EQ(row_position_field.name, format::ROW_POSITION_COLUMN_NAME); + ASSERT_FALSE(row_position_field.type->is_nullable()); + EXPECT_EQ(row_position_field.type->get_primitive_type(), TYPE_BIGINT); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + const auto& global_rowid_field = schema.back(); + EXPECT_EQ(global_rowid_field.local_id, format::GLOBAL_ROWID_COLUMN_ID); + EXPECT_EQ(global_rowid_field.column_type, format::ColumnType::GLOBAL_ROWID); + ASSERT_TRUE(global_rowid_field.type->is_nullable()); + EXPECT_EQ(remove_nullable(global_rowid_field.type)->get_primitive_type(), TYPE_STRING); +} + +TEST_F(NewOrcReaderTest, GetSchemaReturnsNullableMapChildren) { + const auto file_path = (_test_dir / "complex.orc").string(); + write_complex_orc_file(file_path); + auto reader = create_reader_for_path(file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_GE(schema.size(), 3); + const auto& map_field = schema[2]; + ASSERT_EQ(map_field.name, "map_col"); + ASSERT_EQ(map_field.children.size(), 2); + EXPECT_EQ(map_field.children[0].name, "key"); + EXPECT_EQ(map_field.children[1].name, "value"); + ASSERT_NE(map_field.children[0].type, nullptr); + ASSERT_NE(map_field.children[1].type, nullptr); + EXPECT_TRUE(map_field.children[0].type->is_nullable()); + EXPECT_TRUE(map_field.children[1].type->is_nullable()); +} + +TEST_F(NewOrcReaderTest, ReadGlobalRowIdVirtualColumn) { + const format::GlobalRowIdContext context {.version = 7, .backend_id = 12345, .file_id = 678}; + auto reader = create_reader(nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + Block block = build_file_block({schema.back()}); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(format::GLOBAL_ROWID_COLUMN_ID)}; + request->local_positions = { + {format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID), format::LocalIndex(0)}}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, ROW_COUNT); + + const auto& global_rowids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& global_rowids = + assert_cast(global_rowids_nullable.get_nested_column()); + for (int64_t row = 0; row < ROW_COUNT; ++row) { + EXPECT_FALSE(global_rowids_nullable.is_null_at(row)); + const auto rowid = global_rowids.get_data_at(row); + ASSERT_EQ(rowid.size, sizeof(GlobalRowLoacationV2)); + GlobalRowLoacationV2 location(0, 0, 0, 0); + std::memcpy(&location, rowid.data, sizeof(location)); + EXPECT_EQ(location.version, context.version); + EXPECT_EQ(location.backend_id, context.backend_id); + EXPECT_EQ(location.file_id, context.file_id); + EXPECT_EQ(location.row_id, row); + } +} + +TEST_F(NewOrcReaderTest, ReadFileLocalColumnsThenEof) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, ROW_COUNT); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + const auto& values_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& values = assert_cast(values_nullable.get_nested_column()); + ASSERT_EQ(ids.size(), ROW_COUNT); + ASSERT_EQ(values.size(), ROW_COUNT); + EXPECT_EQ(ids.get_element(0), 1); + EXPECT_EQ(ids.get_element(4), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "one"); + EXPECT_EQ(values.get_data_at(4).to_string(), "five"); + + rows = 0; + eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); + EXPECT_EQ(block.rows(), 0); +} + +TEST_F(NewOrcReaderTest, StripePrefetchPublishesMergedReadProfile) { + static constexpr int64_t MAX_MERGE_DISTANCE_BYTES = 1L * 1024L * 1024L; + static constexpr int64_t ONCE_MAX_READ_BYTES = 8L * 1024L * 1024L; + const auto file_path = (_test_dir / "stripe_prefetch.orc").string(); + write_orc_prefetch_file(file_path); + ASSERT_TRUE(has_mergeable_orc_stream_cluster(file_path, MAX_MERGE_DISTANCE_BYTES, + ONCE_MAX_READ_BYTES)); + + RuntimeProfile profile("orc_v2_stripe_prefetch"); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + auto reader = create_reader_for_path(file_path, &profile, io_ctx); + + TQueryOptions query_options; + query_options.__set_orc_once_max_read_bytes(ONCE_MAX_READ_BYTES); + query_options.__set_orc_max_merge_distance_bytes(MAX_MERGE_DISTANCE_BYTES); + RuntimeState state {query_options, TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + auto request = std::make_shared(); + for (const auto& field : schema) { + request->non_predicate_columns.push_back(make_projection(field)); + } + ASSERT_TRUE(reader->open(request).ok()); + + size_t result_rows = 0; + std::vector first_values; + std::vector second_values; + std::vector third_values; + std::vector payload_values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows > 0) { + const auto& first_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + const auto& second_column = assert_cast( + assert_cast(*block.get_by_position(1).column) + .get_nested_column()); + const auto& third_column = assert_cast( + assert_cast(*block.get_by_position(2).column) + .get_nested_column()); + const auto& payload_column = assert_cast( + assert_cast(*block.get_by_position(3).column) + .get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + first_values.push_back(first_column.get_element(row)); + second_values.push_back(second_column.get_element(row)); + third_values.push_back(third_column.get_element(row)); + payload_values.push_back(payload_column.get_data_at(row).to_string()); + } + } + result_rows += rows; + } + EXPECT_EQ(result_rows, PREFETCH_ROW_COUNT); + ASSERT_EQ(first_values.size(), PREFETCH_ROW_COUNT); + ASSERT_EQ(second_values.size(), PREFETCH_ROW_COUNT); + ASSERT_EQ(third_values.size(), PREFETCH_ROW_COUNT); + ASSERT_EQ(payload_values.size(), PREFETCH_ROW_COUNT); + for (size_t row : std::array {0, PREFETCH_ROW_COUNT / 2, PREFETCH_ROW_COUNT - 1}) { + EXPECT_EQ(first_values[row], row); + EXPECT_EQ(second_values[row], row * 2); + EXPECT_EQ(third_values[row], row * 3); + EXPECT_EQ(payload_values[row], "payload-" + std::to_string(row)); + } + ASSERT_TRUE(reader->close().ok()); + + ASSERT_NE(profile.get_counter("RequestIO"), nullptr); + ASSERT_NE(profile.get_counter("MergedIO"), nullptr); + ASSERT_NE(profile.get_counter("ApplyBytes"), nullptr); + ASSERT_NE(profile.get_counter("ClusterNum"), nullptr); + ASSERT_NE(profile.get_counter("OverReadBytes"), nullptr); + EXPECT_GT(profile.get_counter("RequestIO")->value(), profile.get_counter("MergedIO")->value()); + EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); + EXPECT_GT(profile.get_counter("ApplyBytes")->value(), 0); + EXPECT_GT(profile.get_counter("ClusterNum")->value(), 0); + EXPECT_GE(profile.get_counter("OverReadBytes")->value(), 0); +} + +TEST_F(NewOrcReaderTest, StripePrefetchCanBeDisabledByZeroOnceMaxReadBytes) { + static constexpr int64_t MAX_MERGE_DISTANCE_BYTES = 1L * 1024L * 1024L; + const auto file_path = (_test_dir / "stripe_prefetch_disabled.orc").string(); + write_orc_prefetch_file(file_path); + + RuntimeProfile profile("orc_v2_stripe_prefetch_disabled"); + auto reader = create_reader_for_path(file_path, &profile); + + TQueryOptions query_options; + query_options.__set_orc_once_max_read_bytes(0); + query_options.__set_orc_max_merge_distance_bytes(MAX_MERGE_DISTANCE_BYTES); + RuntimeState state {query_options, TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + auto request = std::make_shared(); + request->non_predicate_columns = {make_projection(schema[0]), make_projection(schema[3])}; + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block({schema[0], schema[3]}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_GT(rows, 0); + + const auto& first_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + const auto& payload_column = assert_cast( + assert_cast(*block.get_by_position(1).column) + .get_nested_column()); + EXPECT_EQ(first_column.get_element(0), 0); + EXPECT_EQ(payload_column.get_data_at(0).to_string(), "payload-0"); + EXPECT_EQ(first_column.get_element(rows - 1), rows - 1); + EXPECT_EQ(payload_column.get_data_at(rows - 1).to_string(), + "payload-" + std::to_string(rows - 1)); + + size_t result_rows = rows; + while (!eof) { + Block next_block = build_file_block({schema[0], schema[3]}); + rows = 0; + ASSERT_TRUE(reader->get_block(&next_block, &rows, &eof).ok()); + result_rows += rows; + } + EXPECT_EQ(result_rows, PREFETCH_ROW_COUNT); + ASSERT_TRUE(reader->close().ok()); + + EXPECT_EQ(profile.get_counter("MergedIO"), nullptr); +} + +TEST_F(NewOrcReaderTest, RejectsInvalidNaturalReadSizeConfig) { + const auto old_natural_read_size_mb = config::orc_natural_read_size_mb; + config::orc_natural_read_size_mb = 0; + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + const auto status = reader->init(&state); + config::orc_natural_read_size_mb = old_natural_read_size_mb; + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("orc_natural_read_size_mb"), std::string::npos); +} + +TEST_F(NewOrcReaderTest, GetBlockStopsWhenIoContextShouldStop) { + auto io_ctx = std::make_shared(); + auto reader = create_reader_for_path(_file_path, nullptr, io_ctx); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + Block block = build_file_block({schema[0]}); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + io_ctx->should_stop = true; + size_t rows = 123; + bool eof = false; + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); + EXPECT_EQ(block.rows(), 0); +} + +TEST_F(NewOrcReaderTest, ReadRowPositionVirtualColumn) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + std::vector block_schema { + format::orc::OrcReader::row_position_column_definition(), schema[0]}; + Block block = build_file_block(block_schema); + + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(row_position_column_id)}; + request->non_predicate_columns = {field_projection(0)}; + request->local_positions = { + {format::LocalColumnId(row_position_column_id), format::LocalIndex(0)}, + {format::LocalColumnId(0), format::LocalIndex(1)}}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, ROW_COUNT); + + const auto& row_positions = int64_data_column(*block.get_by_position(0).column); + const auto& ids_nullable = assert_cast(*block.get_by_position(1).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (int64_t row = 0; row < ROW_COUNT; ++row) { + EXPECT_EQ(row_positions.get_element(row), row); + EXPECT_EQ(ids.get_element(row), row + 1); + } +} + +TEST_F(NewOrcReaderTest, ReadOnlyRowPositionVirtualColumn) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + std::vector block_schema { + format::orc::OrcReader::row_position_column_definition()}; + Block block = build_file_block(block_schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(row_position_column_id)}; + request->local_positions = { + {format::LocalColumnId(row_position_column_id), format::LocalIndex(0)}}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, ROW_COUNT); + + const auto& row_positions = int64_data_column(*block.get_by_position(0).column); + for (int64_t row = 0; row < ROW_COUNT; ++row) { + EXPECT_EQ(row_positions.get_element(row), row); + } +} + +TEST_F(NewOrcReaderTest, RowPositionDeletePredicateWithoutPhysicalColumnsSkipsOrcLazy) { + const format::GlobalRowIdContext context {.version = 7, .backend_id = 12345, .file_id = 678}; + auto reader = create_reader(nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + const auto global_rowid_column_id = format::GLOBAL_ROWID_COLUMN_ID; + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + std::vector block_schema { + format::orc::OrcReader::row_position_column_definition(), schema.back()}; + Block block = build_file_block(block_schema); + + static const std::vector deleted_rows {1, 3}; + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->add_child(TableSlotRef::create_shared( + 0, 0, -1, std::make_shared(), format::ROW_POSITION_COLUMN_NAME)); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(row_position_column_id)}; + request->non_predicate_columns = {field_projection(global_rowid_column_id)}; + request->local_positions = { + {format::LocalColumnId(row_position_column_id), format::LocalIndex(0)}, + {format::LocalColumnId(global_rowid_column_id), format::LocalIndex(1)}}; + request->delete_conjuncts.push_back(VExprContext::create_shared(std::move(delete_predicate))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& row_positions = int64_data_column(*block.get_by_position(0).column); + EXPECT_EQ(row_positions.get_element(0), 0); + EXPECT_EQ(row_positions.get_element(1), 2); + EXPECT_EQ(row_positions.get_element(2), 4); + const auto& global_rowids_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& global_rowids = + assert_cast(global_rowids_nullable.get_nested_column()); + ASSERT_EQ(global_rowids.size(), 3); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 0); +} + +TEST_F(NewOrcReaderTest, OrcLazyDecodesSelectedRowPositionRows) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + std::vector block_schema { + schema[0], format::orc::OrcReader::row_position_column_definition()}; + Block block = build_file_block(block_schema); + + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(row_position_column_id)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(row_position_column_id), format::LocalIndex(1)}}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 2))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + const auto& row_positions = int64_data_column(*block.get_by_position(1).column); + ASSERT_EQ(ids.size(), 3); + ASSERT_EQ(row_positions.size(), 3); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(row_positions.get_element(0), 2); + EXPECT_EQ(row_positions.get_element(1), 3); + EXPECT_EQ(row_positions.get_element(2), 4); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 2); +} + +TEST_F(NewOrcReaderTest, OrcLazyKeepsRowPositionAfterRejectedBatch) { + constexpr int64_t row_count = 4101; + constexpr int32_t first_selected_id = 4091; + const auto large_file_path = (_test_dir / "large_lazy_row_position.orc").string(); + write_large_orc_int_file(large_file_path, row_count); + + auto reader = create_reader_for_path(large_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + std::vector block_schema { + schema[0], format::orc::OrcReader::row_position_column_definition()}; + + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(row_position_column_id)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(row_position_column_id), format::LocalIndex(1)}}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, first_selected_id - 1))); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector result_ids; + std::vector result_row_positions; + bool eof = false; + while (!eof) { + Block block = build_file_block(block_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + const auto& row_positions = int64_data_column(*block.get_by_position(1).column); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + result_row_positions.push_back(row_positions.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), static_cast(row_count - first_selected_id + 1)); + ASSERT_EQ(result_row_positions.size(), result_ids.size()); + for (size_t row = 0; row < result_ids.size(); ++row) { + EXPECT_EQ(result_ids[row], first_selected_id + static_cast(row)); + EXPECT_EQ(result_row_positions[row], first_selected_id - 1 + static_cast(row)); + } +} + +TEST_F(NewOrcReaderTest, OrcLazyDeletePredicateUsesRowPositionAcrossBatches) { + constexpr int64_t row_count = 4101; + constexpr int32_t first_selected_id = 4091; + constexpr int64_t deleted_row_position = 4097; + const auto large_file_path = (_test_dir / "large_lazy_delete_position.orc").string(); + write_large_orc_int_file(large_file_path, row_count); + + auto reader = create_reader_for_path(large_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + std::vector block_schema { + schema[0], format::orc::OrcReader::row_position_column_definition()}; + + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0), field_projection(row_position_column_id)}; + request->non_predicate_columns = {field_projection(0)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(row_position_column_id), format::LocalIndex(1)}}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, first_selected_id - 1))); + static const std::vector deleted_rows {deleted_row_position}; + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->add_child(TableSlotRef::create_shared( + 1, 1, -1, std::make_shared(), format::ROW_POSITION_COLUMN_NAME)); + request->delete_conjuncts.push_back(VExprContext::create_shared(std::move(delete_predicate))); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector result_ids; + bool eof = false; + while (!eof) { + Block block = build_file_block(block_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), static_cast(row_count - first_selected_id)); + EXPECT_EQ(result_ids.front(), first_selected_id); + EXPECT_EQ(result_ids.back(), row_count); + EXPECT_EQ(std::find(result_ids.begin(), result_ids.end(), + static_cast(deleted_row_position + 1)), + result_ids.end()); +} + +TEST_F(NewOrcReaderTest, ConjunctFiltersRows) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 2))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + const auto& values_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& values = assert_cast(values_nullable.get_nested_column()); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 4); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "three"); + EXPECT_EQ(values.get_data_at(2).to_string(), "five"); +} + +TEST_F(NewOrcReaderTest, OrcLazyDecodesOnlySelectedNonPredicateRows) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 2))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + const auto& values_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& values = assert_cast(values_nullable.get_nested_column()); + ASSERT_EQ(ids.size(), 3); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "three"); + EXPECT_EQ(values.get_data_at(2).to_string(), "five"); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 2); +} + +TEST_F(NewOrcReaderTest, ClosePublishesOrcLazyStatisticsToRuntimeProfile) { + RuntimeProfile profile("new_orc_lazy_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 2))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_FALSE(eof); + ASSERT_EQ(rows, 3); + ASSERT_TRUE(reader->close().ok()); + + ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 2); +} + +TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { + RuntimeProfile profile("new_orc_lazy_disabled_profile"); + auto reader = create_reader(&profile); + TQueryOptions query_options; + query_options.__set_enable_orc_lazy_mat(false); + RuntimeState state {query_options, TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 2))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_FALSE(eof); + ASSERT_EQ(rows, 3); + ASSERT_TRUE(reader->close().ok()); + + ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); +} + +TEST_F(NewOrcReaderTest, ConditionCacheMissMarksSurvivingGranules) { + constexpr int64_t row_count = ConditionCacheContext::GRANULE_SIZE * 2; + const auto large_file_path = (_test_dir / "condition_cache_miss.orc").string(); + write_large_orc_int_file(large_file_path, row_count); + + auto reader = create_reader_for_path(large_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, ConditionCacheContext::GRANULE_SIZE))); + ASSERT_TRUE(reader->open(request).ok()); + + auto ctx = std::make_shared(); + ctx->is_hit = false; + ctx->filter_result = std::make_shared>(3, false); + reader->set_condition_cache_context(ctx); + + std::vector ids; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + } + } + + ASSERT_EQ(ids.size(), static_cast(ConditionCacheContext::GRANULE_SIZE)); + EXPECT_EQ(ids.front(), ConditionCacheContext::GRANULE_SIZE + 1); + EXPECT_EQ(ids.back(), row_count); + EXPECT_FALSE((*ctx->filter_result)[0]); + EXPECT_TRUE((*ctx->filter_result)[1]); + EXPECT_FALSE((*ctx->filter_result)[2]); +} + +TEST_F(NewOrcReaderTest, ConditionCacheMissMarksSargSkippedLazyRowsByFileRow) { + const auto file_path = (_test_dir / "condition_cache_sarg_skipped_lazy.orc").string(); + write_sarg_skipped_row_group_orc_file(file_path); + + auto reader = create_reader_for_path(file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->local_positions = {{format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(1), format::LocalIndex(1)}}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 5000))); + ASSERT_TRUE(reader->open(request).ok()); + + auto ctx = std::make_shared(); + ctx->is_hit = false; + ctx->filter_result = std::make_shared>(4, false); + reader->set_condition_cache_context(ctx); + + std::vector ids; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + } + } + + ASSERT_EQ(ids.size(), static_cast(ConditionCacheContext::GRANULE_SIZE)); + EXPECT_EQ(ids.front(), 10000); + EXPECT_EQ(ids.back(), 10000 + ConditionCacheContext::GRANULE_SIZE - 1); + EXPECT_FALSE((*ctx->filter_result)[0]); + EXPECT_TRUE((*ctx->filter_result)[1]); + EXPECT_FALSE((*ctx->filter_result)[2]); +} + +TEST_F(NewOrcReaderTest, RowPositionPredicateUsesSargWithoutOrcLazyCallback) { + const auto file_path = (_test_dir / "lazy_row_position_sarg_skipped.orc").string(); + write_sarg_skipped_row_group_orc_file(file_path); + + auto reader = create_reader_for_path(file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + std::vector block_schema { + schema[0], format::orc::OrcReader::row_position_column_definition(), schema[1]}; + + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0), field_projection(row_position_column_id)}; + request->non_predicate_columns = {field_projection(1)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(row_position_column_id), format::LocalIndex(1)}, + {format::LocalColumnId(1), format::LocalIndex(2)}}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 5000))); + static const std::vector deleted_rows {ConditionCacheContext::GRANULE_SIZE}; + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->add_child(TableSlotRef::create_shared( + 1, 1, -1, std::make_shared(), format::ROW_POSITION_COLUMN_NAME)); + request->delete_conjuncts.push_back(VExprContext::create_shared(std::move(delete_predicate))); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + bool eof = false; + while (!eof) { + Block block = build_file_block(block_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + } + } + + ASSERT_EQ(ids.size(), static_cast(ConditionCacheContext::GRANULE_SIZE - 1)); + EXPECT_EQ(ids.front(), 10001); + EXPECT_EQ(ids.back(), 10000 + ConditionCacheContext::GRANULE_SIZE - 1); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 0); +} + +TEST_F(NewOrcReaderTest, ConditionCacheHitSkipsFalseGranulesBeforeColumnRead) { + constexpr int64_t row_count = ConditionCacheContext::GRANULE_SIZE * 2; + const auto large_file_path = (_test_dir / "condition_cache_hit.orc").string(); + write_large_orc_int_file(large_file_path, row_count); + + auto io_ctx = std::make_shared(); + auto reader = create_reader_for_path(large_file_path, nullptr, io_ctx); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + EXPECT_EQ(reader->get_total_rows(), row_count); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, ConditionCacheContext::GRANULE_SIZE))); + ASSERT_TRUE(reader->open(request).ok()); + + auto ctx = std::make_shared(); + ctx->is_hit = true; + ctx->filter_result = + std::make_shared>(std::vector {false, true, false}); + reader->set_condition_cache_context(ctx); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, static_cast(ConditionCacheContext::GRANULE_SIZE)); + EXPECT_EQ(io_ctx->condition_cache_filtered_rows, ConditionCacheContext::GRANULE_SIZE); + + const auto& ids = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + EXPECT_EQ(ids.get_element(0), ConditionCacheContext::GRANULE_SIZE + 1); + EXPECT_EQ(ids.get_element(rows - 1), row_count); + + block = build_file_block(schema); + rows = 0; + eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); +} + +TEST_F(NewOrcReaderTest, ConditionCacheHitHandlesSplitWithoutSelectedStripe) { + const auto multi_stripe_file_path = (_test_dir / "condition_cache_empty_split.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + const auto layout = get_orc_stripe_layout(multi_stripe_file_path); + ASSERT_EQ(layout.size(), 2); + + const int64_t window_start = + layout[0].offset > 0 ? static_cast(layout[0].offset) - 1 : 0; + auto reader = create_reader_with_range(multi_stripe_file_path, window_start, 1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 1))); + ASSERT_TRUE(reader->open(request).ok()); + EXPECT_EQ(reader->get_total_rows(), 0); + + auto ctx = std::make_shared(); + ctx->is_hit = true; + ctx->filter_result = std::make_shared>(std::vector {false}); + reader->set_condition_cache_context(ctx); + + Block block = build_file_block({schema[0]}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); +} + +TEST_F(NewOrcReaderTest, ConditionCacheHitUsesSplitBaseGranule) { + const auto multi_stripe_file_path = (_test_dir / "condition_cache_split_base.orc").string(); + std::vector first_values; + for (int64_t stripe = 0; stripe < 24; ++stripe) { + first_values.push_back(stripe * 1000 + 1); + } + write_multi_stripe_orc_int_only_file(multi_stripe_file_path, first_values); + + const auto layout = get_orc_stripe_layout(multi_stripe_file_path); + ASSERT_FALSE(layout.empty()); + size_t stripe_index = 0; + uint64_t stripe_first_row = 0; + uint64_t accumulated_rows = 0; + for (size_t i = 0; i < layout.size(); ++i) { + if (accumulated_rows / ConditionCacheContext::GRANULE_SIZE > 0) { + stripe_index = i; + stripe_first_row = accumulated_rows; + break; + } + accumulated_rows += layout[i].rows; + } + ASSERT_GT(stripe_first_row / ConditionCacheContext::GRANULE_SIZE, 0); + ASSERT_LT(stripe_index, layout.size()); + auto reader = create_reader_with_range(multi_stripe_file_path, + static_cast(layout[stripe_index].offset), + static_cast(layout[stripe_index].length)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 0))); + ASSERT_TRUE(reader->open(request).ok()); + EXPECT_EQ(reader->get_total_rows(), layout[stripe_index].rows); + + auto ctx = std::make_shared(); + ctx->is_hit = true; + const auto base_granule = stripe_first_row / ConditionCacheContext::GRANULE_SIZE; + ctx->base_granule = static_cast(base_granule); + const auto last_granule = (stripe_first_row + layout[stripe_index].rows - 1) / + ConditionCacheContext::GRANULE_SIZE; + ctx->filter_result = std::make_shared>(last_granule - base_granule + 1, true); + reader->set_condition_cache_context(ctx); + EXPECT_EQ(ctx->base_granule, base_granule); + + Block block = build_file_block({schema[0]}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, layout[stripe_index].rows); + + const auto& ids = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + auto value_for_file_row = [](uint64_t file_row) { + constexpr uint64_t ROWS_PER_BATCH = 200; + return static_cast((file_row / ROWS_PER_BATCH) * 1000 + 1 + + file_row % ROWS_PER_BATCH); + }; + EXPECT_EQ(ids.get_element(0), value_for_file_row(stripe_first_row)); + EXPECT_EQ(ids.get_element(rows - 1), value_for_file_row(stripe_first_row + rows - 1)); +} + +TEST_F(NewOrcReaderTest, OrcLazySkipsNonPredicateColumnsWhenFilterEliminatesBatch) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 999))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); + EXPECT_EQ(block.get_by_position(0).column->size(), 0); + EXPECT_EQ(block.get_by_position(1).column->size(), 0); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, ROW_COUNT); +} + +TEST_F(NewOrcReaderTest, ConjunctFilterDoesNotDecodeDuplicateColumnsTwice) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block({schema[0]}); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 2))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + ASSERT_EQ(ids.size(), 3); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 4); + EXPECT_EQ(ids.get_element(2), 5); +} + +TEST_F(NewOrcReaderTest, OrcLazyDecodesSelectedComplexNonPredicateRows) { + const auto complex_file_path = (_test_dir / "predicate_first_complex.orc").string(); + write_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block({schema[0], schema[1], schema[2]}); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1), field_projection(2)}; + request->local_positions = {{format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(1), format::LocalIndex(1)}, + {format::LocalColumnId(2), format::LocalIndex(2)}}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[0].type, "a", 15))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 2); + + const auto& struct_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& struct_column = + assert_cast(struct_nullable.get_nested_column()); + const auto& struct_a = assert_cast( + assert_cast(struct_column.get_column(0)).get_nested_column()); + ASSERT_EQ(struct_a.size(), 2); + EXPECT_EQ(struct_a.get_element(0), 20); + EXPECT_EQ(struct_a.get_element(1), 30); + + const auto& array_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& array_column = assert_cast(array_nullable.get_nested_column()); + ASSERT_EQ(array_column.get_offsets().size(), 2); + EXPECT_EQ(array_column.get_offsets()[0], 0); + EXPECT_EQ(array_column.get_offsets()[1], 2); + const auto& array_values = assert_cast( + assert_cast(array_column.get_data()).get_nested_column()); + ASSERT_EQ(array_values.size(), 2); + EXPECT_EQ(array_values.get_element(0), 2); + EXPECT_EQ(array_values.get_element(1), 3); + + const auto& map_nullable = assert_cast(*block.get_by_position(2).column); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), 2); + EXPECT_EQ(map_column.get_offsets()[0], 0); + EXPECT_EQ(map_column.get_offsets()[1], 1); + const auto& map_keys = assert_cast( + assert_cast(map_column.get_keys()).get_nested_column()); + const auto& map_values = assert_cast( + assert_cast(map_column.get_values()).get_nested_column()); + ASSERT_EQ(map_keys.size(), 1); + ASSERT_EQ(map_values.size(), 1); + EXPECT_EQ(map_keys.get_data_at(0).to_string(), "c"); + EXPECT_EQ(map_values.get_element(0), 300); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 1); +} + +TEST_F(NewOrcReaderTest, OrcLazySupportsNestedPredicateProjection) { + const auto complex_file_path = (_test_dir / "predicate_nested_projection.orc").string(); + write_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + DataTypes projected_child_types {schema[0].children[0].type}; + Strings projected_child_names {schema[0].children[0].name}; + auto projected_struct_type = make_nullable( + std::make_shared(projected_child_types, projected_child_names)); + + Block block; + block.insert({projected_struct_type->create_column(), projected_struct_type, schema[0].name}); + block.insert({schema[1].type->create_column(), schema[1].type, schema[1].name}); + block.insert({schema[2].type->create_column(), schema[2].type, schema[2].name}); + + auto request = std::make_shared(); + auto struct_projection = make_projection(schema[0], false); + struct_projection.children.push_back(make_projection(schema[0].children[0])); + request->predicate_columns = {std::move(struct_projection)}; + request->non_predicate_columns = {field_projection(1), field_projection(2)}; + request->local_positions = {{format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(1), format::LocalIndex(1)}, + {format::LocalColumnId(2), format::LocalIndex(2)}}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[0].type, "a", 15))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 2); + + const auto& struct_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& struct_column = + assert_cast(struct_nullable.get_nested_column()); + const auto& struct_a = assert_cast( + assert_cast(struct_column.get_column(0)).get_nested_column()); + expect_int32_values(struct_a, {20, 30}); + + const auto& map_nullable = assert_cast(*block.get_by_position(2).column); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), 2); + EXPECT_EQ(map_column.get_offsets()[0], 0); + EXPECT_EQ(map_column.get_offsets()[1], 1); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 1); +} + +TEST_F(NewOrcReaderTest, OrcLazyDecodesSelectedDeepNestedComplexRows) { + const auto complex_file_path = (_test_dir / "predicate_first_deep_complex.orc").string(); + write_deep_nested_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->local_positions = {{format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(1), format::LocalIndex(1)}}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 2))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 2); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + expect_int32_values(ids, {3, 4}); + + expect_deep_nested_column(*block.get_by_position(1).column, {false, false}, {2, 3}, + {"row3_left", "row3_right", "row4"}, {1, 3, 4}, + {"row3_left_key", "row3_right_a", "row3_right_empty", "row4_key"}, + {1, 3, 3, 4}, {30, 31, 32, 40}, + {"thirty", "thirty_one", "thirty_two", "forty"}); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 2); +} + +TEST_F(NewOrcReaderTest, DeleteConjunctFiltersRows) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->delete_conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 3))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 4); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + const auto& values_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& values = assert_cast(values_nullable.get_nested_column()); + EXPECT_EQ(ids.get_element(0), 1); + EXPECT_EQ(ids.get_element(1), 2); + EXPECT_EQ(ids.get_element(2), 4); + EXPECT_EQ(ids.get_element(3), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "one"); + EXPECT_EQ(values.get_data_at(2).to_string(), "four"); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 1); +} + +TEST_F(NewOrcReaderTest, DeleteConjunctFallsBackWhenItReferencesNonDecodedColumn) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->local_positions = {{format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(1), format::LocalIndex(1)}}; + request->delete_conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 1, remove_nullable(schema[1].type), "o", "value"))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 2); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + const auto& values_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& values = assert_cast(values_nullable.get_nested_column()); + EXPECT_EQ(ids.get_element(0), 4); + EXPECT_EQ(ids.get_element(1), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "four"); + EXPECT_EQ(values.get_data_at(1).to_string(), "five"); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 0); +} + +TEST_F(NewOrcReaderTest, DeletePredicateFiltersRowPositions) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + std::vector block_schema { + schema[0], schema[1], format::orc::OrcReader::row_position_column_definition()}; + Block block = build_file_block(block_schema); + + static const std::vector deleted_rows {1, 3}; + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->add_child(TableSlotRef::create_shared( + 2, 2, -1, std::make_shared(), format::ROW_POSITION_COLUMN_NAME)); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(row_position_column_id)}; + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(1), format::LocalIndex(1)}, + {format::LocalColumnId(row_position_column_id), format::LocalIndex(2)}}; + request->delete_conjuncts.push_back(VExprContext::create_shared(std::move(delete_predicate))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + const auto& values_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& values = assert_cast(values_nullable.get_nested_column()); + const auto& row_positions = int64_data_column(*block.get_by_position(2).column); + EXPECT_EQ(ids.get_element(0), 1); + EXPECT_EQ(ids.get_element(1), 3); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "one"); + EXPECT_EQ(values.get_data_at(1).to_string(), "three"); + EXPECT_EQ(values.get_data_at(2).to_string(), "five"); + EXPECT_EQ(row_positions.get_element(0), 0); + EXPECT_EQ(row_positions.get_element(1), 2); + EXPECT_EQ(row_positions.get_element(2), 4); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargConjunctPrunesStripesByStatistics) { + const auto multi_stripe_file_path = (_test_dir / "multi_stripe.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +// A split window covering only the first stripe must return exactly that stripe's rows. +TEST_F(NewOrcReaderTest, SplitRangeSelectsOnlyFirstStripe) { + const auto multi_stripe_file_path = (_test_dir / "split_first_stripe.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + const auto layout = get_orc_stripe_layout(multi_stripe_file_path); + ASSERT_EQ(layout.size(), 2); + + // Window [stripe0.offset, stripe1.offset) covers only the first stripe. + auto reader = + create_reader_with_range(multi_stripe_file_path, static_cast(layout[0].offset), + static_cast(layout[1].offset - layout[0].offset)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1); + EXPECT_EQ(result_ids.back(), 200); +} + +TEST_F(NewOrcReaderTest, SplitRangeSelectsOnlySecondStripe) { + const auto multi_stripe_file_path = (_test_dir / "split_second_stripe.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + const auto layout = get_orc_stripe_layout(multi_stripe_file_path); + ASSERT_EQ(layout.size(), 2); + + // Window [stripe1.offset, stripe1.offset + stripe1.length) covers only the second stripe. + auto reader = + create_reader_with_range(multi_stripe_file_path, static_cast(layout[1].offset), + static_cast(layout[1].length)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); +} + +// A SARG that prunes the first stripe must not count the out-of-split first stripe in the +// filtered statistics: only the in-split stripe is considered, so nothing is pruned. +TEST_F(NewOrcReaderTest, SplitRangeWithSargStaysWithinSplit) { + const auto multi_stripe_file_path = (_test_dir / "split_sarg_within.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + const auto layout = get_orc_stripe_layout(multi_stripe_file_path); + ASSERT_EQ(layout.size(), 2); + + // Restrict to the second stripe (ids 1000..1199). id > 500 keeps every in-split row. + auto reader = + create_reader_with_range(multi_stripe_file_path, static_cast(layout[1].offset), + static_cast(layout[1].length)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + // The out-of-split first stripe must not be counted as filtered. + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SplitRangeCoveringNoStripeReturnsNoRows) { + const auto multi_stripe_file_path = (_test_dir / "split_no_stripe.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + const auto layout = get_orc_stripe_layout(multi_stripe_file_path); + ASSERT_EQ(layout.size(), 2); + + // A one-byte window just before the first stripe covers no stripe offset. + const int64_t window_start = + layout[0].offset > 0 ? static_cast(layout[0].offset) - 1 : 0; + auto reader = create_reader_with_range(multi_stripe_file_path, window_start, 1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, 0); + EXPECT_TRUE(eof); +} + +TEST_F(NewOrcReaderTest, WholeFileWhenRangeSizeNegative) { + const auto multi_stripe_file_path = (_test_dir / "split_whole_file.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + // range_size == -1 is the unset sentinel: the reader must scan the whole file. + auto reader = create_reader_with_range(multi_stripe_file_path, 0, -1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + ASSERT_EQ(result_ids.size(), 400); + EXPECT_EQ(result_ids.front(), 1); + EXPECT_EQ(result_ids.back(), 1199); +} + +TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { + const auto multi_stripe_file_path = (_test_dir / "profile_sarg_pruning.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path, {1, 1000, 2000}); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 3); + + RuntimeProfile profile("new_orc_reader_profile"); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + auto reader = create_reader_for_path(multi_stripe_file_path, &profile, io_ctx); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + ASSERT_EQ(result_rows, 200); + ASSERT_TRUE(reader->close().ok()); + + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsReadNum"), nullptr); + ASSERT_NE(profile.get_counter("FilteredRowsByGroup"), nullptr); + ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); + ASSERT_NE(profile.get_counter("FilteredBytes"), nullptr); + ASSERT_NE(profile.get_counter("FileNum"), nullptr); + const std::array orc_reader_metric_counters { + "ReaderCall", + "ReaderInclusiveLatencyUs", + "DecompressionCall", + "DecompressionLatencyUs", + "DecodingCall", + "DecodingLatencyUs", + "ByteDecodingCall", + "ByteDecodingLatencyUs", + "IOCount", + "IOBlockingLatencyUs", + "SelectedRowGroupCount", + "EvaluatedRowGroupCount", + "ReadRowCount", + }; + for (const auto counter_name : orc_reader_metric_counters) { + ASSERT_NE(profile.get_counter(std::string(counter_name)), nullptr) << counter_name; + } + EXPECT_EQ(profile.get_counter("RowGroupsFiltered")->value(), 2); + EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 2); + EXPECT_EQ(profile.get_counter("RowGroupsReadNum")->value(), 1); + EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 1); + EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 3); + EXPECT_EQ(profile.get_counter("FilteredRowsByGroup")->value(), 400); + EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); + EXPECT_GT(profile.get_counter("FilteredBytes")->value(), 0); + EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); + EXPECT_EQ(profile.get_counter("ReadRowCount")->value(), + static_cast(file_reader_stats.read_rows)); +} + +TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { + const auto multi_stripe_file_path = (_test_dir / "profile_minmax_disabled.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path, {1, 1000, 2000}); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 3); + + RuntimeProfile profile("new_orc_reader_profile_minmax_disabled"); + auto reader = create_reader_for_path(multi_stripe_file_path, &profile); + TQueryOptions query_options; + query_options.__set_enable_orc_filter_by_min_max(false); + query_options.__set_enable_orc_lazy_mat(false); + RuntimeState state {query_options, TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + ASSERT_EQ(result_rows, 200); + ASSERT_TRUE(reader->close().ok()); + + ASSERT_NE(profile.get_counter("SelectedRowGroupCount"), nullptr); + ASSERT_NE(profile.get_counter("EvaluatedRowGroupCount"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); + EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 0); + EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 0); + EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 0); +} + +TEST_F(NewOrcReaderTest, SargConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_conjunct_pruning.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + std::vector block_schema {schema[1], schema[0]}; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1), field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(1, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block(block_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargLiteralOnLeftConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_literal_on_left.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[0].type), Field::create_field(500), "id", + true))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargRuntimeFilterWrapperConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_runtime_filter_wrapper.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + auto impl = std::make_shared>( + 0, remove_nullable(schema[0].type), Field::create_field(500), "id"); + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(std::move(impl)))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargNullAwareRuntimeFilterDoesNotPruneNullStripe) { + const auto multi_stripe_file_path = (_test_dir / "sarg_null_aware_runtime_filter.orc").string(); + write_two_stripe_orc_nullable_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + std::shared_ptr filter(create_set(PrimitiveType::TYPE_INT, true)); + int32_t unmatched_value = 5000; + filter->insert(&unmatched_value); + + auto node = make_filter_in_node(TExprNodeType::NULL_AWARE_IN_PRED); + auto direct_in = VDirectInPredicate::create_shared(node, filter, true); + direct_in->add_child(TableSlotRef::create_shared(0, 0, -1, schema[0].type, "id")); + auto runtime_filter = VRuntimeFilterWrapper::create_shared(node, direct_in, 0.0, true, 7); + auto runtime_filter_context = VExprContext::create_shared(std::move(runtime_filter)); + ASSERT_TRUE(runtime_filter_context->prepare(&state, RowDescriptor()).ok()); + ASSERT_TRUE(runtime_filter_context->open(&state).ok()); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(std::move(runtime_filter_context)); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < rows; ++row) { + EXPECT_TRUE(ids_nullable.is_null_at(row)); + } + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); +} + +TEST_F(NewOrcReaderTest, SargNullSafeEqualNullConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_null_safe_equal_null.orc").string(); + write_two_stripe_orc_nullable_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, false))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < rows; ++row) { + EXPECT_TRUE(ids_nullable.is_null_at(row)); + } + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargNullSafeEqualNullLiteralOnLeftConjunctPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_null_safe_equal_null_literal_on_left.orc").string(); + write_two_stripe_orc_nullable_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, true))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < rows; ++row) { + EXPECT_TRUE(ids_nullable.is_null_at(row)); + } + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargIsNullConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_is_null.orc").string(); + write_two_stripe_orc_nullable_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < rows; ++row) { + EXPECT_TRUE(ids_nullable.is_null_at(row)); + } + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargNullSafeEqualLiteralConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_null_safe_equal_literal.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 1000, false))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + EXPECT_FALSE(ids_nullable.is_null_at(row)); + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 1); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargNullSafeEqualLiteralOnLeftConjunctPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_null_safe_equal_literal_on_left.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 1000, true))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + EXPECT_FALSE(ids_nullable.is_null_at(row)); + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 1); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargSlotToSlotNullSafeEqualDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_slot_to_slot_null_safe_equal.orc").string(); + write_multi_stripe_orc_pair_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + + std::vector block_schema {schema[0], schema[1]}; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0), field_projection(1)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 1))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block(block_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 400); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargNullSafeEqualInsideOrDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_null_safe_equal_inside_or.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + TExprOpcode::COMPOUND_OR, + VExprSPtrs {std::make_shared(0, 1000, + false), + std::make_shared(0, 1001, + false)}))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + EXPECT_FALSE(ids_nullable.is_null_at(row)); + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 2); + EXPECT_EQ(result_ids[0], 1000); + EXPECT_EQ(result_ids[1], 1001); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargNullSafeEqualInsideNestedNotDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_null_safe_equal_inside_nested_not.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + auto null_safe_equal = std::make_shared(0, 1000, false); + auto inner_not = std::make_shared(TExprOpcode::COMPOUND_NOT, + VExprSPtrs {null_safe_equal}); + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + TExprOpcode::COMPOUND_NOT, VExprSPtrs {inner_not}))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + EXPECT_FALSE(ids_nullable.is_null_at(row)); + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 1); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargVarcharCastToStringConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_varchar_cast_to_string.orc").string(); + write_multi_stripe_orc_varchar_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_VARCHAR); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, remove_nullable(schema[0].type), "m", "value"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargStringEqualsKeepsMatchingHiveDictRows) { + const auto fixture_path = find_repo_file( + "docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/" + "orc_table/test_string_dict_filter_orc/test_string_dict_filter.orc"); + ASSERT_TRUE(std::filesystem::exists(fixture_path)); + + auto reader = create_reader_for_path(fixture_path.string()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 9); + EXPECT_EQ(schema[2].name, "o_orderstatus"); + EXPECT_EQ(remove_nullable(schema[2].type)->get_primitive_type(), TYPE_STRING); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(2)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, remove_nullable(schema[2].type), "F", schema[2].name))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector statuses; + while (!eof) { + Block block = build_file_block({schema[2]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& status_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& status_values = + assert_cast(status_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + EXPECT_FALSE(status_nullable.is_null_at(row)); + statuses.push_back(status_values.get_data_at(row).to_string()); + } + } + + ASSERT_EQ(statuses.size(), 2); + EXPECT_EQ(statuses[0], "F"); + EXPECT_EQ(statuses[1], "F"); +} + +TEST_F(NewOrcReaderTest, SargBinaryVarbinaryLiteralConjunctPrunesRowGroups) { + const auto multi_stripe_file_path = + (_test_dir / "binary_varbinary_literal_conjunct_sarg.orc").string(); + write_multi_stripe_orc_binary_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_STRING); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, std::make_shared(), "zzz_1000", "value"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 50); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargBinaryCastFromVarbinaryToStringConjunctPrunesRowGroups) { + const auto multi_stripe_file_path = + (_test_dir / "binary_varbinary_to_string_cast_sarg.orc").string(); + write_multi_stripe_orc_binary_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_STRING); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, std::make_shared(), "zzz_1000", "value"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 50); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargVarcharToCharThenStringCastDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_varchar_to_char_then_string_cast.orc").string(); + write_multi_stripe_orc_varchar_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_VARCHAR); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, std::make_shared(16, TYPE_CHAR), "m", "value"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargCharCastToStringDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_char_cast_to_string.orc").string(); + write_multi_stripe_orc_char_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_CHAR); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, remove_nullable(schema[0].type), "m", "value"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargIntegerWideningCastConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_integer_widening_cast.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargIntegerWideningCastNullSafeEqualNullPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_integer_widening_cast_null_safe_equal_null.orc").string(); + write_two_stripe_orc_nullable_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < rows; ++row) { + EXPECT_TRUE(ids_nullable.is_null_at(row)); + } + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargIntegerWideningCastNullSafeEqualLiteralOnLeftPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_integer_widening_cast_null_safe_equal_literal.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 1000, true))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + EXPECT_FALSE(ids_nullable.is_null_at(row)); + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 1); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargIntegerToDoubleCastConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_integer_to_double_cast.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 500.5))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargIntegerToDoubleCastOutOfInt64BoundaryDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_integer_to_double_cast_out_of_int64_boundary.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 9223372036854775808.0))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 400); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargIntegerToFloatCastDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_integer_to_float_cast.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 500.5F))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargIntegerToFloatCastInDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_integer_to_float_cast_in.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, std::vector {1000.0F, 500.5F}))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 1); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargLongToDoubleCastDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_long_to_double_cast.orc").string(); + write_multi_stripe_orc_long_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 500.5))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargLongToIntCastDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_long_to_int_cast.orc").string(); + write_multi_stripe_orc_long_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargLongToDoubleNullSafeEqualDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_long_to_double_null_safe_equal.orc").string(); + write_multi_stripe_orc_long_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 1000.0))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 1); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargLongToDoubleCastInDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_long_to_double_cast_in.orc").string(); + write_multi_stripe_orc_long_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, std::vector {1000.0, 500.5}))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 1); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargIntegerToDoubleCastInConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_integer_to_double_cast_in.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, std::vector {1000.0, 500.5}))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 1); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargShortToFloatCastConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_short_to_float_cast.orc").string(); + write_multi_stripe_orc_short_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 500.5F))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargConjunctStructChildPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_conjunct_struct_child.orc").string(); + write_multi_stripe_orc_struct_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + ASSERT_EQ(schema[0].children.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, schema[0].type, "a", 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargConjunctStructChildWideningCastPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_conjunct_struct_child_widening_cast.orc").string(); + write_multi_stripe_orc_struct_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + ASSERT_EQ(schema[0].children.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[0].type, + "a", 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargConjunctStructChildNullSafeEqualPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_conjunct_struct_child_null_safe_equal.orc").string(); + write_multi_stripe_orc_struct_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + ASSERT_EQ(schema[0].children.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[0].type, + "a", 1000))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargConjunctStructChildComplexNullSafeEqualDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_conjunct_struct_child_complex_null_safe_equal.orc").string(); + write_multi_stripe_orc_struct_array_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + ASSERT_EQ(schema[0].children.size(), 1); + EXPECT_EQ(remove_nullable(schema[0].children[0].type)->get_primitive_type(), TYPE_ARRAY); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, schema[0].type, schema[0].children[0].type, "items", 1000))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargConjunctNestedStructChildPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_conjunct_nested_struct_child.orc").string(); + write_multi_stripe_orc_nested_struct_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + ASSERT_EQ(schema[0].children.size(), 2); + ASSERT_EQ(schema[0].children[0].children.size(), 1); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, schema[0].type, schema[0].children[0].type, "nested", "a", 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargConjunctArrayPredicateDoesNotPruneStripes) { + const auto complex_file_path = (_test_dir / "sarg_array_local_conjunct.orc").string(); + write_two_stripe_orc_array_map_file(complex_file_path); + ASSERT_EQ(get_orc_stripe_count(complex_file_path), 2); + + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_ARRAY); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[0].type, 2))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargConjunctMapPredicateDoesNotPruneStripes) { + const auto complex_file_path = (_test_dir / "sarg_map_local_conjunct.orc").string(); + write_two_stripe_orc_array_map_file(complex_file_path); + ASSERT_EQ(get_orc_stripe_count(complex_file_path), 2); + + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_MAP); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[1].type, "c"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[1]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargConjunctElementAtDoesNotPruneStripes) { + const auto complex_file_path = (_test_dir / "sarg_map_element_at_local_conjunct.orc").string(); + write_two_stripe_orc_array_map_file(complex_file_path); + ASSERT_EQ(get_orc_stripe_count(complex_file_path), 2); + + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_MAP); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[1].type, "c", 200))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[1]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargConjunctMapKeysInDoesNotPruneStripes) { + const auto complex_file_path = (_test_dir / "sarg_map_keys_in_local_conjunct.orc").string(); + write_two_stripe_orc_array_map_file(complex_file_path); + ASSERT_EQ(get_orc_stripe_count(complex_file_path), 2); + + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_MAP); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[1].type, "c"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[1]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargConjunctArraySizeDoesNotPruneStripes) { + const auto complex_file_path = (_test_dir / "sarg_array_size_local_conjunct.orc").string(); + write_two_stripe_orc_array_size_file(complex_file_path); + ASSERT_EQ(get_orc_stripe_count(complex_file_path), 2); + + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_ARRAY); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, schema[0].type, 0))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargConjunctListStructChildDoesNotPruneStripes) { + const auto complex_file_path = + (_test_dir / "sarg_array_struct_child_local_conjunct.orc").string(); + write_two_stripe_orc_array_struct_file(complex_file_path); + ASSERT_EQ(get_orc_stripe_count(complex_file_path), 2); + + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + ASSERT_EQ(schema[0].children.size(), 1); + ASSERT_EQ(schema[0].children[0].children.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, schema[0].type, schema[0].children[0].type, "a", 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargFloatWideningCastConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_float_widening_cast.orc").string(); + write_multi_stripe_orc_float_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 500.5))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDateV2CastToDateConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_date_v2_cast_to_date.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + VecDateTimeValue literal; + ASSERT_TRUE(literal.check_range_and_set_time(2020, 1, 1, 0, 0, 0, TIME_DATE)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, make_date_v2(2020, 1, 1), literal))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDateV2CastToDateTimeV2MidnightConjunctPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_date_v2_cast_to_datetime_v2_midnight.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, make_date_v2(2020, 1, 1), make_datetime_v2(2020, 1, 1)))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDateV2CastToDateTimeV2NonMidnightGreaterEqualPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_date_v2_cast_to_datetime_v2_non_midnight_ge.orc").string(); + constexpr int64_t DAYS_TO_2020_01_01 = 18262; + constexpr int64_t DAYS_TO_2021_01_01 = 18628; + write_two_stripe_constant_date_file(multi_stripe_file_path, DAYS_TO_2020_01_01, + DAYS_TO_2021_01_01); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, TExprOpcode::GE, make_datetime_v2(2020, 1, 1, 12)))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDateV2CastToDateTimeV2NonMidnightLessThanPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_date_v2_cast_to_datetime_v2_non_midnight_lt.orc").string(); + constexpr int64_t DAYS_TO_2020_01_01 = 18262; + constexpr int64_t DAYS_TO_2021_01_01 = 18628; + write_two_stripe_constant_date_file(multi_stripe_file_path, DAYS_TO_2020_01_01, + DAYS_TO_2021_01_01); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, TExprOpcode::LT, make_datetime_v2(2020, 1, 1, 12)))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDateV2CastToDateTimeV2InPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_date_v2_cast_to_datetime_v2_in.orc").string(); + constexpr int64_t DAYS_TO_2020_01_01 = 18262; + constexpr int64_t DAYS_TO_2021_01_01 = 18628; + write_two_stripe_constant_date_file(multi_stripe_file_path, DAYS_TO_2020_01_01, + DAYS_TO_2021_01_01); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, std::vector> { + make_datetime_v2(2021, 1, 1), make_datetime_v2(2020, 1, 1, 12)}))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDateV2CastToDateTimeMidnightConjunctPrunesStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_date_v2_cast_to_datetime_midnight.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + VecDateTimeValue literal; + ASSERT_TRUE(literal.check_range_and_set_time(2020, 1, 1, 0, 0, 0, TIME_DATETIME)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, make_date_v2(2020, 1, 1), literal))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDateConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_date.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + const auto literal = Field::create_field(make_date_v2(2020, 1, 1)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[0].type), literal, "date_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDateCastToStringDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_date_cast_to_string.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, "2020-01-01", make_date_v2(2020, 1, 1)))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargTimestampConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_timestamp.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + const auto literal = Field::create_field(make_datetime_v2(2020, 1, 1)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[1].type), literal, "timestamp_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[1]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargTimestampInstantConjunctUsesSessionTimezone) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_timestamp_instant_timezone.orc").string(); + write_multi_stripe_orc_timestamp_instant_sarg_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("Asia/Shanghai"); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + const auto literal = + Field::create_field(make_datetime_v2(2021, 1, 1, 7, 59, 59)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[0].type), literal, "timestamp_instant_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargTimestampInstantRepeatedCivilTimeDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_timestamp_instant_dst_rollback.orc").string(); + // These UTC instants both decode to 2021-11-07 01:30:00.123 in America/New_York, + // once before and once after the UTC-04:00 to UTC-05:00 rollback. + write_multi_stripe_orc_timestamp_instant_sarg_file(multi_stripe_file_path, 1636263000, + 1636266600); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("America/New_York"); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + const auto literal = + Field::create_field(make_datetime_v2(2021, 11, 7, 1, 30, 0, 123000)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[0].type), literal, "timestamp_instant_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 2); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargTimestampLowerPrecisionCastDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_timestamp_lower_precision_cast.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, remove_nullable(schema[1].type), make_datetime_v2(2020, 1, 1)))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[1]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargDecimalConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_decimal.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + const auto literal = Field::create_field(Decimal128V3(50000)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(2)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[2].type), literal, "decimal_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[2]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDecimalDifferentScaleConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_decimal_different_scale.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + const auto literal = Field::create_field(Decimal128V3(5000000)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(2)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, remove_nullable(schema[2].type), std::make_shared(14, 4), + literal, Decimal128V3(50000), "decimal_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[2]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDecimalWideningCastConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_decimal_widening_cast.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + const auto cast_type = std::make_shared(14, 4); + const auto literal = Field::create_field(Decimal128V3(5000000)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(2)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, remove_nullable(schema[2].type), cast_type, literal, Decimal128V3(50000), + "decimal_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[2]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargDecimalScaleReducingCastDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_decimal_scale_reducing_cast.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + const auto cast_type = std::make_shared(12, 1); + const auto literal = Field::create_field(Decimal128V3(5000)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(2)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, remove_nullable(schema[2].type), cast_type, literal, Decimal128V3(50000), + "decimal_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[2]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargDecimalCastToStringDoesNotPruneStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_decimal_cast_to_string.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(2)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, remove_nullable(schema[2].type), "1.99", "decimal_col", 2))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[2]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + +TEST_F(NewOrcReaderTest, SargTimestampInListConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_timestamp_in.orc").string(); + write_multi_stripe_orc_sarg_types_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[1].type), + std::vector {Field::create_field( + make_datetime_v2(2021, 1, 1, 0, 0, 0, 123000))}, + "timestamp_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[1]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargTimestampNotInListConjunctPrunesStripes) { + const auto multi_stripe_file_path = (_test_dir / "sarg_timestamp_not_in.orc").string(); + write_two_stripe_constant_timestamp_file(multi_stripe_file_path, 0, 1609459200); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[0].type), + std::vector {Field::create_field( + make_datetime_v2(1970, 1, 1, 0, 0, 0, 123000))}, + "timestamp_col", true))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 200); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, SargConjunctPruningPreservesRowPosition) { + const auto multi_stripe_file_path = (_test_dir / "row_position_after_pruning.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + const auto row_position_column_id = format::ROW_POSITION_COLUMN_ID; + std::vector block_schema { + format::orc::OrcReader::row_position_column_definition(), schema[0]}; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(row_position_column_id)}; + request->local_positions = { + {format::LocalColumnId(row_position_column_id), format::LocalIndex(0)}, + {format::LocalColumnId(0), format::LocalIndex(1)}}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(1, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_row_positions; + std::vector result_ids; + while (!eof) { + Block block = build_file_block(block_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& row_positions = int64_data_column(*block.get_by_position(0).column); + const auto& ids_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_row_positions.push_back(row_positions.get_element(row)); + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_row_positions.size(), 200); + ASSERT_EQ(result_ids.size(), 200); + EXPECT_EQ(result_row_positions.front(), 200); + EXPECT_EQ(result_row_positions.back(), 399); + EXPECT_EQ(result_ids.front(), 1000); + EXPECT_EQ(result_ids.back(), 1199); +} + +TEST_F(NewOrcReaderTest, SargConjunctReadsNonAdjacentStripeRangesAfterPruning) { + const auto multi_stripe_file_path = (_test_dir / "non_adjacent_stripes.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path, {1, 1000, 201}); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 3); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 500))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + std::vector result_ids; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result_ids.push_back(ids.get_element(row)); + } + } + + ASSERT_EQ(result_ids.size(), 400); + EXPECT_EQ(result_ids.front(), 1); + EXPECT_EQ(result_ids[199], 200); + EXPECT_EQ(result_ids[200], 201); + EXPECT_EQ(result_ids.back(), 400); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); +} + +TEST_F(NewOrcReaderTest, ConditionCacheMissExtendsAcrossNonAdjacentStripeRanges) { + constexpr int64_t ROWS_PER_STRIPE = 2500; + const auto multi_stripe_file_path = + (_test_dir / "condition_cache_non_adjacent_stripes.orc").string(); + write_multi_stripe_orc_int_only_file(multi_stripe_file_path, {1, 10000, 3000}, ROWS_PER_STRIPE); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 3); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 5000))); + ASSERT_TRUE(reader->open(request).ok()); + EXPECT_EQ(reader->get_total_rows(), ROWS_PER_STRIPE); + + auto cache_ctx = std::make_shared(); + cache_ctx->filter_result = std::make_shared>( + (reader->get_total_rows() + ConditionCacheContext::GRANULE_SIZE - 1) / + ConditionCacheContext::GRANULE_SIZE + + 1, + false); + reader->set_condition_cache_context(cache_ctx); + EXPECT_EQ(cache_ctx->base_granule, 0); + EXPECT_EQ(cache_ctx->num_granules, 2); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 4500); + ASSERT_NE(cache_ctx->filter_result, nullptr); + EXPECT_EQ(cache_ctx->num_granules, 4); + ASSERT_EQ(cache_ctx->filter_result->size(), 4); + EXPECT_TRUE((*cache_ctx->filter_result)[0]); + EXPECT_TRUE((*cache_ctx->filter_result)[1]); + EXPECT_TRUE((*cache_ctx->filter_result)[2]); + EXPECT_TRUE((*cache_ctx->filter_result)[3]); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, ROWS_PER_STRIPE); +} + +TEST_F(NewOrcReaderTest, SargConjunctReturnsEofWhenAllStripesArePruned) { + const auto multi_stripe_file_path = (_test_dir / "all_pruned_stripes.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 5000))); + ASSERT_TRUE(reader->open(request).ok()); + // TableReader uses this value to decide whether a condition-cache MISS bitmap should be + // created. When SARG pruning removes every stripe, there is no row-reader range to cache. + EXPECT_EQ(reader->get_total_rows(), 0); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 2); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 2); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 400); +} + +TEST_F(NewOrcReaderTest, CloseClearsFileLocalState) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_FALSE(schema.empty()); + + ASSERT_TRUE(reader->close().ok()); + ASSERT_TRUE(reader->close().ok()); + + schema.clear(); + EXPECT_FALSE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + EXPECT_FALSE(reader->open(request).ok()); +} + +TEST_F(NewOrcReaderTest, ReadPrimitiveTypesWithNulls) { + const auto primitive_file_path = (_test_dir / "primitive.orc").string(); + write_primitive_orc_file(primitive_file_path); + auto reader = create_reader_for_path(primitive_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("UTC"); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 16); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_BOOLEAN); + EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_TINYINT); + EXPECT_EQ(remove_nullable(schema[2].type)->get_primitive_type(), TYPE_SMALLINT); + EXPECT_EQ(remove_nullable(schema[3].type)->get_primitive_type(), TYPE_INT); + EXPECT_EQ(remove_nullable(schema[4].type)->get_primitive_type(), TYPE_BIGINT); + EXPECT_EQ(remove_nullable(schema[5].type)->get_primitive_type(), TYPE_FLOAT); + EXPECT_EQ(remove_nullable(schema[6].type)->get_primitive_type(), TYPE_DOUBLE); + EXPECT_EQ(remove_nullable(schema[7].type)->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(remove_nullable(schema[8].type)->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(remove_nullable(schema[9].type)->get_primitive_type(), TYPE_VARCHAR); + EXPECT_EQ(remove_nullable(schema[10].type)->get_primitive_type(), TYPE_CHAR); + EXPECT_EQ(remove_nullable(schema[11].type)->get_primitive_type(), TYPE_DATEV2); + EXPECT_EQ(remove_nullable(schema[12].type)->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_EQ(remove_nullable(schema[13].type)->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_EQ(remove_nullable(schema[14].type)->get_primitive_type(), TYPE_DECIMAL128I); + EXPECT_EQ(remove_nullable(schema[15].type)->get_primitive_type(), TYPE_DECIMAL128I); + + Block block = build_file_block(schema); + auto request = std::make_shared(); + for (size_t column_id = 0; column_id < schema.size(); ++column_id) { + request->non_predicate_columns.push_back(field_projection(static_cast(column_id))); + } + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, PRIMITIVE_ROW_COUNT); + + for (size_t column_id = 0; column_id < schema.size(); ++column_id) { + const auto& nullable_column = + assert_cast(*block.get_by_position(column_id).column); + ASSERT_EQ(nullable_column.size(), PRIMITIVE_ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(NULL_ROW)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + } + + const auto& bool_values = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + const auto& byte_values = assert_cast( + assert_cast(*block.get_by_position(1).column) + .get_nested_column()); + const auto& short_values = assert_cast( + assert_cast(*block.get_by_position(2).column) + .get_nested_column()); + const auto& int_values = assert_cast( + assert_cast(*block.get_by_position(3).column) + .get_nested_column()); + const auto& long_values = assert_cast( + assert_cast(*block.get_by_position(4).column) + .get_nested_column()); + const auto& float_values = assert_cast( + assert_cast(*block.get_by_position(5).column) + .get_nested_column()); + const auto& double_values = assert_cast( + assert_cast(*block.get_by_position(6).column) + .get_nested_column()); + const auto& string_values = assert_cast( + assert_cast(*block.get_by_position(7).column) + .get_nested_column()); + const auto& binary_values = assert_cast( + assert_cast(*block.get_by_position(8).column) + .get_nested_column()); + const auto& varchar_values = assert_cast( + assert_cast(*block.get_by_position(9).column) + .get_nested_column()); + const auto& char_values = assert_cast( + assert_cast(*block.get_by_position(10).column) + .get_nested_column()); + + EXPECT_EQ(bool_values.get_element(0), 1); + EXPECT_EQ(bool_values.get_element(2), 0); + EXPECT_EQ(byte_values.get_element(0), -7); + EXPECT_EQ(byte_values.get_element(2), 8); + EXPECT_EQ(short_values.get_element(0), -300); + EXPECT_EQ(short_values.get_element(2), 301); + EXPECT_EQ(int_values.get_element(0), -70000); + EXPECT_EQ(int_values.get_element(2), 70001); + EXPECT_EQ(long_values.get_element(0), -9000000000L); + EXPECT_EQ(long_values.get_element(2), 9000000001L); + EXPECT_FLOAT_EQ(float_values.get_element(0), 1.25F); + EXPECT_FLOAT_EQ(float_values.get_element(2), -2.5F); + EXPECT_DOUBLE_EQ(double_values.get_element(0), 10.5); + EXPECT_DOUBLE_EQ(double_values.get_element(2), -20.25); + EXPECT_EQ(string_values.get_data_at(0).to_string(), "alpha"); + EXPECT_EQ(string_values.get_data_at(2).to_string(), "gamma"); + EXPECT_EQ(binary_values.get_data_at(0).to_string(), "bin_a"); + EXPECT_EQ(binary_values.get_data_at(2).to_string(), "bin_c"); + EXPECT_EQ(varchar_values.get_data_at(0).to_string(), "varchar"); + EXPECT_EQ(varchar_values.get_data_at(2).to_string(), "tail"); + EXPECT_EQ(char_values.get_data_at(0).to_string(), "ab"); + EXPECT_EQ(char_values.get_data_at(2).to_string(), "xy"); + EXPECT_EQ(schema[11].type->to_string(*block.get_by_position(11).column, 0), "1970-01-01"); + EXPECT_EQ(schema[11].type->to_string(*block.get_by_position(11).column, 2), "2021-01-01"); + EXPECT_EQ(schema[12].type->to_string(*block.get_by_position(12).column, 0), + "1970-01-02 00:00:01.234567"); + EXPECT_EQ(schema[12].type->to_string(*block.get_by_position(12).column, 2), + "2021-01-01 00:00:00.123456"); + EXPECT_EQ(schema[13].type->to_string(*block.get_by_position(13).column, 0), + "1970-01-01 00:00:02.345678"); + EXPECT_EQ(schema[13].type->to_string(*block.get_by_position(13).column, 2), + "2021-01-01 00:00:01.654321"); + EXPECT_EQ(schema[14].type->to_string(*block.get_by_position(14).column, 0), "123.45"); + EXPECT_EQ(schema[14].type->to_string(*block.get_by_position(14).column, 2), "-67.00"); + EXPECT_EQ(schema[15].type->to_string(*block.get_by_position(15).column, 0), + "123456789012.345678"); + EXPECT_EQ(schema[15].type->to_string(*block.get_by_position(15).column, 2), + "-987654321.000000"); + EXPECT_EQ(schema[0].type->to_string(*block.get_by_position(0).column, NULL_ROW), "NULL"); + EXPECT_EQ(schema[11].type->to_string(*block.get_by_position(11).column, NULL_ROW), "NULL"); + EXPECT_EQ(schema[15].type->to_string(*block.get_by_position(15).column, NULL_ROW), "NULL"); +} + +TEST_F(NewOrcReaderTest, ReadHive11DecimalUsesBatchScale) { + const auto file_path = find_repo_file( + "contrib/apache-orc/java/core/src/test/resources/orc-file-11-format.orc"); + ASSERT_TRUE(std::filesystem::exists(file_path)) << file_path; + auto reader = create_reader_for_path(file_path.string()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + const auto decimal_it = std::ranges::find_if( + schema, [](const format::ColumnDefinition& field) { return field.name == "decimal1"; }); + ASSERT_NE(decimal_it, schema.end()); + + Block block = build_file_block({*decimal_it}); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(decimal_it->file_local_id())}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_GE(rows, 1); + EXPECT_EQ(decimal_it->type->to_string(*block.get_by_position(0).column, 0), + "12345678.6547450000"); +} + +TEST_F(NewOrcReaderTest, TimestampInstantWithoutMappingUsesSessionTimezone) { + const auto primitive_file_path = (_test_dir / "timestamp_instant_datetime.orc").string(); + write_primitive_orc_file(primitive_file_path); + auto reader = create_reader_for_path(primitive_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("Asia/Shanghai"); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 16); + ASSERT_EQ(remove_nullable(schema[13].type)->get_primitive_type(), TYPE_DATETIMEV2); + + Block block = build_file_block({schema[13]}); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(13)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, PRIMITIVE_ROW_COUNT); + EXPECT_EQ(schema[13].type->to_string(*block.get_by_position(0).column, 0), + "1970-01-01 08:00:02.345678"); + EXPECT_EQ(schema[13].type->to_string(*block.get_by_position(0).column, 2), + "2021-01-01 08:00:01.654321"); +} + +TEST_F(NewOrcReaderTest, TimestampInstantWithMappingReadsTimestampTz) { + const auto primitive_file_path = (_test_dir / "timestamp_instant_tz.orc").string(); + write_primitive_orc_file(primitive_file_path); + auto reader = create_reader_for_path(primitive_file_path, nullptr, nullptr, std::nullopt, true); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("Asia/Shanghai"); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 16); + ASSERT_EQ(remove_nullable(schema[13].type)->get_primitive_type(), TYPE_TIMESTAMPTZ); + ASSERT_EQ(remove_nullable(schema[13].type)->get_scale(), 6); + + Block block = build_file_block({schema[13]}); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(13)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, PRIMITIVE_ROW_COUNT); + + const auto& timestamp_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& timestamp_values = + assert_cast(timestamp_nullable.get_nested_column()); + EXPECT_FALSE(timestamp_nullable.is_null_at(0)); + EXPECT_TRUE(timestamp_nullable.is_null_at(NULL_ROW)); + EXPECT_EQ(timestamp_values.get_element(0).to_string(state.timezone_obj(), 6), + "1970-01-01 08:00:02.345678+08:00"); + EXPECT_EQ(timestamp_values.get_element(2).to_string(state.timezone_obj(), 6), + "2021-01-01 08:00:01.654321+08:00"); + EXPECT_EQ(timestamp_values.get_element(0).to_string(cctz::utc_time_zone(), 6), + "1970-01-01 00:00:02.345678+00:00"); +} + +TEST_F(NewOrcReaderTest, ReadsProjectedColumnIntoRequestedBlockPosition) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block; + block.insert({schema[1].type->create_column(), schema[1].type, schema[1].name}); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(1)}; + request->local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, ROW_COUNT); + + const auto& values_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& values = assert_cast(values_nullable.get_nested_column()); + EXPECT_EQ(values.get_data_at(0).to_string(), "one"); + EXPECT_EQ(values.get_data_at(4).to_string(), "five"); +} + +TEST_F(NewOrcReaderTest, ReadComplexTypes) { + const auto complex_file_path = (_test_dir / "complex.orc").string(); + write_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 5); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_STRUCT); + EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_ARRAY); + EXPECT_EQ(remove_nullable(schema[2].type)->get_primitive_type(), TYPE_MAP); + EXPECT_EQ(remove_nullable(schema[3].type)->get_primitive_type(), TYPE_ARRAY); + EXPECT_EQ(remove_nullable(schema[4].type)->get_primitive_type(), TYPE_MAP); + ASSERT_EQ(schema[0].children.size(), 2); + ASSERT_EQ(schema[1].children.size(), 1); + ASSERT_EQ(schema[2].children.size(), 2); + ASSERT_EQ(schema[3].children.size(), 1); + ASSERT_EQ(schema[3].children[0].children.size(), 2); + ASSERT_EQ(schema[4].children.size(), 2); + ASSERT_EQ(schema[4].children[1].children.size(), 2); + EXPECT_EQ(schema[0].children[0].local_id, 0); + EXPECT_EQ(schema[1].children[0].local_id, 0); + EXPECT_EQ(schema[2].children[0].local_id, 0); + EXPECT_EQ(schema[2].children[1].local_id, 1); + + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1), field_projection(2), + field_projection(3), field_projection(4)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, COMPLEX_ROW_COUNT); + + const auto& struct_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& struct_column = + assert_cast(struct_nullable.get_nested_column()); + ASSERT_EQ(struct_column.tuple_size(), 2); + const auto& struct_a = assert_cast( + assert_cast(struct_column.get_column(0)).get_nested_column()); + const auto& struct_b = assert_cast( + assert_cast(struct_column.get_column(1)).get_nested_column()); + EXPECT_EQ(struct_a.get_element(0), 10); + EXPECT_EQ(struct_a.get_element(2), 30); + EXPECT_EQ(struct_b.get_data_at(0).to_string(), "ten"); + EXPECT_EQ(struct_b.get_data_at(2).to_string(), "thirty"); + + const auto& array_nullable = + assert_cast(*block.get_by_position(1).column); + const auto& array_column = assert_cast(array_nullable.get_nested_column()); + ASSERT_EQ(array_column.get_offsets().size(), COMPLEX_ROW_COUNT); + EXPECT_EQ(array_column.get_offsets()[0], 1); + EXPECT_EQ(array_column.get_offsets()[1], 1); + EXPECT_EQ(array_column.get_offsets()[2], 3); + const auto& array_values = assert_cast( + assert_cast(array_column.get_data()).get_nested_column()); + ASSERT_EQ(array_values.size(), 3); + EXPECT_EQ(array_values.get_element(0), 1); + EXPECT_EQ(array_values.get_element(2), 3); + + const auto& map_nullable = assert_cast(*block.get_by_position(2).column); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), COMPLEX_ROW_COUNT); + EXPECT_EQ(map_column.get_offsets()[0], 2); + EXPECT_EQ(map_column.get_offsets()[1], 2); + EXPECT_EQ(map_column.get_offsets()[2], 3); + const auto& map_keys = assert_cast( + assert_cast(map_column.get_keys()).get_nested_column()); + const auto& map_values = assert_cast( + assert_cast(map_column.get_values()).get_nested_column()); + EXPECT_EQ(map_keys.get_data_at(0).to_string(), "a"); + EXPECT_EQ(map_keys.get_data_at(2).to_string(), "c"); + EXPECT_EQ(map_values.get_element(0), 100); + EXPECT_EQ(map_values.get_element(2), 300); +} + +TEST_F(NewOrcReaderTest, ReadMapDecimalDateWithCenturyBoundary) { + const auto file_path = (_test_dir / "map_decimal_date.orc").string(); + write_map_decimal_date_orc_file(file_path); + auto reader = create_reader_for_path(file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + ASSERT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_MAP); + ASSERT_EQ(schema[1].children.size(), 2); + + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 4); + + const auto& map_nullable = assert_cast(*block.get_by_position(1).column); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), 4); + ASSERT_EQ(map_column.get_values().size(), 4); + EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 0), "1900-01-01"); + EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 1), "9999-12-31"); + EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 2), "0000-12-29"); + EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 3), "1000-10-16"); +} + +TEST_F(NewOrcReaderTest, ReadDeepNestedComplexTypes) { + const auto complex_file_path = (_test_dir / "deep_complex.orc").string(); + write_deep_nested_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_INT); + EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(schema[1].children.size(), 1); + ASSERT_EQ(schema[1].children[0].children.size(), 2); + ASSERT_EQ(schema[1].children[0].children[1].children.size(), 2); + ASSERT_EQ(schema[1].children[0].children[1].children[1].children.size(), 1); + ASSERT_EQ(schema[1].children[0].children[1].children[1].children[0].children.size(), 2); + + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, DEEP_NESTED_ROW_COUNT); + + const auto& ids_nullable = assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + expect_int32_values(ids, {1, 2, 3, 4}); + + expect_deep_nested_column( + *block.get_by_position(1).column, {false, true, false, false}, {1, 1, 3, 4}, + {"row1", "row3_left", "row3_right", "row4"}, {1, 2, 4, 5}, + {"row1_key", "row3_left_key", "row3_right_a", "row3_right_empty", "row4_key"}, + {2, 3, 5, 5, 6}, {10, 11, 30, 31, 32, 40}, + {"ten", "eleven", "thirty", "thirty_one", "thirty_two", "forty"}); +} + +TEST_F(NewOrcReaderTest, ReadProjectedStructChildren) { + const auto complex_file_path = (_test_dir / "complex_struct_projection.orc").string(); + write_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema[0].children.size(), 2); + + DataTypes child_types {schema[0].children[0].type}; + Strings child_names {schema[0].children[0].name}; + auto projected_type = make_nullable(std::make_shared(child_types, child_names)); + Block block; + block.insert({projected_type->create_column(), projected_type, schema[0].name}); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + auto projection = make_projection(schema[0], false); + projection.children.push_back(make_projection(schema[0].children[0])); + request->non_predicate_columns[0] = std::move(projection); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, COMPLEX_ROW_COUNT); + + const auto& struct_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + ASSERT_EQ(struct_column.tuple_size(), 1); + const auto& struct_a = assert_cast( + assert_cast(struct_column.get_column(0)).get_nested_column()); + EXPECT_EQ(struct_a.get_element(0), 10); + EXPECT_EQ(struct_a.get_element(1), 20); + EXPECT_EQ(struct_a.get_element(2), 30); +} + +TEST_F(NewOrcReaderTest, ReadProjectedListElementStructChildren) { + const auto complex_file_path = (_test_dir / "complex_list_projection.orc").string(); + write_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema[3].children.size(), 1); + ASSERT_EQ(schema[3].children[0].children.size(), 2); + + DataTypes projected_element_child_types {schema[3].children[0].children[0].type}; + Strings projected_element_child_names {schema[3].children[0].children[0].name}; + auto projected_element_type = make_nullable(std::make_shared( + projected_element_child_types, projected_element_child_names)); + auto projected_type = make_nullable(std::make_shared(projected_element_type)); + Block block; + block.insert({projected_type->create_column(), projected_type, schema[3].name}); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(3)}; + request->local_positions.emplace(format::LocalColumnId(3), format::LocalIndex(0)); + auto projection = make_projection(schema[3], false); + auto element_projection = make_projection(schema[3].children[0], false); + element_projection.children.push_back(make_projection(schema[3].children[0].children[0])); + projection.children.push_back(std::move(element_projection)); + request->non_predicate_columns[0] = std::move(projection); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, COMPLEX_ROW_COUNT); + + const auto& array_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + ASSERT_EQ(array_column.get_offsets().size(), COMPLEX_ROW_COUNT); + EXPECT_EQ(array_column.get_offsets()[0], 1); + EXPECT_EQ(array_column.get_offsets()[1], 3); + EXPECT_EQ(array_column.get_offsets()[2], 3); + + const auto& element_nullable = assert_cast(array_column.get_data()); + const auto& element_struct = + assert_cast(element_nullable.get_nested_column()); + ASSERT_EQ(element_struct.tuple_size(), 1); + const auto& element_a = assert_cast( + assert_cast(element_struct.get_column(0)).get_nested_column()); + ASSERT_EQ(element_a.size(), 3); + EXPECT_EQ(element_a.get_element(0), 1); + EXPECT_EQ(element_a.get_element(1), 2); + EXPECT_EQ(element_a.get_element(2), 3); +} + +TEST_F(NewOrcReaderTest, ReadProjectedMapValueStructChildren) { + const auto complex_file_path = (_test_dir / "complex_map_projection.orc").string(); + write_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema[4].children.size(), 2); + ASSERT_EQ(schema[4].children[1].children.size(), 2); + + const auto& key_field = schema[4].children[0]; + const auto& value_field = schema[4].children[1]; + DataTypes projected_value_child_types {value_field.children[0].type}; + Strings projected_value_child_names {value_field.children[0].name}; + auto projected_value_type = make_nullable(std::make_shared( + projected_value_child_types, projected_value_child_names)); + auto projected_type = + make_nullable(std::make_shared(key_field.type, projected_value_type)); + Block block; + block.insert({projected_type->create_column(), projected_type, schema[4].name}); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(4)}; + request->local_positions.emplace(format::LocalColumnId(4), format::LocalIndex(0)); + auto projection = make_projection(schema[4], false); + projection.children.push_back(make_projection(key_field)); + auto value_projection = make_projection(value_field, false); + value_projection.children.push_back(make_projection(value_field.children[0])); + projection.children.push_back(std::move(value_projection)); + request->non_predicate_columns[0] = std::move(projection); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, COMPLEX_ROW_COUNT); + + const auto& map_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), COMPLEX_ROW_COUNT); + EXPECT_EQ(map_column.get_offsets()[0], 1); + EXPECT_EQ(map_column.get_offsets()[1], 1); + EXPECT_EQ(map_column.get_offsets()[2], 2); + + const auto& keys = assert_cast( + assert_cast(map_column.get_keys()).get_nested_column()); + EXPECT_EQ(keys.get_data_at(0).to_string(), "first"); + EXPECT_EQ(keys.get_data_at(1).to_string(), "second"); + + const auto& value_nullable = assert_cast(map_column.get_values()); + const auto& value_struct = assert_cast(value_nullable.get_nested_column()); + ASSERT_EQ(value_struct.tuple_size(), 1); + const auto& value_a = assert_cast( + assert_cast(value_struct.get_column(0)).get_nested_column()); + ASSERT_EQ(value_a.size(), 2); + EXPECT_EQ(value_a.get_element(0), 7); + EXPECT_EQ(value_a.get_element(1), 8); +} + +TEST_F(NewOrcReaderTest, ReadProjectedNestedStructChildFromHiveOrcFixture) { + const auto archive = find_repo_file( + "docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/" + "orc_nested_types/data.tar.gz"); + ASSERT_TRUE(std::filesystem::exists(archive)) << archive; + const auto extract_dir = _test_dir / "orc_nested_types"; + std::filesystem::create_directories(extract_dir); + const std::string extract_cmd = + "tar -xzf '" + archive.string() + "' -C '" + extract_dir.string() + "'"; + ASSERT_EQ(std::system(extract_cmd.c_str()), 0); + const auto file_path = + extract_dir / + "data/nested_types1_orc/part-00001-af6ae8cd-ef39-4a9b-a809-bcbd8aec7ccb-c000." + "snappy.orc"; + ASSERT_TRUE(std::filesystem::exists(file_path)) << file_path; + + auto reader = create_reader_for_path(file_path.string()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 9); + const auto& complex_struct = schema[8]; + ASSERT_EQ(complex_struct.name, "complex_struct_col"); + ASSERT_EQ(complex_struct.children.size(), 3); + const auto& c_field = complex_struct.children[2]; + ASSERT_EQ(c_field.name, "c"); + + auto projected_type = make_nullable( + std::make_shared(DataTypes {c_field.type}, Strings {c_field.name})); + Block block; + block.insert({projected_type->create_column(), projected_type, complex_struct.name}); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(complex_struct.file_local_id())}; + request->local_positions.emplace(format::LocalColumnId(complex_struct.file_local_id()), + format::LocalIndex(0)); + auto projection = make_projection(complex_struct, false); + projection.children.push_back(make_projection(c_field)); + request->non_predicate_columns[0] = std::move(projection); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& root_struct = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + ASSERT_EQ(root_struct.tuple_size(), 1); + const auto& c_nullable = assert_cast(root_struct.get_column(0)); + EXPECT_FALSE(c_nullable.is_null_at(0)); + const auto& c_struct = assert_cast(c_nullable.get_nested_column()); + ASSERT_EQ(c_struct.tuple_size(), 2); + const auto& y_nullable = assert_cast(c_struct.get_column(1)); + const auto& y_values = assert_cast(y_nullable.get_nested_column()); + EXPECT_FALSE(y_nullable.is_null_at(0)); + EXPECT_EQ(y_values.get_data_at(0).to_string(), "Hello"); +} + +TEST_F(NewOrcReaderTest, ReadLazyProjectedNestedStructChildFromHiveOrcFixture) { + const auto archive = find_repo_file( + "docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/" + "orc_nested_types/data.tar.gz"); + ASSERT_TRUE(std::filesystem::exists(archive)) << archive; + const auto extract_dir = _test_dir / "lazy_orc_nested_types"; + std::filesystem::create_directories(extract_dir); + const std::string extract_cmd = + "tar -xzf '" + archive.string() + "' -C '" + extract_dir.string() + "'"; + ASSERT_EQ(std::system(extract_cmd.c_str()), 0); + const auto file_path = + extract_dir / + "data/nested_types1_orc/part-00001-af6ae8cd-ef39-4a9b-a809-bcbd8aec7ccb-c000." + "snappy.orc"; + ASSERT_TRUE(std::filesystem::exists(file_path)) << file_path; + + auto reader = create_reader_for_path(file_path.string()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 9); + const auto& id_field = schema[0]; + const auto& complex_struct = schema[8]; + ASSERT_EQ(complex_struct.name, "complex_struct_col"); + ASSERT_EQ(complex_struct.children.size(), 3); + const auto& c_field = complex_struct.children[2]; + ASSERT_EQ(c_field.name, "c"); + + auto projected_type = make_nullable( + std::make_shared(DataTypes {c_field.type}, Strings {c_field.name})); + Block block = build_file_block({id_field}); + block.insert({projected_type->create_column(), projected_type, complex_struct.name}); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(id_field.file_local_id())}; + request->non_predicate_columns = {field_projection(complex_struct.file_local_id())}; + request->local_positions.emplace(format::LocalColumnId(id_field.file_local_id()), + format::LocalIndex(0)); + request->local_positions.emplace(format::LocalColumnId(complex_struct.file_local_id()), + format::LocalIndex(1)); + auto projection = make_projection(complex_struct, false); + projection.children.push_back(make_projection(c_field)); + request->non_predicate_columns[0] = std::move(projection); + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared(0, 3))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 1); + + const auto& root_struct = assert_cast( + assert_cast(*block.get_by_position(1).column) + .get_nested_column()); + ASSERT_EQ(root_struct.tuple_size(), 1); + const auto& c_nullable = assert_cast(root_struct.get_column(0)); + EXPECT_FALSE(c_nullable.is_null_at(0)); + const auto& c_struct = assert_cast(c_nullable.get_nested_column()); + ASSERT_EQ(c_struct.tuple_size(), 2); + const auto& y_nullable = assert_cast(c_struct.get_column(1)); + const auto& y_values = assert_cast(y_nullable.get_nested_column()); + EXPECT_FALSE(y_nullable.is_null_at(0)); + EXPECT_EQ(y_values.get_data_at(0).to_string(), "Hello"); + EXPECT_EQ(reader->reader_statistics().lazy_read_filtered_rows, 2); +} + +TEST_F(NewOrcReaderTest, ReadProjectedComplexChildrenWithNulls) { + const auto complex_file_path = (_test_dir / "nullable_complex_projection.orc").string(); + write_nullable_complex_orc_file(complex_file_path); + auto reader = create_reader_for_path(complex_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + ASSERT_EQ(schema[0].children.size(), 2); + ASSERT_EQ(schema[1].children.size(), 1); + ASSERT_EQ(schema[1].children[0].children.size(), 2); + ASSERT_EQ(schema[2].children.size(), 2); + ASSERT_EQ(schema[2].children[1].children.size(), 2); + + DataTypes projected_struct_child_types {schema[0].children[1].type}; + Strings projected_struct_child_names {schema[0].children[1].name}; + auto projected_struct_type = make_nullable(std::make_shared( + projected_struct_child_types, projected_struct_child_names)); + + DataTypes projected_element_child_types {schema[1].children[0].children[0].type}; + Strings projected_element_child_names {schema[1].children[0].children[0].name}; + auto projected_element_type = make_nullable(std::make_shared( + projected_element_child_types, projected_element_child_names)); + auto projected_array_type = + make_nullable(std::make_shared(projected_element_type)); + + const auto& key_field = schema[2].children[0]; + const auto& value_field = schema[2].children[1]; + DataTypes projected_value_child_types {value_field.children[0].type}; + Strings projected_value_child_names {value_field.children[0].name}; + auto projected_value_type = make_nullable(std::make_shared( + projected_value_child_types, projected_value_child_names)); + auto projected_map_type = + make_nullable(std::make_shared(key_field.type, projected_value_type)); + + Block block; + block.insert({projected_struct_type->create_column(), projected_struct_type, schema[0].name}); + block.insert({projected_array_type->create_column(), projected_array_type, schema[1].name}); + block.insert({projected_map_type->create_column(), projected_map_type, schema[2].name}); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1), + field_projection(2)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); + request->local_positions.emplace(format::LocalColumnId(2), format::LocalIndex(2)); + + auto struct_projection = make_projection(schema[0], false); + struct_projection.children.push_back(make_projection(schema[0].children[1])); + request->non_predicate_columns[0] = std::move(struct_projection); + + auto array_projection = make_projection(schema[1], false); + auto element_projection = make_projection(schema[1].children[0], false); + element_projection.children.push_back(make_projection(schema[1].children[0].children[0])); + array_projection.children.push_back(std::move(element_projection)); + request->non_predicate_columns[1] = std::move(array_projection); + + auto map_projection = make_projection(schema[2], false); + map_projection.children.push_back(make_projection(key_field)); + auto value_projection = make_projection(value_field, false); + value_projection.children.push_back(make_projection(value_field.children[0])); + map_projection.children.push_back(std::move(value_projection)); + request->non_predicate_columns[2] = std::move(map_projection); + + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, COMPLEX_ROW_COUNT); + + const auto& struct_nullable = + assert_cast(*block.get_by_position(0).column); + EXPECT_FALSE(struct_nullable.is_null_at(0)); + EXPECT_TRUE(struct_nullable.is_null_at(1)); + EXPECT_FALSE(struct_nullable.is_null_at(2)); + const auto& struct_column = + assert_cast(struct_nullable.get_nested_column()); + ASSERT_EQ(struct_column.tuple_size(), 1); + const auto& struct_b = assert_cast(struct_column.get_column(0)); + const auto& struct_b_values = assert_cast(struct_b.get_nested_column()); + EXPECT_FALSE(struct_b.is_null_at(0)); + EXPECT_TRUE(struct_b.is_null_at(2)); + EXPECT_EQ(struct_b_values.get_data_at(0).to_string(), "ten"); + + const auto& array_nullable = + assert_cast(*block.get_by_position(1).column); + EXPECT_FALSE(array_nullable.is_null_at(0)); + EXPECT_TRUE(array_nullable.is_null_at(1)); + EXPECT_FALSE(array_nullable.is_null_at(2)); + const auto& array_column = assert_cast(array_nullable.get_nested_column()); + ASSERT_EQ(array_column.get_offsets().size(), COMPLEX_ROW_COUNT); + EXPECT_EQ(array_column.get_offsets()[0], 1); + EXPECT_EQ(array_column.get_offsets()[1], 1); + EXPECT_EQ(array_column.get_offsets()[2], 2); + const auto& element_struct = assert_cast( + assert_cast(array_column.get_data()).get_nested_column()); + ASSERT_EQ(element_struct.tuple_size(), 1); + const auto& element_a = assert_cast( + assert_cast(element_struct.get_column(0)).get_nested_column()); + ASSERT_EQ(element_a.size(), 2); + EXPECT_EQ(element_a.get_element(0), 11); + EXPECT_EQ(element_a.get_element(1), 22); + + const auto& map_nullable = assert_cast(*block.get_by_position(2).column); + EXPECT_FALSE(map_nullable.is_null_at(0)); + EXPECT_TRUE(map_nullable.is_null_at(1)); + EXPECT_FALSE(map_nullable.is_null_at(2)); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), COMPLEX_ROW_COUNT); + EXPECT_EQ(map_column.get_offsets()[0], 1); + EXPECT_EQ(map_column.get_offsets()[1], 1); + EXPECT_EQ(map_column.get_offsets()[2], 2); + const auto& map_keys = assert_cast( + assert_cast(map_column.get_keys()).get_nested_column()); + EXPECT_EQ(map_keys.get_data_at(0).to_string(), "left"); + EXPECT_EQ(map_keys.get_data_at(1).to_string(), "right"); + const auto& value_struct = assert_cast( + assert_cast(map_column.get_values()).get_nested_column()); + ASSERT_EQ(value_struct.tuple_size(), 1); + const auto& value_a = assert_cast( + assert_cast(value_struct.get_column(0)).get_nested_column()); + ASSERT_EQ(value_a.size(), 2); + EXPECT_EQ(value_a.get_element(0), 101); + EXPECT_EQ(value_a.get_element(1), 202); +} + +} // namespace +} // namespace doris diff --git a/be/test/format_v2/parquet/parquet_column_reader_test.cpp b/be/test/format_v2/parquet/parquet_column_reader_test.cpp new file mode 100644 index 00000000000000..fb4cd129e1b03d --- /dev/null +++ b/be/test/format_v2/parquet/parquet_column_reader_test.cpp @@ -0,0 +1,3682 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_array.h" +#include "core/column/column_decimal.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "core/types.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/selection_vector.h" + +namespace doris::format::parquet { +namespace { + +constexpr int64_t ROW_COUNT = 5; + +std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { + std::shared_ptr array; + EXPECT_TRUE(builder->Finish(&array).ok()); + return array; +} + +template +const ColumnType& get_nullable_nested_column(const IColumn& column) { + // File-local schema exposed by the parquet reader follows Doris external-table semantics: + // nested STRUCT fields, LIST elements, and MAP keys/values are nullable even when the parquet + // field is required. + const auto& nullable_column = assert_cast(column); + return assert_cast(nullable_column.get_nested_column()); +} + +ParquetColumnSchema mock_column_schema() { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = "mock"; + schema.type = std::make_shared(); + return schema; +} + +class BaseUnsupportedReader final : public ParquetColumnReader { +public: + BaseUnsupportedReader() + : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} + + Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } +}; + +class DefaultSelectReader final : public ParquetColumnReader { +public: + DefaultSelectReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { + auto& values = assert_cast(*column); + for (int64_t row = 0; row < rows; ++row) { + values.insert_value(static_cast(_cursor + row)); + } + _cursor += rows; + *rows_read = rows; + _read_ranges.push_back(rows); + return Status::OK(); + } + + Status skip(int64_t rows) override { + _cursor += rows; + _skip_ranges.push_back(rows); + return Status::OK(); + } + + const std::vector& read_ranges() const { return _read_ranges; } + const std::vector& skip_ranges() const { return _skip_ranges; } + +private: + int64_t _cursor = 0; + std::vector _read_ranges; + std::vector _skip_ranges; +}; + +class NestedSkipReader final : public ParquetColumnReader { +public: + NestedSkipReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} + + Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } + + Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { + *values_consumed = length_upper_bound; + return Status::OK(); + } +}; + +class ParquetColumnReaderTest : public testing::Test { +protected: + void SetUp() override { + _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_column_reader_test"; + std::filesystem::remove_all(_test_dir); + std::filesystem::create_directories(_test_dir); + _file_path = (_test_dir / "reader.parquet").string(); + _plain_file_path = (_test_dir / "plain_reader.parquet").string(); + write_parquet_file(); + _file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + auto metadata = _file_reader->metadata(); + ASSERT_EQ(metadata->num_row_groups(), 1); + _row_group = _file_reader->RowGroup(0); + ASSERT_NE(_row_group, nullptr); + auto schema_descriptor = _file_reader->metadata()->schema(); + ASSERT_NE(schema_descriptor, nullptr); + auto st = build_parquet_column_schema(*schema_descriptor, &_fields); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(_fields.size(), _expected_by_field.size()); + } + + void TearDown() override { std::filesystem::remove_all(_test_dir); } + + template + std::shared_ptr build_required_array(const std::vector& values) { + Builder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_string_array(const std::vector& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_nullable_int32_array() { + arrow::Int32Builder builder; + EXPECT_TRUE(builder.Append(1).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.Append(3).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.Append(5).ok()); + return finish_array(&builder); + } + + std::shared_ptr build_all_null_int32_array() { + arrow::Int32Builder builder; + for (int64_t row = 0; row < ROW_COUNT; ++row) { + EXPECT_TRUE(builder.AppendNull().ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_required_struct_array() { + auto struct_type = arrow::struct_({arrow::field("a", arrow::int32(), false), + arrow::field("b", arrow::utf8(), false)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto b_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* a_builder = assert_cast(builder.field_builder(0)); + auto* b_builder = assert_cast(builder.field_builder(1)); + const std::vector a_values = {101, 102, 103, 104, 105}; + const std::vector b_values = {"sa", "sb", "sc", "sd", "se"}; + for (size_t row = 0; row < a_values.size(); ++row) { + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(a_values[row]).ok()); + EXPECT_TRUE(b_builder->Append(b_values[row]).ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_nullable_struct_array() { + auto struct_type = arrow::struct_( + {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto b_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* a_builder = assert_cast(builder.field_builder(0)); + auto* b_builder = assert_cast(builder.field_builder(1)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(201).ok()); + EXPECT_TRUE(b_builder->Append("nsa").ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(203).ok()); + EXPECT_TRUE(b_builder->AppendNull().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(204).ok()); + EXPECT_TRUE(b_builder->Append("nsd").ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_struct_with_decimal_array() { + auto decimal_type = arrow::decimal128(38, 6); + auto struct_type = arrow::struct_( + {arrow::field("a", arrow::int32(), false), arrow::field("d", decimal_type, true)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto d_array_builder = std::make_unique( + decimal_type, arrow::default_memory_pool()); + field_builders.push_back(std::shared_ptr(std::move(d_array_builder))); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* a_builder = assert_cast(builder.field_builder(0)); + auto* d_builder = assert_cast(builder.field_builder(1)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(301).ok()); + EXPECT_TRUE(d_builder->Append(arrow::Decimal128(123456789)).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(303).ok()); + EXPECT_TRUE(d_builder->AppendNull().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(304).ok()); + EXPECT_TRUE(d_builder->Append(arrow::Decimal128(-987654321)).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_struct_with_list_array() { + auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); + auto struct_type = arrow::struct_( + {arrow::field("a", arrow::int32(), false), arrow::field("xs", list_type, true)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto value_builder = std::make_shared(); + auto list_builder = std::make_shared(arrow::default_memory_pool(), + value_builder, list_type); + field_builders.push_back(list_builder); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* a_builder = assert_cast(builder.field_builder(0)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(301).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(value_builder->Append(1).ok()); + EXPECT_TRUE(value_builder->Append(2).ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(303).ok()); + EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(304).ok()); + EXPECT_TRUE(list_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(305).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(value_builder->Append(5).ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_struct_with_map_array() { + auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + auto struct_type = arrow::struct_( + {arrow::field("a", arrow::int32(), false), arrow::field("kv", map_type, true)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + auto map_builder = std::make_shared( + arrow::default_memory_pool(), key_builder, value_builder, map_type); + field_builders.push_back(map_builder); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* a_builder = assert_cast(builder.field_builder(0)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(401).ok()); + EXPECT_TRUE(map_builder->Append().ok()); + EXPECT_TRUE(key_builder->Append(1).ok()); + EXPECT_TRUE(value_builder->Append("one").ok()); + EXPECT_TRUE(key_builder->Append(2).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(403).ok()); + EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(404).ok()); + EXPECT_TRUE(map_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(a_builder->Append(405).ok()); + EXPECT_TRUE(map_builder->Append().ok()); + EXPECT_TRUE(key_builder->Append(5).ok()); + EXPECT_TRUE(value_builder->Append("five").ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_struct_with_nested_struct_list_array() { + auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); + auto nested_type = arrow::struct_({arrow::field("xs", list_type, true)}); + auto struct_type = arrow::struct_({arrow::field("nested", nested_type, true)}); + + auto value_builder = std::make_shared(); + auto list_builder = std::make_shared(arrow::default_memory_pool(), + value_builder, list_type); + std::vector> nested_field_builders; + nested_field_builders.push_back(list_builder); + auto nested_builder = std::make_shared( + nested_type, arrow::default_memory_pool(), std::move(nested_field_builders)); + std::vector> field_builders; + field_builders.push_back(nested_builder); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(nested_builder->Append().ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(value_builder->Append(7).ok()); + EXPECT_TRUE(value_builder->Append(8).ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(nested_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(nested_builder->Append().ok()); + EXPECT_TRUE(list_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(nested_builder->Append().ok()); + EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); + return finish_array(&builder); + } + + std::shared_ptr build_required_int_list_array() { + auto value_builder = std::make_shared(); + arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, + arrow::list(arrow::field("element", arrow::int32(), false))); + const std::vector> values = { + {1, 2}, {3}, {4, 5, 6}, {7}, {8, 9}, + }; + for (const auto& row : values) { + EXPECT_TRUE(builder.Append().ok()); + for (const auto value : row) { + EXPECT_TRUE(value_builder->Append(value).ok()); + } + } + return finish_array(&builder); + } + + std::shared_ptr build_nullable_int_list_array() { + auto value_builder = std::make_shared(); + arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, + arrow::list(arrow::field("element", arrow::int32(), true))); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->Append(10).ok()); + EXPECT_TRUE(value_builder->Append(20).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(value_builder->Append(30).ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->Append(40).ok()); + return finish_array(&builder); + } + + std::shared_ptr build_required_nullable_int_list_array() { + auto value_builder = std::make_shared(); + arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, + arrow::list(arrow::field("element", arrow::int32(), true))); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(value_builder->Append(110).ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->Append(120).ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->Append(130).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(builder.Append().ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_struct_list_array() { + auto struct_type = arrow::struct_( + {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto b_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); + auto struct_builder = std::make_shared( + struct_type, arrow::default_memory_pool(), std::move(field_builders)); + arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, + arrow::list(arrow::field("element", struct_type, true))); + auto* a_builder = assert_cast(struct_builder->field_builder(0)); + auto* b_builder = assert_cast(struct_builder->field_builder(1)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(11).ok()); + EXPECT_TRUE(b_builder->Append("la").ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(12).ok()); + EXPECT_TRUE(b_builder->AppendNull().ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->AppendNull().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(13).ok()); + EXPECT_TRUE(b_builder->Append("ld").ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(14).ok()); + EXPECT_TRUE(b_builder->Append("le").ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_list_list_int_array() { + auto value_builder = std::make_shared(); + auto inner_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); + auto inner_list_builder = std::make_shared( + arrow::default_memory_pool(), value_builder, inner_list_type); + arrow::ListBuilder builder(arrow::default_memory_pool(), inner_list_builder, + arrow::list(arrow::field("element", inner_list_type, true))); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(inner_list_builder->Append().ok()); + EXPECT_TRUE(value_builder->Append(1).ok()); + EXPECT_TRUE(value_builder->Append(2).ok()); + EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); + EXPECT_TRUE(inner_list_builder->AppendNull().ok()); + EXPECT_TRUE(inner_list_builder->Append().ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(value_builder->Append(3).ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(inner_list_builder->Append().ok()); + EXPECT_TRUE(value_builder->Append(4).ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); + EXPECT_TRUE(inner_list_builder->Append().ok()); + EXPECT_TRUE(value_builder->Append(5).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + return finish_array(&builder); + } + + std::shared_ptr build_required_int_string_map_array() { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, + map_type); + const std::vector>> values = { + {{1, "a"}, {2, "b"}}, {{3, "c"}}, {{4, "d"}, {5, "e"}, {6, "f"}}, + {{7, "g"}}, {{8, "h"}, {9, "i"}}, + }; + for (const auto& row : values) { + EXPECT_TRUE(builder.Append().ok()); + for (const auto& [key, value] : row) { + EXPECT_TRUE(key_builder->Append(key).ok()); + EXPECT_TRUE(value_builder->Append(value).ok()); + } + } + return finish_array(&builder); + } + + std::shared_ptr build_nullable_int_string_map_array() { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, + map_type); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(10).ok()); + EXPECT_TRUE(value_builder->Append("aa").ok()); + EXPECT_TRUE(key_builder->Append(20).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(30).ok()); + EXPECT_TRUE(value_builder->Append("cc").ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(40).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + return finish_array(&builder); + } + + std::shared_ptr build_required_nullable_string_map_array() { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, + map_type); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(101).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(key_builder->Append(102).ok()); + EXPECT_TRUE(value_builder->Append("bb").ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(103).ok()); + EXPECT_TRUE(value_builder->Append("cc").ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(104).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_int_struct_map_array() { + auto key_builder = std::make_shared(); + auto struct_type = arrow::struct_( + {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto b_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); + auto value_builder = std::make_shared( + struct_type, arrow::default_memory_pool(), std::move(field_builders)); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", struct_type, true)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, + map_type); + auto* a_builder = assert_cast(value_builder->field_builder(0)); + auto* b_builder = assert_cast(value_builder->field_builder(1)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(101).ok()); + EXPECT_TRUE(value_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(21).ok()); + EXPECT_TRUE(b_builder->Append("ma").ok()); + EXPECT_TRUE(key_builder->Append(102).ok()); + EXPECT_TRUE(value_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(22).ok()); + EXPECT_TRUE(b_builder->AppendNull().ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(103).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(104).ok()); + EXPECT_TRUE(value_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(24).ok()); + EXPECT_TRUE(b_builder->Append("me").ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_int_list_map_array() { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); + auto list_builder = std::make_shared(arrow::default_memory_pool(), + value_builder, list_type); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, + map_type); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(201).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(value_builder->Append(1).ok()); + EXPECT_TRUE(value_builder->Append(2).ok()); + EXPECT_TRUE(key_builder->Append(202).ok()); + EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(203).ok()); + EXPECT_TRUE(list_builder->AppendNull().ok()); + EXPECT_TRUE(key_builder->Append(204).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(value_builder->Append(3).ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(205).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(value_builder->Append(4).ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_map_list_array() { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + auto map_builder = std::make_shared( + arrow::default_memory_pool(), key_builder, value_builder, map_type); + arrow::ListBuilder builder(arrow::default_memory_pool(), map_builder, + arrow::list(arrow::field("element", map_type, true))); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(map_builder->Append().ok()); + EXPECT_TRUE(key_builder->Append(1).ok()); + EXPECT_TRUE(value_builder->Append("a").ok()); + EXPECT_TRUE(key_builder->Append(2).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(map_builder->AppendNull().ok()); + EXPECT_TRUE(map_builder->Append().ok()); + EXPECT_TRUE(key_builder->Append(3).ok()); + EXPECT_TRUE(value_builder->Append("c").ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(map_builder->Append().ok()); + EXPECT_TRUE(key_builder->Append(4).ok()); + EXPECT_TRUE(value_builder->Append("d").ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_int_map_map_array() { + auto key_builder = std::make_shared(); + auto nested_key_builder = std::make_shared(); + auto nested_value_builder = std::make_shared(); + auto nested_map_type = + arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + auto nested_map_builder = std::make_shared( + arrow::default_memory_pool(), nested_key_builder, nested_value_builder, + nested_map_type); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, nested_map_builder, + map_type); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(10).ok()); + EXPECT_TRUE(nested_map_builder->Append().ok()); + EXPECT_TRUE(nested_key_builder->Append(101).ok()); + EXPECT_TRUE(nested_value_builder->Append("aa").ok()); + EXPECT_TRUE(key_builder->Append(20).ok()); + EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(30).ok()); + EXPECT_TRUE(nested_map_builder->AppendNull().ok()); + EXPECT_TRUE(key_builder->Append(40).ok()); + EXPECT_TRUE(nested_map_builder->Append().ok()); + EXPECT_TRUE(nested_key_builder->Append(401).ok()); + EXPECT_TRUE(nested_value_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + return finish_array(&builder); + } + + std::shared_ptr build_deep_list_struct_map_list_array() { + auto element_builder = std::make_shared(); + auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); + auto list_builder = std::make_shared(arrow::default_memory_pool(), + element_builder, list_type); + auto key_builder = std::make_shared(); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); + auto map_builder = std::make_shared(arrow::default_memory_pool(), + key_builder, list_builder, map_type); + auto struct_type = arrow::struct_({arrow::field("kv", map_type, true)}); + std::vector> struct_field_builders; + struct_field_builders.push_back(map_builder); + auto struct_builder = std::make_shared( + struct_type, arrow::default_memory_pool(), std::move(struct_field_builders)); + arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, + arrow::list(arrow::field("element", struct_type, true))); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(map_builder->Append().ok()); + EXPECT_TRUE(key_builder->Append(1).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(element_builder->Append(10).ok()); + EXPECT_TRUE(element_builder->AppendNull().ok()); + EXPECT_TRUE(key_builder->Append(2).ok()); + EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); + EXPECT_TRUE(struct_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(map_builder->AppendNull().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(map_builder->Append().ok()); + EXPECT_TRUE(key_builder->Append(3).ok()); + EXPECT_TRUE(list_builder->AppendNull().ok()); + EXPECT_TRUE(key_builder->Append(4).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(element_builder->Append(40).ok()); + return finish_array(&builder); + } + + std::shared_ptr build_deep_map_list_map_array() { + auto nested_key_builder = std::make_shared(); + auto nested_value_builder = std::make_shared(); + auto nested_map_type = + arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + auto nested_map_builder = std::make_shared( + arrow::default_memory_pool(), nested_key_builder, nested_value_builder, + nested_map_type); + auto list_type = arrow::list(arrow::field("element", nested_map_type, true)); + auto list_builder = std::make_shared(arrow::default_memory_pool(), + nested_map_builder, list_type); + auto key_builder = std::make_shared(); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, + map_type); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(10).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(nested_map_builder->Append().ok()); + EXPECT_TRUE(nested_key_builder->Append(1).ok()); + EXPECT_TRUE(nested_value_builder->Append("a").ok()); + EXPECT_TRUE(nested_key_builder->Append(2).ok()); + EXPECT_TRUE(nested_value_builder->AppendNull().ok()); + EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); + EXPECT_TRUE(nested_map_builder->AppendNull().ok()); + EXPECT_TRUE(key_builder->Append(20).ok()); + EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(30).ok()); + EXPECT_TRUE(list_builder->AppendNull().ok()); + EXPECT_TRUE(key_builder->Append(40).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(nested_map_builder->Append().ok()); + EXPECT_TRUE(nested_key_builder->Append(3).ok()); + EXPECT_TRUE(nested_value_builder->Append("c").ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(50).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(nested_map_builder->AppendNull().ok()); + EXPECT_TRUE(nested_map_builder->Append().ok()); + EXPECT_TRUE(nested_key_builder->Append(4).ok()); + EXPECT_TRUE(nested_value_builder->Append("d").ok()); + return finish_array(&builder); + } + + void add_field(const std::shared_ptr& field, std::shared_ptr array, + std::function validator) { + _arrow_fields.push_back(field); + _arrays.push_back(std::move(array)); + _expected_by_field.push_back(std::move(validator)); + } + + void write_parquet_file() { + add_field(arrow::field("int32_col", arrow::int32(), false), + build_required_array({10, 20, 30, 40, 50}), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_element(0), 10); + EXPECT_EQ(values.get_element(4), 50); + }); + add_field(arrow::field("string_col", arrow::utf8(), false), + build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"}), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type_descriptor.is_string_like); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); + EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); + }); + add_field(arrow::field("nullable_int_col", arrow::int32(), true), + build_nullable_int32_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + const auto& nested_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_TRUE(nullable_column.is_null_at(3)); + EXPECT_EQ(nested_column.get_element(0), 1); + EXPECT_EQ(nested_column.get_element(2), 3); + }); + add_field(arrow::field("all_null_int_col", arrow::int32(), true), + build_all_null_int32_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + for (size_t row = 0; row < ROW_COUNT; ++row) { + EXPECT_TRUE(nullable_column.is_null_at(row)); + } + }); + add_field(arrow::field("struct_col", + arrow::struct_({ + arrow::field("a", arrow::int32(), false), + arrow::field("b", arrow::utf8(), false), + }), + false), + build_required_struct_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_STRUCT); + const auto& struct_column = assert_cast(column); + ASSERT_EQ(struct_column.get_columns().size(), 2); + const auto& a_values = + get_nullable_nested_column(struct_column.get_column(0)); + const auto& b_values = + get_nullable_nested_column(struct_column.get_column(1)); + EXPECT_EQ(a_values.get_element(0), 101); + EXPECT_EQ(a_values.get_element(4), 105); + EXPECT_EQ(b_values.get_data_at(1).to_string(), "sb"); + EXPECT_EQ(b_values.get_data_at(4).to_string(), "se"); + }); + add_field(arrow::field("nullable_struct_col", + arrow::struct_({ + arrow::field("a", arrow::int32(), false), + arrow::field("b", arrow::utf8(), true), + }), + true), + build_nullable_struct_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_TRUE(nullable_column.is_null_at(4)); + + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 2); + const auto& a_values = + get_nullable_nested_column(struct_column.get_column(0)); + const auto& b_values = + assert_cast(struct_column.get_column(1)); + const auto& b_nested = + assert_cast(b_values.get_nested_column()); + EXPECT_EQ(a_values.get_element(0), 201); + EXPECT_EQ(a_values.get_element(2), 203); + EXPECT_EQ(a_values.get_element(3), 204); + EXPECT_FALSE(b_values.is_null_at(0)); + EXPECT_TRUE(b_values.is_null_at(2)); + EXPECT_FALSE(b_values.is_null_at(3)); + EXPECT_EQ(b_nested.get_data_at(0).to_string(), "nsa"); + EXPECT_EQ(b_nested.get_data_at(3).to_string(), "nsd"); + }); + add_field(arrow::field("nullable_struct_decimal_col", + arrow::struct_({ + arrow::field("a", arrow::int32(), false), + arrow::field("d", arrow::decimal128(38, 6), true), + }), + true), + build_nullable_struct_with_decimal_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_TRUE(nullable_column.is_null_at(4)); + + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 2); + const auto& a_values = + get_nullable_nested_column(struct_column.get_column(0)); + const auto& d_values = + assert_cast(struct_column.get_column(1)); + const auto& d_nested = + assert_cast(d_values.get_nested_column()); + EXPECT_EQ(a_values.get_element(0), 301); + EXPECT_EQ(a_values.get_element(2), 303); + EXPECT_EQ(a_values.get_element(3), 304); + EXPECT_FALSE(d_values.is_null_at(0)); + EXPECT_TRUE(d_values.is_null_at(2)); + EXPECT_FALSE(d_values.is_null_at(3)); + EXPECT_EQ(d_nested.get_element(0), Decimal128V3(123456789)); + EXPECT_EQ(d_nested.get_element(3), Decimal128V3(-987654321)); + }); + auto struct_list_type = arrow::struct_({ + arrow::field("a", arrow::int32(), false), + arrow::field("xs", arrow::list(arrow::field("element", arrow::int32(), true)), + true), + }); + add_field(arrow::field("nullable_struct_list_col", struct_list_type, true), + build_nullable_struct_with_list_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 2); + const auto& a_values = + get_nullable_nested_column(struct_column.get_column(0)); + EXPECT_EQ(a_values.get_element(0), 301); + EXPECT_EQ(a_values.get_element(2), 303); + EXPECT_EQ(a_values.get_element(3), 304); + EXPECT_EQ(a_values.get_element(4), 305); + + const auto& xs_nullable = + assert_cast(struct_column.get_column(1)); + ASSERT_EQ(xs_nullable.size(), ROW_COUNT); + EXPECT_FALSE(xs_nullable.is_null_at(0)); + EXPECT_FALSE(xs_nullable.is_null_at(2)); + EXPECT_TRUE(xs_nullable.is_null_at(3)); + EXPECT_FALSE(xs_nullable.is_null_at(4)); + const auto& xs_array = + assert_cast(xs_nullable.get_nested_column()); + const auto& offsets = xs_array.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 2); + EXPECT_EQ(offsets[3], 2); + EXPECT_EQ(offsets[4], 4); + const auto& elements = + assert_cast(xs_array.get_data()); + ASSERT_EQ(elements.size(), 4); + EXPECT_FALSE(elements.is_null_at(0)); + EXPECT_FALSE(elements.is_null_at(1)); + EXPECT_TRUE(elements.is_null_at(2)); + EXPECT_FALSE(elements.is_null_at(3)); + const auto& values = + assert_cast(elements.get_nested_column()); + EXPECT_EQ(values.get_element(0), 1); + EXPECT_EQ(values.get_element(1), 2); + EXPECT_EQ(values.get_element(3), 5); + }); + auto struct_map_type = arrow::struct_({ + arrow::field("a", arrow::int32(), false), + arrow::field("kv", + arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), + true), + }); + add_field(arrow::field("nullable_struct_map_col", struct_map_type, true), + build_nullable_struct_with_map_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 2); + const auto& a_values = + get_nullable_nested_column(struct_column.get_column(0)); + EXPECT_EQ(a_values.get_element(0), 401); + EXPECT_EQ(a_values.get_element(2), 403); + EXPECT_EQ(a_values.get_element(3), 404); + EXPECT_EQ(a_values.get_element(4), 405); + + const auto& kv_nullable = + assert_cast(struct_column.get_column(1)); + ASSERT_EQ(kv_nullable.size(), ROW_COUNT); + EXPECT_FALSE(kv_nullable.is_null_at(0)); + EXPECT_FALSE(kv_nullable.is_null_at(2)); + EXPECT_TRUE(kv_nullable.is_null_at(3)); + EXPECT_FALSE(kv_nullable.is_null_at(4)); + const auto& kv_map = + assert_cast(kv_nullable.get_nested_column()); + const auto& offsets = kv_map.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 2); + EXPECT_EQ(offsets[3], 2); + EXPECT_EQ(offsets[4], 3); + const auto& keys = get_nullable_nested_column(kv_map.get_keys()); + const auto& values = assert_cast(kv_map.get_values()); + const auto& value_data = + assert_cast(values.get_nested_column()); + ASSERT_EQ(keys.size(), 3); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(1), 2); + EXPECT_EQ(keys.get_element(2), 5); + EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); + EXPECT_TRUE(values.is_null_at(1)); + EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); + }); + auto nested_struct_list_type = arrow::struct_({ + arrow::field("nested", + arrow::struct_({ + arrow::field("xs", + arrow::list(arrow::field("element", + arrow::int32(), true)), + true), + }), + true), + }); + add_field(arrow::field("nullable_struct_nested_struct_list_col", nested_struct_list_type, + true), + build_nullable_struct_with_nested_struct_list_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + const auto& nested_nullable = + assert_cast(struct_column.get_column(0)); + EXPECT_FALSE(nested_nullable.is_null_at(0)); + EXPECT_TRUE(nested_nullable.is_null_at(2)); + EXPECT_FALSE(nested_nullable.is_null_at(3)); + EXPECT_FALSE(nested_nullable.is_null_at(4)); + }); + add_field(arrow::field("list_int_col", + arrow::list(arrow::field("element", arrow::int32(), false)), false), + build_required_int_list_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_ARRAY); + const auto* array_type = + assert_cast(remove_nullable(schema.type).get()); + EXPECT_EQ( + remove_nullable(array_type->get_nested_type())->get_primitive_type(), + TYPE_INT); + const auto& array_column = assert_cast(column); + ASSERT_EQ(array_column.size(), ROW_COUNT); + const auto array_size_at = [&array_column](size_t row_idx) { + return array_column.get_offsets()[row_idx] - + (row_idx == 0 ? 0 : array_column.get_offsets()[row_idx - 1]); + }; + EXPECT_EQ(array_size_at(0), 2); + EXPECT_EQ(array_size_at(1), 1); + EXPECT_EQ(array_size_at(2), 3); + EXPECT_EQ(array_size_at(4), 2); + const auto& values = + get_nullable_nested_column(array_column.get_data()); + ASSERT_EQ(values.size(), 9); + EXPECT_EQ(values.get_element(0), 1); + EXPECT_EQ(values.get_element(5), 6); + EXPECT_EQ(values.get_element(8), 9); + }); + add_field(arrow::field("nullable_list_int_col", + arrow::list(arrow::field("element", arrow::int32(), true)), true), + build_nullable_int_list_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + const auto& array_column = + assert_cast(nullable_column.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 2); + EXPECT_EQ(offsets[3], 4); + EXPECT_EQ(offsets[4], 5); + const auto& elements = + assert_cast(array_column.get_data()); + const auto& values = + assert_cast(elements.get_nested_column()); + ASSERT_EQ(elements.size(), 5); + EXPECT_EQ(values.get_element(0), 10); + EXPECT_EQ(values.get_element(1), 20); + EXPECT_TRUE(elements.is_null_at(2)); + EXPECT_EQ(values.get_element(3), 30); + EXPECT_EQ(values.get_element(4), 40); + }); + add_field(arrow::field("required_nullable_list_int_col", + arrow::list(arrow::field("element", arrow::int32(), true)), false), + build_required_nullable_int_list_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_FALSE(schema.type->is_nullable()); + const auto& array_column = assert_cast(column); + const auto& offsets = array_column.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 3); + EXPECT_EQ(offsets[3], 5); + EXPECT_EQ(offsets[4], 5); + const auto& elements = + assert_cast(array_column.get_data()); + ASSERT_EQ(elements.size(), 5); + EXPECT_TRUE(elements.is_null_at(0)); + EXPECT_FALSE(elements.is_null_at(1)); + EXPECT_TRUE(elements.is_null_at(4)); + }); + auto list_struct_type = arrow::struct_({ + arrow::field("a", arrow::int32(), false), + arrow::field("b", arrow::utf8(), true), + }); + add_field(arrow::field("nullable_list_struct_col", + arrow::list(arrow::field("element", list_struct_type, true)), true), + build_nullable_struct_list_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& array_column = + assert_cast(nullable_column.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 2); + EXPECT_EQ(offsets[3], 4); + EXPECT_EQ(offsets[4], 5); + + const auto& elements = + assert_cast(array_column.get_data()); + const auto& struct_column = + assert_cast(elements.get_nested_column()); + const auto& a_values = + get_nullable_nested_column(struct_column.get_column(0)); + const auto& b_values = + assert_cast(struct_column.get_column(1)); + const auto& b_data = + assert_cast(b_values.get_nested_column()); + ASSERT_EQ(elements.size(), 5); + EXPECT_FALSE(elements.is_null_at(0)); + EXPECT_FALSE(elements.is_null_at(1)); + EXPECT_TRUE(elements.is_null_at(2)); + EXPECT_FALSE(elements.is_null_at(3)); + EXPECT_EQ(a_values.get_element(0), 11); + EXPECT_EQ(a_values.get_element(1), 12); + EXPECT_EQ(a_values.get_element(3), 13); + EXPECT_EQ(a_values.get_element(4), 14); + EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); + EXPECT_TRUE(b_values.is_null_at(1)); + EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); + EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); + }); + auto nested_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); + add_field(arrow::field("nullable_list_list_int_col", + arrow::list(arrow::field("element", nested_list_type, true)), true), + build_nullable_list_list_int_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& outer_array = + assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), ROW_COUNT); + EXPECT_EQ(outer_offsets[0], 4); + EXPECT_EQ(outer_offsets[1], 4); + EXPECT_EQ(outer_offsets[2], 4); + EXPECT_EQ(outer_offsets[3], 5); + EXPECT_EQ(outer_offsets[4], 7); + + const auto& inner_nullable = + assert_cast(outer_array.get_data()); + ASSERT_EQ(inner_nullable.size(), 7); + EXPECT_FALSE(inner_nullable.is_null_at(0)); + EXPECT_FALSE(inner_nullable.is_null_at(1)); + EXPECT_TRUE(inner_nullable.is_null_at(2)); + EXPECT_FALSE(inner_nullable.is_null_at(3)); + EXPECT_FALSE(inner_nullable.is_null_at(6)); + + const auto& inner_array = + assert_cast(inner_nullable.get_nested_column()); + const auto& inner_offsets = inner_array.get_offsets(); + ASSERT_EQ(inner_offsets.size(), 7); + EXPECT_EQ(inner_offsets[0], 2); + EXPECT_EQ(inner_offsets[1], 2); + EXPECT_EQ(inner_offsets[2], 2); + EXPECT_EQ(inner_offsets[3], 4); + EXPECT_EQ(inner_offsets[4], 5); + EXPECT_EQ(inner_offsets[5], 5); + EXPECT_EQ(inner_offsets[6], 7); + + const auto& elements = + assert_cast(inner_array.get_data()); + const auto& values = + assert_cast(elements.get_nested_column()); + ASSERT_EQ(elements.size(), 7); + EXPECT_EQ(values.get_element(0), 1); + EXPECT_EQ(values.get_element(1), 2); + EXPECT_TRUE(elements.is_null_at(2)); + EXPECT_EQ(values.get_element(3), 3); + EXPECT_EQ(values.get_element(4), 4); + EXPECT_EQ(values.get_element(5), 5); + EXPECT_TRUE(elements.is_null_at(6)); + }); + add_field(arrow::field( + "map_int_string_col", + arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)), + false), + build_required_int_string_map_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_MAP); + const auto* map_type = + assert_cast(remove_nullable(schema.type).get()); + EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), + TYPE_INT); + EXPECT_EQ(remove_nullable(map_type->get_value_type())->get_primitive_type(), + TYPE_STRING); + const auto& map_column = assert_cast(column); + ASSERT_EQ(map_column.size(), ROW_COUNT); + const auto map_size_at = [&map_column](size_t row_idx) { + return map_column.get_offsets()[row_idx] - + (row_idx == 0 ? 0 : map_column.get_offsets()[row_idx - 1]); + }; + EXPECT_EQ(map_size_at(0), 2); + EXPECT_EQ(map_size_at(1), 1); + EXPECT_EQ(map_size_at(2), 3); + EXPECT_EQ(map_size_at(4), 2); + const auto& keys = + get_nullable_nested_column(map_column.get_keys()); + const auto& values = + get_nullable_nested_column(map_column.get_values()); + ASSERT_EQ(keys.size(), 9); + ASSERT_EQ(values.size(), 9); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(5), 6); + EXPECT_EQ(keys.get_element(8), 9); + EXPECT_EQ(values.get_data_at(0).to_string(), "a"); + EXPECT_EQ(values.get_data_at(5).to_string(), "f"); + EXPECT_EQ(values.get_data_at(8).to_string(), "i"); + }); + add_field( + arrow::field("nullable_map_int_string_col", + arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), + true), + build_nullable_int_string_map_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& map_column = + assert_cast(nullable_column.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 2); + EXPECT_EQ(offsets[3], 3); + EXPECT_EQ(offsets[4], 4); + const auto& keys = + get_nullable_nested_column(map_column.get_keys()); + const auto& values = + assert_cast(map_column.get_values()); + const auto& value_data = + assert_cast(values.get_nested_column()); + ASSERT_EQ(keys.size(), 4); + EXPECT_EQ(keys.get_element(0), 10); + EXPECT_EQ(keys.get_element(1), 20); + EXPECT_EQ(keys.get_element(3), 40); + EXPECT_EQ(value_data.get_data_at(0).to_string(), "aa"); + EXPECT_TRUE(values.is_null_at(1)); + EXPECT_EQ(value_data.get_data_at(2).to_string(), "cc"); + EXPECT_TRUE(values.is_null_at(3)); + }); + add_field( + arrow::field("required_nullable_map_int_string_col", + arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), + false), + build_required_nullable_string_map_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_FALSE(schema.type->is_nullable()); + const auto& map_column = assert_cast(column); + const auto& offsets = map_column.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 3); + EXPECT_EQ(offsets[3], 3); + EXPECT_EQ(offsets[4], 4); + const auto& values = + assert_cast(map_column.get_values()); + ASSERT_EQ(values.size(), 4); + EXPECT_TRUE(values.is_null_at(0)); + EXPECT_FALSE(values.is_null_at(1)); + EXPECT_TRUE(values.is_null_at(3)); + }); + auto map_struct_type = arrow::struct_({ + arrow::field("a", arrow::int32(), false), + arrow::field("b", arrow::utf8(), true), + }); + add_field(arrow::field( + "nullable_map_int_struct_col", + arrow::map(arrow::int32(), arrow::field("value", map_struct_type, true)), + true), + build_nullable_int_struct_map_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& map_column = + assert_cast(nullable_column.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 2); + EXPECT_EQ(offsets[3], 3); + EXPECT_EQ(offsets[4], 4); + + const auto& keys = + get_nullable_nested_column(map_column.get_keys()); + const auto& values = + assert_cast(map_column.get_values()); + const auto& struct_column = + assert_cast(values.get_nested_column()); + const auto& a_values = + get_nullable_nested_column(struct_column.get_column(0)); + const auto& b_values = + assert_cast(struct_column.get_column(1)); + const auto& b_data = + assert_cast(b_values.get_nested_column()); + ASSERT_EQ(keys.size(), 4); + ASSERT_EQ(values.size(), 4); + EXPECT_EQ(keys.get_element(0), 101); + EXPECT_EQ(keys.get_element(1), 102); + EXPECT_EQ(keys.get_element(3), 104); + EXPECT_FALSE(values.is_null_at(0)); + EXPECT_FALSE(values.is_null_at(1)); + EXPECT_TRUE(values.is_null_at(2)); + EXPECT_FALSE(values.is_null_at(3)); + EXPECT_EQ(a_values.get_element(0), 21); + EXPECT_EQ(a_values.get_element(1), 22); + EXPECT_EQ(a_values.get_element(3), 24); + EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); + EXPECT_TRUE(b_values.is_null_at(1)); + EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); + }); + auto map_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); + add_field( + arrow::field("nullable_map_int_list_col", + arrow::map(arrow::int32(), arrow::field("value", map_list_type, true)), + true), + build_nullable_int_list_map_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& map_column = + assert_cast(nullable_column.get_nested_column()); + const auto& map_offsets = map_column.get_offsets(); + ASSERT_EQ(map_offsets.size(), ROW_COUNT); + EXPECT_EQ(map_offsets[0], 2); + EXPECT_EQ(map_offsets[1], 2); + EXPECT_EQ(map_offsets[2], 2); + EXPECT_EQ(map_offsets[3], 4); + EXPECT_EQ(map_offsets[4], 5); + + const auto& keys = + get_nullable_nested_column(map_column.get_keys()); + ASSERT_EQ(keys.size(), 5); + EXPECT_EQ(keys.get_element(0), 201); + EXPECT_EQ(keys.get_element(1), 202); + EXPECT_EQ(keys.get_element(2), 203); + EXPECT_EQ(keys.get_element(3), 204); + EXPECT_EQ(keys.get_element(4), 205); + + const auto& values = + assert_cast(map_column.get_values()); + ASSERT_EQ(values.size(), 5); + EXPECT_FALSE(values.is_null_at(0)); + EXPECT_FALSE(values.is_null_at(1)); + EXPECT_TRUE(values.is_null_at(2)); + EXPECT_FALSE(values.is_null_at(3)); + EXPECT_FALSE(values.is_null_at(4)); + + const auto& list_column = + assert_cast(values.get_nested_column()); + const auto& list_offsets = list_column.get_offsets(); + ASSERT_EQ(list_offsets.size(), 5); + EXPECT_EQ(list_offsets[0], 2); + EXPECT_EQ(list_offsets[1], 2); + EXPECT_EQ(list_offsets[2], 2); + EXPECT_EQ(list_offsets[3], 4); + EXPECT_EQ(list_offsets[4], 5); + + const auto& elements = + assert_cast(list_column.get_data()); + const auto& element_values = + assert_cast(elements.get_nested_column()); + ASSERT_EQ(elements.size(), 5); + EXPECT_EQ(element_values.get_element(0), 1); + EXPECT_EQ(element_values.get_element(1), 2); + EXPECT_TRUE(elements.is_null_at(2)); + EXPECT_EQ(element_values.get_element(3), 3); + EXPECT_EQ(element_values.get_element(4), 4); + }); + auto list_map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + add_field(arrow::field("nullable_list_map_int_string_col", + arrow::list(arrow::field("element", list_map_type, true)), true), + build_nullable_map_list_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& outer_array = + assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), ROW_COUNT); + EXPECT_EQ(outer_offsets[0], 2); + EXPECT_EQ(outer_offsets[1], 2); + EXPECT_EQ(outer_offsets[2], 2); + EXPECT_EQ(outer_offsets[3], 4); + EXPECT_EQ(outer_offsets[4], 5); + + const auto& map_values = + assert_cast(outer_array.get_data()); + ASSERT_EQ(map_values.size(), 5); + EXPECT_FALSE(map_values.is_null_at(0)); + EXPECT_FALSE(map_values.is_null_at(1)); + EXPECT_TRUE(map_values.is_null_at(2)); + EXPECT_FALSE(map_values.is_null_at(3)); + EXPECT_FALSE(map_values.is_null_at(4)); + + const auto& map_column = + assert_cast(map_values.get_nested_column()); + const auto& map_offsets = map_column.get_offsets(); + ASSERT_EQ(map_offsets.size(), 5); + EXPECT_EQ(map_offsets[0], 2); + EXPECT_EQ(map_offsets[1], 2); + EXPECT_EQ(map_offsets[2], 2); + EXPECT_EQ(map_offsets[3], 3); + EXPECT_EQ(map_offsets[4], 4); + const auto& keys = + get_nullable_nested_column(map_column.get_keys()); + const auto& values = + assert_cast(map_column.get_values()); + const auto& value_data = + assert_cast(values.get_nested_column()); + ASSERT_EQ(keys.size(), 4); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(1), 2); + EXPECT_EQ(keys.get_element(2), 3); + EXPECT_EQ(keys.get_element(3), 4); + EXPECT_EQ(value_data.get_data_at(0).to_string(), "a"); + EXPECT_TRUE(values.is_null_at(1)); + EXPECT_EQ(value_data.get_data_at(2).to_string(), "c"); + EXPECT_EQ(value_data.get_data_at(3).to_string(), "d"); + }); + auto nested_map_type = + arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + add_field(arrow::field( + "nullable_map_int_map_int_string_col", + arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)), + true), + build_nullable_int_map_map_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& outer_map = + assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_map.get_offsets(); + ASSERT_EQ(outer_offsets.size(), ROW_COUNT); + EXPECT_EQ(outer_offsets[0], 2); + EXPECT_EQ(outer_offsets[1], 2); + EXPECT_EQ(outer_offsets[2], 2); + EXPECT_EQ(outer_offsets[3], 4); + EXPECT_EQ(outer_offsets[4], 4); + + const auto& outer_keys = + get_nullable_nested_column(outer_map.get_keys()); + ASSERT_EQ(outer_keys.size(), 4); + EXPECT_EQ(outer_keys.get_element(0), 10); + EXPECT_EQ(outer_keys.get_element(1), 20); + EXPECT_EQ(outer_keys.get_element(2), 30); + EXPECT_EQ(outer_keys.get_element(3), 40); + + const auto& inner_values = + assert_cast(outer_map.get_values()); + ASSERT_EQ(inner_values.size(), 4); + EXPECT_FALSE(inner_values.is_null_at(0)); + EXPECT_FALSE(inner_values.is_null_at(1)); + EXPECT_TRUE(inner_values.is_null_at(2)); + EXPECT_FALSE(inner_values.is_null_at(3)); + + const auto& inner_map = + assert_cast(inner_values.get_nested_column()); + const auto& inner_offsets = inner_map.get_offsets(); + ASSERT_EQ(inner_offsets.size(), 4); + EXPECT_EQ(inner_offsets[0], 1); + EXPECT_EQ(inner_offsets[1], 1); + EXPECT_EQ(inner_offsets[2], 1); + EXPECT_EQ(inner_offsets[3], 2); + const auto& inner_keys = + get_nullable_nested_column(inner_map.get_keys()); + const auto& inner_strings = + assert_cast(inner_map.get_values()); + const auto& inner_string_data = + assert_cast(inner_strings.get_nested_column()); + ASSERT_EQ(inner_keys.size(), 2); + EXPECT_EQ(inner_keys.get_element(0), 101); + EXPECT_EQ(inner_keys.get_element(1), 401); + EXPECT_EQ(inner_string_data.get_data_at(0).to_string(), "aa"); + EXPECT_TRUE(inner_strings.is_null_at(1)); + }); + auto deep_list_value_type = arrow::list(arrow::field("element", arrow::int32(), true)); + auto deep_list_map_type = + arrow::map(arrow::int32(), arrow::field("value", deep_list_value_type, true)); + auto deep_list_struct_type = arrow::struct_({arrow::field("kv", deep_list_map_type, true)}); + add_field(arrow::field("nullable_list_struct_map_list_col", + arrow::list(arrow::field("element", deep_list_struct_type, true)), + true), + build_deep_list_struct_map_list_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& outer_array = + assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), ROW_COUNT); + EXPECT_EQ(outer_offsets[0], 2); + EXPECT_EQ(outer_offsets[1], 2); + EXPECT_EQ(outer_offsets[2], 2); + EXPECT_EQ(outer_offsets[3], 4); + EXPECT_EQ(outer_offsets[4], 5); + + const auto& struct_values = + assert_cast(outer_array.get_data()); + ASSERT_EQ(struct_values.size(), 5); + EXPECT_FALSE(struct_values.is_null_at(0)); + EXPECT_TRUE(struct_values.is_null_at(1)); + EXPECT_FALSE(struct_values.is_null_at(2)); + EXPECT_FALSE(struct_values.is_null_at(3)); + EXPECT_FALSE(struct_values.is_null_at(4)); + + const auto& struct_column = + assert_cast(struct_values.get_nested_column()); + const auto& map_values = + assert_cast(struct_column.get_column(0)); + ASSERT_EQ(map_values.size(), 5); + EXPECT_FALSE(map_values.is_null_at(0)); + EXPECT_TRUE(map_values.is_null_at(1)); + EXPECT_TRUE(map_values.is_null_at(2)); + EXPECT_FALSE(map_values.is_null_at(3)); + EXPECT_FALSE(map_values.is_null_at(4)); + + const auto& map_column = + assert_cast(map_values.get_nested_column()); + const auto& map_offsets = map_column.get_offsets(); + ASSERT_EQ(map_offsets.size(), 5); + EXPECT_EQ(map_offsets[0], 2); + EXPECT_EQ(map_offsets[1], 2); + EXPECT_EQ(map_offsets[2], 2); + EXPECT_EQ(map_offsets[3], 2); + EXPECT_EQ(map_offsets[4], 4); + const auto& keys = + get_nullable_nested_column(map_column.get_keys()); + ASSERT_EQ(keys.size(), 4); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(1), 2); + EXPECT_EQ(keys.get_element(2), 3); + EXPECT_EQ(keys.get_element(3), 4); + + const auto& lists = + assert_cast(map_column.get_values()); + ASSERT_EQ(lists.size(), 4); + EXPECT_FALSE(lists.is_null_at(0)); + EXPECT_FALSE(lists.is_null_at(1)); + EXPECT_TRUE(lists.is_null_at(2)); + EXPECT_FALSE(lists.is_null_at(3)); + const auto& list_column = + assert_cast(lists.get_nested_column()); + const auto& list_offsets = list_column.get_offsets(); + ASSERT_EQ(list_offsets.size(), 4); + EXPECT_EQ(list_offsets[0], 2); + EXPECT_EQ(list_offsets[1], 2); + EXPECT_EQ(list_offsets[2], 2); + EXPECT_EQ(list_offsets[3], 3); + const auto& elements = + assert_cast(list_column.get_data()); + const auto& element_values = + assert_cast(elements.get_nested_column()); + ASSERT_EQ(elements.size(), 3); + EXPECT_EQ(element_values.get_element(0), 10); + EXPECT_TRUE(elements.is_null_at(1)); + EXPECT_EQ(element_values.get_element(2), 40); + }); + auto deep_map_nested_map_type = + arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + auto deep_map_list_type = + arrow::list(arrow::field("element", deep_map_nested_map_type, true)); + add_field( + arrow::field( + "nullable_map_int_list_map_int_string_col", + arrow::map(arrow::int32(), arrow::field("value", deep_map_list_type, true)), + true), + build_deep_map_list_map_array(), + [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& outer_map = + assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_map.get_offsets(); + ASSERT_EQ(outer_offsets.size(), ROW_COUNT); + EXPECT_EQ(outer_offsets[0], 2); + EXPECT_EQ(outer_offsets[1], 2); + EXPECT_EQ(outer_offsets[2], 2); + EXPECT_EQ(outer_offsets[3], 4); + EXPECT_EQ(outer_offsets[4], 5); + const auto& outer_keys = + get_nullable_nested_column(outer_map.get_keys()); + ASSERT_EQ(outer_keys.size(), 5); + EXPECT_EQ(outer_keys.get_element(0), 10); + EXPECT_EQ(outer_keys.get_element(1), 20); + EXPECT_EQ(outer_keys.get_element(2), 30); + EXPECT_EQ(outer_keys.get_element(3), 40); + EXPECT_EQ(outer_keys.get_element(4), 50); + + const auto& lists = assert_cast(outer_map.get_values()); + ASSERT_EQ(lists.size(), 5); + EXPECT_FALSE(lists.is_null_at(0)); + EXPECT_FALSE(lists.is_null_at(1)); + EXPECT_TRUE(lists.is_null_at(2)); + EXPECT_FALSE(lists.is_null_at(3)); + EXPECT_FALSE(lists.is_null_at(4)); + const auto& list_column = + assert_cast(lists.get_nested_column()); + const auto& list_offsets = list_column.get_offsets(); + ASSERT_EQ(list_offsets.size(), 5); + EXPECT_EQ(list_offsets[0], 3); + EXPECT_EQ(list_offsets[1], 3); + EXPECT_EQ(list_offsets[2], 3); + EXPECT_EQ(list_offsets[3], 4); + EXPECT_EQ(list_offsets[4], 6); + + const auto& inner_maps = + assert_cast(list_column.get_data()); + ASSERT_EQ(inner_maps.size(), 6); + EXPECT_FALSE(inner_maps.is_null_at(0)); + EXPECT_FALSE(inner_maps.is_null_at(1)); + EXPECT_TRUE(inner_maps.is_null_at(2)); + EXPECT_FALSE(inner_maps.is_null_at(3)); + EXPECT_TRUE(inner_maps.is_null_at(4)); + EXPECT_FALSE(inner_maps.is_null_at(5)); + const auto& inner_map_column = + assert_cast(inner_maps.get_nested_column()); + const auto& inner_offsets = inner_map_column.get_offsets(); + ASSERT_EQ(inner_offsets.size(), 6); + EXPECT_EQ(inner_offsets[0], 2); + EXPECT_EQ(inner_offsets[1], 2); + EXPECT_EQ(inner_offsets[2], 2); + EXPECT_EQ(inner_offsets[3], 3); + EXPECT_EQ(inner_offsets[4], 3); + EXPECT_EQ(inner_offsets[5], 4); + const auto& inner_keys = + get_nullable_nested_column(inner_map_column.get_keys()); + ASSERT_EQ(inner_keys.size(), 4); + EXPECT_EQ(inner_keys.get_element(0), 1); + EXPECT_EQ(inner_keys.get_element(1), 2); + EXPECT_EQ(inner_keys.get_element(2), 3); + EXPECT_EQ(inner_keys.get_element(3), 4); + const auto& strings = + assert_cast(inner_map_column.get_values()); + const auto& string_data = + assert_cast(strings.get_nested_column()); + ASSERT_EQ(strings.size(), 4); + EXPECT_EQ(string_data.get_data_at(0).to_string(), "a"); + EXPECT_TRUE(strings.is_null_at(1)); + EXPECT_EQ(string_data.get_data_at(2).to_string(), "c"); + EXPECT_EQ(string_data.get_data_at(3).to_string(), "d"); + }); + + auto schema = arrow::schema(_arrow_fields); + auto table = arrow::Table::Make(schema, _arrays); + + auto file_result = arrow::io::FileOutputStream::Open(_file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); + } + + std::unique_ptr create_reader(size_t field_idx) const { + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + auto st = factory.create(*_fields[field_idx], &reader); + EXPECT_TRUE(st.ok()) << st; + return reader; + } + + std::unique_ptr create_plain_reader(size_t field_idx) { + // Keep the normal fixture dictionary encoded. This one test writes a plain-encoded copy + // because Arrow BinaryRecordReader has a stricter reset contract than + // DictionaryRecordReader. + auto schema = arrow::schema(_arrow_fields); + auto table = arrow::Table::Make(schema, _arrays); + auto plain_file_result = arrow::io::FileOutputStream::Open(_plain_file_path); + DORIS_CHECK(plain_file_result.ok()); + std::shared_ptr plain_out = *plain_file_result; + ::parquet::WriterProperties::Builder plain_builder; + plain_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + plain_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + plain_builder.compression(::parquet::Compression::UNCOMPRESSED); + plain_builder.disable_dictionary(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( + *table, arrow::default_memory_pool(), plain_out, ROW_COUNT, plain_builder.build())); + DORIS_CHECK(plain_out->Close().ok()); + + _plain_file_reader = ::parquet::ParquetFileReader::OpenFile(_plain_file_path, false); + auto metadata = _plain_file_reader->metadata(); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(metadata->num_row_groups() == 1); + _plain_row_group = _plain_file_reader->RowGroup(0); + DORIS_CHECK(_plain_row_group != nullptr); + + ParquetColumnReaderFactory factory(_plain_row_group, metadata->num_columns()); + std::unique_ptr reader; + auto st = factory.create(*_fields[field_idx], &reader); + EXPECT_TRUE(st.ok()) << st; + return reader; + } + + std::unique_ptr create_projected_child_reader(size_t field_idx, + size_t child_idx) const { + const auto& struct_schema = *_fields[field_idx]; + EXPECT_LT(child_idx, struct_schema.children.size()); + + format::LocalColumnIndex projection; + projection.index = struct_schema.local_id; + projection.project_all_children = false; + format::LocalColumnIndex child_projection; + child_projection.index = struct_schema.children[child_idx]->local_id; + projection.children.push_back(std::move(child_projection)); + + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + auto st = factory.create(struct_schema, &projection, &reader); + EXPECT_TRUE(st.ok()) << st; + return reader; + } + + std::unique_ptr create_projected_grandchild_reader( + size_t field_idx, size_t child_idx, size_t grandchild_idx) const { + const auto& struct_schema = *_fields[field_idx]; + EXPECT_LT(child_idx, struct_schema.children.size()); + const auto& child_schema = *struct_schema.children[child_idx]; + EXPECT_LT(grandchild_idx, child_schema.children.size()); + + format::LocalColumnIndex projection; + projection.index = struct_schema.local_id; + projection.project_all_children = false; + format::LocalColumnIndex child_projection; + child_projection.index = child_schema.local_id; + child_projection.project_all_children = false; + format::LocalColumnIndex grandchild_projection; + grandchild_projection.index = child_schema.children[grandchild_idx]->local_id; + child_projection.children.push_back(std::move(grandchild_projection)); + projection.children.push_back(std::move(child_projection)); + + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + auto st = factory.create(struct_schema, &projection, &reader); + EXPECT_TRUE(st.ok()) << st; + return reader; + } + + void read_and_validate(size_t field_idx) const { + auto reader = create_reader(field_idx); + ASSERT_NE(reader, nullptr); + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + auto st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + ASSERT_EQ(column->size(), ROW_COUNT); + _expected_by_field[field_idx](*_fields[field_idx], *column); + } + + size_t find_field_idx(const std::string& name) const { + for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { + if (_fields[field_idx]->name == name) { + return field_idx; + } + } + ADD_FAILURE() << "Cannot find parquet test field " << name; + return _fields.size(); + } + + std::filesystem::path _test_dir; + std::string _file_path; + std::string _plain_file_path; + std::unique_ptr<::parquet::ParquetFileReader> _file_reader; + std::unique_ptr<::parquet::ParquetFileReader> _plain_file_reader; + std::shared_ptr<::parquet::RowGroupReader> _row_group; + std::shared_ptr<::parquet::RowGroupReader> _plain_row_group; + std::vector> _fields; + std::vector> _arrow_fields; + std::vector> _arrays; + std::vector> _expected_by_field; +}; + +TEST(ParquetColumnReaderBaseTest, SelectionVectorRangesAndValidation) { + SelectionVector identity; + ASSERT_TRUE(identity.verify(4, 5).ok()); + auto ranges = selection_to_ranges(identity, 4); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges[0].start, 0); + EXPECT_EQ(ranges[0].length, 4); + + std::array selected = {0, 2, 3, 6, 6}; + SelectionVector external(selected.data(), 4); + auto status = external.verify(3, 7); + ASSERT_TRUE(status.ok()) << status; + ranges = selection_to_ranges(external, 3); + ASSERT_EQ(ranges.size(), 2); + EXPECT_EQ(ranges[0].start, 0); + EXPECT_EQ(ranges[0].length, 1); + EXPECT_EQ(ranges[1].start, 2); + EXPECT_EQ(ranges[1].length, 2); + + EXPECT_FALSE(external.verify(8, 7).ok()); + EXPECT_FALSE(external.verify(5, 7).ok()); + EXPECT_FALSE(external.verify(4, 6).ok()); + + std::array duplicate = {0, 2, 2}; + SelectionVector non_strict(duplicate.data(), duplicate.size()); + EXPECT_FALSE(non_strict.verify(3, 5).ok()); + EXPECT_FALSE(identity.verify(1, -1).ok()); +} + +TEST(ParquetColumnReaderBaseTest, DefaultSelectUsesSkipReadRangesAndNestedConsumeIsExplicit) { + DefaultSelectReader reader; + std::array selected = {1, 3, 4}; + SelectionVector selection(selected.data(), selected.size()); + auto column = ColumnInt32::create(); + MutableColumnPtr mutable_column = std::move(column); + auto status = reader.select(selection, selected.size(), 6, mutable_column); + ASSERT_TRUE(status.ok()) << status; + + const auto& values = assert_cast(*mutable_column); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(values.get_element(0), 1); + EXPECT_EQ(values.get_element(1), 3); + EXPECT_EQ(values.get_element(2), 4); + EXPECT_EQ(reader.skip_ranges(), std::vector({1, 1, 1})); + EXPECT_EQ(reader.read_ranges(), std::vector({1, 2})); + + BaseUnsupportedReader unsupported_reader; + auto skip_status = unsupported_reader.skip(1); + EXPECT_FALSE(skip_status.ok()); + EXPECT_NE(skip_status.to_string().find("skip is not implemented"), std::string::npos); + EXPECT_FALSE(unsupported_reader.load_nested_batch(1).ok()); + int64_t values_read = 0; + EXPECT_FALSE(unsupported_reader.build_nested_column(1, mutable_column, &values_read).ok()); + EXPECT_FALSE(unsupported_reader.consume_nested_column(1, &values_read).ok()); + + NestedSkipReader nested_reader; + auto nested_status = nested_reader.consume_nested_column(3, &values_read); + ASSERT_TRUE(nested_status.ok()) << nested_status; + EXPECT_EQ(values_read, 3); +} + +TEST_F(ParquetColumnReaderTest, ScalarReadCoversRequiredNullableAllNullAndMultipleBatches) { + read_and_validate(find_field_idx("int32_col")); + read_and_validate(find_field_idx("string_col")); + read_and_validate(find_field_idx("nullable_int_col")); + read_and_validate(find_field_idx("all_null_int_col")); + + auto reader = create_reader(find_field_idx("int32_col")); + auto column = reader->type()->create_column(); + int64_t rows_read = 0; + ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); + ASSERT_EQ(rows_read, 2); + ASSERT_TRUE(reader->read(3, column, &rows_read).ok()); + ASSERT_EQ(rows_read, 3); + const auto& values = assert_cast(*column); + ASSERT_EQ(values.size(), ROW_COUNT); + EXPECT_EQ(values.get_element(0), 10); + EXPECT_EQ(values.get_element(1), 20); + EXPECT_EQ(values.get_element(2), 30); + EXPECT_EQ(values.get_element(4), 50); +} + +TEST_F(ParquetColumnReaderTest, ScalarSkipCoversZeroSomeAllAndNulls) { + auto reader = create_reader(find_field_idx("int32_col")); + ASSERT_TRUE(reader->skip(0).ok()); + auto column = reader->type()->create_column(); + int64_t rows_read = 0; + ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); + ASSERT_EQ(rows_read, 1); + const auto& first_value = assert_cast(*column); + EXPECT_EQ(first_value.get_element(0), 10); + + reader = create_reader(find_field_idx("int32_col")); + ASSERT_TRUE(reader->skip(2).ok()); + column = reader->type()->create_column(); + ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); + ASSERT_EQ(rows_read, 2); + const auto& skipped_values = assert_cast(*column); + EXPECT_EQ(skipped_values.get_element(0), 30); + EXPECT_EQ(skipped_values.get_element(1), 40); + + reader = create_reader(find_field_idx("int32_col")); + ASSERT_TRUE(reader->skip(ROW_COUNT).ok()); + column = reader->type()->create_column(); + ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); + EXPECT_EQ(rows_read, 0); + EXPECT_EQ(column->size(), 0); + + reader = create_reader(find_field_idx("nullable_int_col")); + ASSERT_TRUE(reader->skip(1).ok()); + column = reader->type()->create_column(); + ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); + ASSERT_EQ(rows_read, 2); + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 2); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); +} + +TEST_F(ParquetColumnReaderTest, ScalarSelectCoversAllDisjointSingleZeroThenReadAndNulls) { + auto reader = create_reader(find_field_idx("int32_col")); + SelectionVector all_selected(ROW_COUNT); + auto column = reader->type()->create_column(); + ASSERT_TRUE(reader->select(all_selected, ROW_COUNT, ROW_COUNT, column).ok()); + const auto& all_values = assert_cast(*column); + ASSERT_EQ(all_values.size(), ROW_COUNT); + EXPECT_EQ(all_values.get_element(0), 10); + EXPECT_EQ(all_values.get_element(4), 50); + + reader = create_reader(find_field_idx("int32_col")); + std::array disjoint = {0, 2, 4}; + SelectionVector disjoint_selection(disjoint.data(), disjoint.size()); + column = reader->type()->create_column(); + ASSERT_TRUE(reader->select(disjoint_selection, disjoint.size(), ROW_COUNT, column).ok()); + const auto& disjoint_values = assert_cast(*column); + ASSERT_EQ(disjoint_values.size(), 3); + EXPECT_EQ(disjoint_values.get_element(0), 10); + EXPECT_EQ(disjoint_values.get_element(1), 30); + EXPECT_EQ(disjoint_values.get_element(2), 50); + + reader = create_reader(find_field_idx("int32_col")); + std::array single = {2}; + SelectionVector single_selection(single.data(), single.size()); + column = reader->type()->create_column(); + ASSERT_TRUE(reader->select(single_selection, single.size(), ROW_COUNT, column).ok()); + const auto& single_value = assert_cast(*column); + ASSERT_EQ(single_value.size(), 1); + EXPECT_EQ(single_value.get_element(0), 30); + + reader = create_reader(find_field_idx("int32_col")); + std::array first_last = {0, 4}; + SelectionVector first_last_selection(first_last.data(), first_last.size()); + column = reader->type()->create_column(); + ASSERT_TRUE(reader->select(first_last_selection, first_last.size(), ROW_COUNT, column).ok()); + const auto& first_last_values = assert_cast(*column); + ASSERT_EQ(first_last_values.size(), 2); + EXPECT_EQ(first_last_values.get_element(0), 10); + EXPECT_EQ(first_last_values.get_element(1), 50); + + reader = create_reader(find_field_idx("int32_col")); + SelectionVector empty_selection; + column = reader->type()->create_column(); + ASSERT_TRUE(reader->select(empty_selection, 0, 2, column).ok()); + ASSERT_EQ(column->size(), 0); + int64_t rows_read = 0; + ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); + ASSERT_EQ(rows_read, 1); + const auto& after_empty_select = assert_cast(*column); + ASSERT_EQ(after_empty_select.size(), 1); + EXPECT_EQ(after_empty_select.get_element(0), 30); + + reader = create_reader(find_field_idx("nullable_int_col")); + std::array nullable_rows = {0, 1, 2}; + SelectionVector nullable_selection(nullable_rows.data(), nullable_rows.size()); + column = reader->type()->create_column(); + ASSERT_TRUE(reader->select(nullable_selection, nullable_rows.size(), ROW_COUNT, column).ok()); + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); +} + +TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidScalarInputsAndNestedScalarProjection) { + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + + const auto& int_schema = *_fields[find_field_idx("int32_col")]; + ParquetColumnSchema invalid_leaf; + invalid_leaf.kind = ParquetColumnSchemaKind::PRIMITIVE; + invalid_leaf.name = "invalid_leaf"; + invalid_leaf.type = int_schema.type; + invalid_leaf.type_descriptor = int_schema.type_descriptor; + invalid_leaf.descriptor = int_schema.descriptor; + invalid_leaf.leaf_column_id = _file_reader->metadata()->num_columns(); + auto status = factory.create(invalid_leaf, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); + + ParquetColumnSchema null_descriptor; + null_descriptor.kind = ParquetColumnSchemaKind::PRIMITIVE; + null_descriptor.name = "null_descriptor"; + null_descriptor.type = int_schema.type; + null_descriptor.type_descriptor = int_schema.type_descriptor; + null_descriptor.leaf_column_id = int_schema.leaf_column_id; + status = factory.create(null_descriptor, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("descriptor is null"), std::string::npos); + + const auto& list_element_schema = + *_fields[find_field_idx("nullable_list_int_col")]->children[0]; + status = factory.create(list_element_schema, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("flat primitive columns"), std::string::npos); + + const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; + format::LocalColumnIndex projection = + format::LocalColumnIndex::partial_local(list_schema.local_id); + format::LocalColumnIndex element_projection = + format::LocalColumnIndex::partial_local(list_element_schema.local_id); + projection.children.push_back(std::move(element_projection)); + status = factory.create(list_schema, &projection, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("scalar projection is invalid"), std::string::npos); +} + +TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidComplexProjections) { + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + + const auto& struct_schema = *_fields[find_field_idx("struct_col")]; + format::LocalColumnIndex struct_empty = + format::LocalColumnIndex::partial_local(struct_schema.local_id); + auto status = factory.create(struct_schema, &struct_empty, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); + + format::LocalColumnIndex struct_invalid = + format::LocalColumnIndex::partial_local(struct_schema.local_id); + struct_invalid.children.push_back(format::LocalColumnIndex::local(9999)); + status = factory.create(struct_schema, &struct_invalid, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); + + const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; + format::LocalColumnIndex list_empty = + format::LocalColumnIndex::partial_local(list_schema.local_id); + status = factory.create(list_schema, &list_empty, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); + + const auto& map_schema = *_fields[find_field_idx("nullable_map_int_struct_col")]; + const auto& value_schema = *map_schema.children[1]; + format::LocalColumnIndex map_invalid = + format::LocalColumnIndex::partial_local(map_schema.local_id); + map_invalid.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); + map_invalid.children.push_back(format::LocalColumnIndex::local(9999)); + status = factory.create(map_schema, &map_invalid, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); +} + +TEST_F(ParquetColumnReaderTest, ReadSupportedComplexTypes) { + read_and_validate(find_field_idx("struct_col")); + read_and_validate(find_field_idx("nullable_struct_col")); + read_and_validate(find_field_idx("nullable_struct_decimal_col")); + read_and_validate(find_field_idx("list_int_col")); + read_and_validate(find_field_idx("nullable_list_int_col")); + read_and_validate(find_field_idx("required_nullable_list_int_col")); + read_and_validate(find_field_idx("nullable_list_struct_col")); + read_and_validate(find_field_idx("nullable_list_list_int_col")); + read_and_validate(find_field_idx("map_int_string_col")); + read_and_validate(find_field_idx("nullable_map_int_string_col")); + read_and_validate(find_field_idx("required_nullable_map_int_string_col")); + read_and_validate(find_field_idx("nullable_map_int_struct_col")); + read_and_validate(find_field_idx("nullable_map_int_list_col")); + read_and_validate(find_field_idx("nullable_list_map_int_string_col")); + read_and_validate(find_field_idx("nullable_map_int_map_int_string_col")); + read_and_validate(find_field_idx("nullable_list_struct_map_list_col")); + read_and_validate(find_field_idx("nullable_map_int_list_map_int_string_col")); +} + +TEST_F(ParquetColumnReaderTest, SkipThenRead) { + auto reader = create_reader(find_field_idx("int32_col")); + auto st = reader->skip(2); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + + const auto& int_values = assert_cast(*column); + ASSERT_EQ(int_values.size(), 2); + EXPECT_EQ(int_values.get_element(0), 30); + EXPECT_EQ(int_values.get_element(1), 40); +} + +TEST_F(ParquetColumnReaderTest, SelectReadsOnlySelectedRanges) { + auto reader = create_reader(find_field_idx("int32_col")); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 2); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& int_values = assert_cast(*column); + ASSERT_EQ(int_values.size(), 3); + EXPECT_EQ(int_values.get_element(0), 10); + EXPECT_EQ(int_values.get_element(1), 30); + EXPECT_EQ(int_values.get_element(2), 50); +} + +TEST_F(ParquetColumnReaderTest, ReadProjectedStructChildren) { + const auto field_idx = find_field_idx("struct_col"); + ASSERT_LT(field_idx, _fields.size()); + const auto& struct_schema = *_fields[field_idx]; + ASSERT_EQ(struct_schema.name, "struct_col"); + ASSERT_EQ(struct_schema.children.size(), 2); + + format::LocalColumnIndex projection; + projection.index = struct_schema.local_id; + projection.project_all_children = false; + format::LocalColumnIndex child_projection; + child_projection.index = struct_schema.children[1]->local_id; + projection.children.push_back(std::move(child_projection)); + + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + auto st = factory.create(struct_schema, &projection, &reader); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); + const auto* projected_type = + assert_cast(remove_nullable(reader->type()).get()); + ASSERT_EQ(projected_type->get_elements().size(), 1); + EXPECT_EQ(projected_type->get_element_name(0), "b"); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + const auto& struct_column = assert_cast(*column); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& values = get_nullable_nested_column(struct_column.get_column(0)); + EXPECT_EQ(values.get_data_at(0).to_string(), "sa"); + EXPECT_EQ(values.get_data_at(4).to_string(), "se"); +} + +TEST_F(ParquetColumnReaderTest, ReadProjectedNullableStructChildren) { + const auto field_idx = find_field_idx("nullable_struct_col"); + ASSERT_LT(field_idx, _fields.size()); + const auto& struct_schema = *_fields[field_idx]; + ASSERT_EQ(struct_schema.name, "nullable_struct_col"); + ASSERT_EQ(struct_schema.children.size(), 2); + + format::LocalColumnIndex projection; + projection.index = struct_schema.local_id; + projection.project_all_children = false; + format::LocalColumnIndex child_projection; + child_projection.index = struct_schema.children[1]->local_id; + projection.children.push_back(std::move(child_projection)); + + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + auto st = factory.create(struct_schema, &projection, &reader); + ASSERT_TRUE(st.ok()) << st; + ASSERT_TRUE(reader->type()->is_nullable()); + ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); + const auto* projected_type = + assert_cast(remove_nullable(reader->type()).get()); + ASSERT_EQ(projected_type->get_elements().size(), 1); + EXPECT_EQ(projected_type->get_element_name(0), "b"); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + const auto& nullable_column = assert_cast(*column); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_TRUE(nullable_column.is_null_at(4)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& values = assert_cast(struct_column.get_column(0)); + const auto& nested_values = assert_cast(values.get_nested_column()); + EXPECT_FALSE(values.is_null_at(0)); + EXPECT_TRUE(values.is_null_at(2)); + EXPECT_FALSE(values.is_null_at(3)); + EXPECT_EQ(nested_values.get_data_at(0).to_string(), "nsa"); + EXPECT_EQ(nested_values.get_data_at(3).to_string(), "nsd"); +} + +TEST_F(ParquetColumnReaderTest, ReadProjectedListStructElementChildren) { + const auto field_idx = find_field_idx("nullable_list_struct_col"); + ASSERT_LT(field_idx, _fields.size()); + const auto& list_schema = *_fields[field_idx]; + ASSERT_EQ(list_schema.name, "nullable_list_struct_col"); + ASSERT_EQ(list_schema.children.size(), 1); + const auto& element_schema = *list_schema.children[0]; + ASSERT_EQ(element_schema.children.size(), 2); + + format::LocalColumnIndex projection; + projection.index = list_schema.local_id; + projection.project_all_children = false; + format::LocalColumnIndex element_projection; + element_projection.index = element_schema.local_id; + element_projection.project_all_children = false; + format::LocalColumnIndex child_projection; + child_projection.index = element_schema.children[1]->local_id; + element_projection.children.push_back(std::move(child_projection)); + projection.children.push_back(std::move(element_projection)); + + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + auto st = factory.create(list_schema, &projection, &reader); + ASSERT_TRUE(st.ok()) << st; + ASSERT_TRUE(reader->type()->is_nullable()); + const auto* array_type = + assert_cast(remove_nullable(reader->type()).get()); + const auto* element_type = assert_cast( + remove_nullable(array_type->get_nested_type()).get()); + ASSERT_EQ(element_type->get_elements().size(), 1); + EXPECT_EQ(element_type->get_element_name(0), "b"); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + + const auto& nullable_column = assert_cast(*column); + const auto& array_column = assert_cast(nullable_column.get_nested_column()); + const auto& elements = assert_cast(array_column.get_data()); + const auto& struct_column = assert_cast(elements.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& b_values = assert_cast(struct_column.get_column(0)); + const auto& b_data = assert_cast(b_values.get_nested_column()); + ASSERT_EQ(elements.size(), 5); + EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); + EXPECT_TRUE(b_values.is_null_at(1)); + EXPECT_TRUE(elements.is_null_at(2)); + EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); + EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); +} + +TEST_F(ParquetColumnReaderTest, ReadProjectedMapStructValueChildren) { + const auto field_idx = find_field_idx("nullable_map_int_struct_col"); + ASSERT_LT(field_idx, _fields.size()); + const auto& map_schema = *_fields[field_idx]; + ASSERT_EQ(map_schema.name, "nullable_map_int_struct_col"); + ASSERT_EQ(map_schema.children.size(), 2); + const auto& value_schema = *map_schema.children[1]; + ASSERT_EQ(value_schema.children.size(), 2); + + format::LocalColumnIndex projection; + projection.index = map_schema.local_id; + projection.project_all_children = false; + format::LocalColumnIndex value_projection; + value_projection.index = value_schema.local_id; + value_projection.project_all_children = false; + format::LocalColumnIndex child_projection; + child_projection.index = value_schema.children[1]->local_id; + value_projection.children.push_back(std::move(child_projection)); + projection.children.push_back(std::move(value_projection)); + + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + auto st = factory.create(map_schema, &projection, &reader); + ASSERT_TRUE(st.ok()) << st; + ASSERT_TRUE(reader->type()->is_nullable()); + const auto* map_type = assert_cast(remove_nullable(reader->type()).get()); + EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), TYPE_INT); + const auto* value_type = + assert_cast(remove_nullable(map_type->get_value_type()).get()); + ASSERT_EQ(value_type->get_elements().size(), 1); + EXPECT_EQ(value_type->get_element_name(0), "b"); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + + const auto& nullable_column = assert_cast(*column); + const auto& map_column = assert_cast(nullable_column.get_nested_column()); + const auto& keys = get_nullable_nested_column(map_column.get_keys()); + const auto& values = assert_cast(map_column.get_values()); + const auto& struct_column = assert_cast(values.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& b_values = assert_cast(struct_column.get_column(0)); + const auto& b_data = assert_cast(b_values.get_nested_column()); + ASSERT_EQ(keys.size(), 4); + ASSERT_EQ(values.size(), 4); + EXPECT_EQ(keys.get_element(0), 101); + EXPECT_EQ(keys.get_element(1), 102); + EXPECT_EQ(keys.get_element(3), 104); + EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); + EXPECT_TRUE(b_values.is_null_at(1)); + EXPECT_TRUE(values.is_null_at(2)); + EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); +} + +TEST_F(ParquetColumnReaderTest, AllowsMapKeyWithValueProjection) { + const auto field_idx = find_field_idx("nullable_map_int_struct_col"); + ASSERT_LT(field_idx, _fields.size()); + const auto& map_schema = *_fields[field_idx]; + ASSERT_EQ(map_schema.children.size(), 2); + const auto& key_schema = *map_schema.children[0]; + const auto& value_schema = *map_schema.children[1]; + + auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); + projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); + projection.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); + + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + const auto st = factory.create(map_schema, &projection, &reader); + ASSERT_TRUE(st.ok()) << st; + ASSERT_NE(reader, nullptr); +} + +TEST_F(ParquetColumnReaderTest, RejectMapKeyOnlyProjection) { + const auto field_idx = find_field_idx("nullable_map_int_struct_col"); + ASSERT_LT(field_idx, _fields.size()); + const auto& map_schema = *_fields[field_idx]; + ASSERT_EQ(map_schema.children.size(), 2); + const auto& key_schema = *map_schema.children[0]; + + auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); + projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); + + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + const auto st = factory.create(map_schema, &projection, &reader); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("contains no value"), std::string::npos); +} + +TEST_F(ParquetColumnReaderTest, ReadProjectedStructListChildOnly) { + const auto field_idx = find_field_idx("nullable_struct_list_col"); + ASSERT_LT(field_idx, _fields.size()); + const auto& struct_schema = *_fields[field_idx]; + ASSERT_EQ(struct_schema.name, "nullable_struct_list_col"); + ASSERT_EQ(struct_schema.children.size(), 2); + + auto reader = create_projected_child_reader(field_idx, 1); + ASSERT_NE(reader, nullptr); + ASSERT_TRUE(reader->type()->is_nullable()); + const auto* projected_type = + assert_cast(remove_nullable(reader->type()).get()); + ASSERT_EQ(projected_type->get_elements().size(), 1); + EXPECT_EQ(projected_type->get_element_name(0), "xs"); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& xs_nullable = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(xs_nullable.size(), ROW_COUNT); + EXPECT_FALSE(xs_nullable.is_null_at(0)); + EXPECT_FALSE(xs_nullable.is_null_at(2)); + EXPECT_TRUE(xs_nullable.is_null_at(3)); + EXPECT_FALSE(xs_nullable.is_null_at(4)); + const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); + const auto& offsets = xs_array.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 2); + EXPECT_EQ(offsets[3], 2); + EXPECT_EQ(offsets[4], 4); + const auto& elements = assert_cast(xs_array.get_data()); + const auto& values = assert_cast(elements.get_nested_column()); + ASSERT_EQ(elements.size(), 4); + EXPECT_EQ(values.get_element(0), 1); + EXPECT_EQ(values.get_element(1), 2); + EXPECT_TRUE(elements.is_null_at(2)); + EXPECT_EQ(values.get_element(3), 5); +} + +TEST_F(ParquetColumnReaderTest, SkipProjectedStructListChildOnlyThenRead) { + const auto field_idx = find_field_idx("nullable_struct_list_col"); + auto reader = create_projected_child_reader(field_idx, 1); + ASSERT_NE(reader, nullptr); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& xs_nullable = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(xs_nullable.size(), 3); + EXPECT_FALSE(xs_nullable.is_null_at(1)); + EXPECT_TRUE(xs_nullable.is_null_at(2)); + const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); + const auto& offsets = xs_array.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 0); + EXPECT_EQ(offsets[2], 0); +} + +TEST_F(ParquetColumnReaderTest, SelectProjectedStructListChildOnly) { + const auto field_idx = find_field_idx("nullable_struct_list_col"); + auto reader = create_projected_child_reader(field_idx, 1); + ASSERT_NE(reader, nullptr); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& xs_nullable = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(xs_nullable.size(), 3); + EXPECT_FALSE(xs_nullable.is_null_at(0)); + EXPECT_TRUE(xs_nullable.is_null_at(1)); + EXPECT_FALSE(xs_nullable.is_null_at(2)); + const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); + const auto& offsets = xs_array.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 4); +} + +TEST_F(ParquetColumnReaderTest, ReadProjectedStructMapChildOnly) { + const auto field_idx = find_field_idx("nullable_struct_map_col"); + ASSERT_LT(field_idx, _fields.size()); + const auto& struct_schema = *_fields[field_idx]; + ASSERT_EQ(struct_schema.name, "nullable_struct_map_col"); + ASSERT_EQ(struct_schema.children.size(), 2); + + auto reader = create_projected_child_reader(field_idx, 1); + ASSERT_NE(reader, nullptr); + ASSERT_TRUE(reader->type()->is_nullable()); + const auto* projected_type = + assert_cast(remove_nullable(reader->type()).get()); + ASSERT_EQ(projected_type->get_elements().size(), 1); + EXPECT_EQ(projected_type->get_element_name(0), "kv"); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& kv_nullable = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(kv_nullable.size(), ROW_COUNT); + EXPECT_FALSE(kv_nullable.is_null_at(0)); + EXPECT_FALSE(kv_nullable.is_null_at(2)); + EXPECT_TRUE(kv_nullable.is_null_at(3)); + EXPECT_FALSE(kv_nullable.is_null_at(4)); + const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); + const auto& offsets = kv_map.get_offsets(); + ASSERT_EQ(offsets.size(), ROW_COUNT); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 2); + EXPECT_EQ(offsets[3], 2); + EXPECT_EQ(offsets[4], 3); + const auto& keys = get_nullable_nested_column(kv_map.get_keys()); + const auto& values = assert_cast(kv_map.get_values()); + const auto& value_data = assert_cast(values.get_nested_column()); + ASSERT_EQ(keys.size(), 3); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(1), 2); + EXPECT_EQ(keys.get_element(2), 5); + EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); + EXPECT_TRUE(values.is_null_at(1)); + EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); +} + +TEST_F(ParquetColumnReaderTest, NullableStructUsesListChildAsShapeSource) { + const auto field_idx = find_field_idx("nullable_struct_list_col"); + auto reader = create_projected_child_reader(field_idx, 1); + ASSERT_NE(reader, nullptr); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + auto st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); +} + +TEST_F(ParquetColumnReaderTest, NullableStructUsesMapChildAsShapeSource) { + const auto field_idx = find_field_idx("nullable_struct_map_col"); + auto reader = create_projected_child_reader(field_idx, 1); + ASSERT_NE(reader, nullptr); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + auto st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); +} + +TEST_F(ParquetColumnReaderTest, NullableStructUsesNestedStructComplexChildAsShapeSource) { + const auto field_idx = find_field_idx("nullable_struct_nested_struct_list_col"); + auto reader = create_projected_grandchild_reader(field_idx, 0, 0); + ASSERT_NE(reader, nullptr); + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + auto st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_FALSE(nullable_column.is_null_at(4)); + + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + const auto& nested_nullable = assert_cast(struct_column.get_column(0)); + EXPECT_FALSE(nested_nullable.is_null_at(0)); + EXPECT_TRUE(nested_nullable.is_null_at(2)); + EXPECT_FALSE(nested_nullable.is_null_at(3)); + EXPECT_FALSE(nested_nullable.is_null_at(4)); +} + +TEST_F(ParquetColumnReaderTest, SkipProjectedStructMapChildOnlyThenRead) { + const auto field_idx = find_field_idx("nullable_struct_map_col"); + auto reader = create_projected_child_reader(field_idx, 1); + ASSERT_NE(reader, nullptr); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& kv_nullable = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(kv_nullable.size(), 3); + EXPECT_FALSE(kv_nullable.is_null_at(1)); + EXPECT_TRUE(kv_nullable.is_null_at(2)); + const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); + const auto& offsets = kv_map.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 0); + EXPECT_EQ(offsets[2], 0); +} + +TEST_F(ParquetColumnReaderTest, SelectProjectedStructMapChildOnly) { + const auto field_idx = find_field_idx("nullable_struct_map_col"); + auto reader = create_projected_child_reader(field_idx, 1); + ASSERT_NE(reader, nullptr); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(struct_column.get_columns().size(), 1); + const auto& kv_nullable = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(kv_nullable.size(), 3); + EXPECT_FALSE(kv_nullable.is_null_at(0)); + EXPECT_TRUE(kv_nullable.is_null_at(1)); + EXPECT_FALSE(kv_nullable.is_null_at(2)); + const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); + const auto& offsets = kv_map.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 3); + const auto& keys = get_nullable_nested_column(kv_map.get_keys()); + ASSERT_EQ(keys.size(), 3); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(1), 2); + EXPECT_EQ(keys.get_element(2), 5); +} + +TEST_F(ParquetColumnReaderTest, ReadListWithOverflowAcrossChunks) { + const auto field_idx = find_field_idx("nullable_list_int_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipListWithOverflowThenRead) { + const auto field_idx = find_field_idx("nullable_list_int_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + const auto& array_column = assert_cast(nullable_column.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 0); + EXPECT_EQ(offsets[2], 2); +} + +TEST_F(ParquetColumnReaderTest, SelectListWithOverflow) { + const auto field_idx = find_field_idx("nullable_list_int_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& array_column = assert_cast(nullable_column.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 4); + EXPECT_EQ(offsets[2], 5); +} + +TEST_F(ParquetColumnReaderTest, ReadStructListWithOverflowAcrossChunks) { + const auto field_idx = find_field_idx("nullable_struct_list_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipStructListWithOverflowThenRead) { + const auto field_idx = find_field_idx("nullable_struct_list_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + const auto& xs_nullable = assert_cast(struct_column.get_column(1)); + ASSERT_EQ(xs_nullable.size(), 3); + EXPECT_FALSE(xs_nullable.is_null_at(1)); + EXPECT_TRUE(xs_nullable.is_null_at(2)); + const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); + const auto& offsets = xs_array.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 0); + EXPECT_EQ(offsets[2], 0); +} + +TEST_F(ParquetColumnReaderTest, SelectStructListWithOverflow) { + const auto field_idx = find_field_idx("nullable_struct_list_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); + EXPECT_EQ(a_values.get_element(0), 301); + EXPECT_EQ(a_values.get_element(1), 304); + EXPECT_EQ(a_values.get_element(2), 305); + const auto& xs_nullable = assert_cast(struct_column.get_column(1)); + ASSERT_EQ(xs_nullable.size(), 3); + EXPECT_FALSE(xs_nullable.is_null_at(0)); + EXPECT_TRUE(xs_nullable.is_null_at(1)); + EXPECT_FALSE(xs_nullable.is_null_at(2)); + const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); + const auto& offsets = xs_array.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 4); +} + +TEST_F(ParquetColumnReaderTest, ReadStructMapWithOverflowAcrossChunks) { + const auto field_idx = find_field_idx("nullable_struct_map_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipStructMapWithOverflowThenRead) { + const auto field_idx = find_field_idx("nullable_struct_map_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + const auto& kv_nullable = assert_cast(struct_column.get_column(1)); + ASSERT_EQ(kv_nullable.size(), 3); + EXPECT_FALSE(kv_nullable.is_null_at(1)); + EXPECT_TRUE(kv_nullable.is_null_at(2)); + const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); + const auto& offsets = kv_map.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 0); + EXPECT_EQ(offsets[2], 0); +} + +TEST_F(ParquetColumnReaderTest, SelectStructMapWithOverflow) { + const auto field_idx = find_field_idx("nullable_struct_map_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); + EXPECT_EQ(a_values.get_element(0), 401); + EXPECT_EQ(a_values.get_element(1), 404); + EXPECT_EQ(a_values.get_element(2), 405); + const auto& kv_nullable = assert_cast(struct_column.get_column(1)); + ASSERT_EQ(kv_nullable.size(), 3); + EXPECT_FALSE(kv_nullable.is_null_at(0)); + EXPECT_TRUE(kv_nullable.is_null_at(1)); + EXPECT_FALSE(kv_nullable.is_null_at(2)); + const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); + const auto& offsets = kv_map.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 2); + EXPECT_EQ(offsets[2], 3); + const auto& keys = get_nullable_nested_column(kv_map.get_keys()); + const auto& values = assert_cast(kv_map.get_values()); + const auto& value_data = assert_cast(values.get_nested_column()); + ASSERT_EQ(keys.size(), 3); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(1), 2); + EXPECT_EQ(keys.get_element(2), 5); + EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); + EXPECT_TRUE(values.is_null_at(1)); + EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); +} + +TEST_F(ParquetColumnReaderTest, ReadListStructWithOverflowAcrossChunks) { + const auto field_idx = find_field_idx("nullable_list_struct_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipListStructWithOverflowThenRead) { + const auto field_idx = find_field_idx("nullable_list_struct_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + const auto& array_column = assert_cast(nullable_column.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 0); + EXPECT_EQ(offsets[2], 2); +} + +TEST_F(ParquetColumnReaderTest, SelectListStructWithOverflow) { + const auto field_idx = find_field_idx("nullable_list_struct_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& array_column = assert_cast(nullable_column.get_nested_column()); + const auto& offsets = array_column.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 4); + EXPECT_EQ(offsets[2], 5); +} + +TEST_F(ParquetColumnReaderTest, ReadListListWithOverflowAcrossChunks) { + const auto field_idx = find_field_idx("nullable_list_list_int_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipListListWithOverflowThenRead) { + const auto field_idx = find_field_idx("nullable_list_list_int_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + const auto& outer_array = assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), 3); + EXPECT_EQ(outer_offsets[0], 0); + EXPECT_EQ(outer_offsets[1], 0); + EXPECT_EQ(outer_offsets[2], 1); + + const auto& inner_nullable = assert_cast(outer_array.get_data()); + ASSERT_EQ(inner_nullable.size(), 1); + EXPECT_FALSE(inner_nullable.is_null_at(0)); + const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); + const auto& inner_offsets = inner_array.get_offsets(); + ASSERT_EQ(inner_offsets.size(), 1); + EXPECT_EQ(inner_offsets[0], 1); +} + +TEST_F(ParquetColumnReaderTest, SelectListListWithOverflow) { + const auto field_idx = find_field_idx("nullable_list_list_int_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& outer_array = assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), 3); + EXPECT_EQ(outer_offsets[0], 4); + EXPECT_EQ(outer_offsets[1], 5); + EXPECT_EQ(outer_offsets[2], 7); + + const auto& inner_nullable = assert_cast(outer_array.get_data()); + ASSERT_EQ(inner_nullable.size(), 7); + EXPECT_TRUE(inner_nullable.is_null_at(2)); + const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); + const auto& inner_offsets = inner_array.get_offsets(); + ASSERT_EQ(inner_offsets.size(), 7); + EXPECT_EQ(inner_offsets[0], 2); + EXPECT_EQ(inner_offsets[3], 4); + EXPECT_EQ(inner_offsets[4], 5); + EXPECT_EQ(inner_offsets[6], 7); +} + +TEST_F(ParquetColumnReaderTest, ReadMapWithOverflowAcrossChunks) { + const auto field_idx = find_field_idx("nullable_map_int_string_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipMapWithOverflowThenRead) { + const auto field_idx = find_field_idx("nullable_map_int_string_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + const auto& map_column = assert_cast(nullable_column.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 0); + EXPECT_EQ(offsets[2], 1); +} + +TEST_F(ParquetColumnReaderTest, SkipPlainBinaryMapThenReadResetsArrowBuilder) { + const auto field_idx = find_field_idx("nullable_map_int_string_col"); + auto reader = create_plain_reader(field_idx); + + // Row 0 contains two STRING values. The levels-only skip must release (and discard) those + // Arrow BinaryRecordReader builder chunks before the next normal read. If they leak into the + // next batch, ParquetLeafReader observes more values than current definition/repetition levels. + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + const auto& map_column = assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), 3); + EXPECT_EQ(map_column.get_offsets()[0], 0); + EXPECT_EQ(map_column.get_offsets()[1], 0); + EXPECT_EQ(map_column.get_offsets()[2], 1); + const auto& values = get_nullable_nested_column(map_column.get_values()); + ASSERT_EQ(values.size(), 1); + EXPECT_EQ(values.get_data_at(0).to_string(), "cc"); +} + +TEST_F(ParquetColumnReaderTest, SelectMapWithOverflow) { + const auto field_idx = find_field_idx("nullable_map_int_string_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& map_column = assert_cast(nullable_column.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 3); + EXPECT_EQ(offsets[2], 4); +} + +TEST_F(ParquetColumnReaderTest, ReadMapStructWithOverflowAcrossChunks) { + const auto field_idx = find_field_idx("nullable_map_int_struct_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipMapStructWithOverflowThenRead) { + const auto field_idx = find_field_idx("nullable_map_int_struct_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + const auto& map_column = assert_cast(nullable_column.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 0); + EXPECT_EQ(offsets[1], 0); + EXPECT_EQ(offsets[2], 1); +} + +TEST_F(ParquetColumnReaderTest, SelectMapStructWithOverflow) { + const auto field_idx = find_field_idx("nullable_map_int_struct_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& map_column = assert_cast(nullable_column.get_nested_column()); + const auto& offsets = map_column.get_offsets(); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 2); + EXPECT_EQ(offsets[1], 3); + EXPECT_EQ(offsets[2], 4); +} + +TEST_F(ParquetColumnReaderTest, ReadMapListWithOverflowAcrossChunks) { + const auto field_idx = find_field_idx("nullable_map_int_list_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipMapListWithOverflowThenRead) { + const auto field_idx = find_field_idx("nullable_map_int_list_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(3, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 3); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_TRUE(nullable_column.is_null_at(0)); + const auto& map_column = assert_cast(nullable_column.get_nested_column()); + const auto& map_offsets = map_column.get_offsets(); + ASSERT_EQ(map_offsets.size(), 3); + EXPECT_EQ(map_offsets[0], 0); + EXPECT_EQ(map_offsets[1], 0); + EXPECT_EQ(map_offsets[2], 2); + + const auto& values = assert_cast(map_column.get_values()); + ASSERT_EQ(values.size(), 2); + EXPECT_TRUE(values.is_null_at(0)); + EXPECT_FALSE(values.is_null_at(1)); + const auto& list_column = assert_cast(values.get_nested_column()); + const auto& list_offsets = list_column.get_offsets(); + ASSERT_EQ(list_offsets.size(), 2); + EXPECT_EQ(list_offsets[0], 0); + EXPECT_EQ(list_offsets[1], 2); +} + +TEST_F(ParquetColumnReaderTest, SelectMapListWithOverflow) { + const auto field_idx = find_field_idx("nullable_map_int_list_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& map_column = assert_cast(nullable_column.get_nested_column()); + const auto& map_offsets = map_column.get_offsets(); + ASSERT_EQ(map_offsets.size(), 3); + EXPECT_EQ(map_offsets[0], 2); + EXPECT_EQ(map_offsets[1], 4); + EXPECT_EQ(map_offsets[2], 5); + + const auto& values = assert_cast(map_column.get_values()); + ASSERT_EQ(values.size(), 5); + EXPECT_FALSE(values.is_null_at(0)); + EXPECT_TRUE(values.is_null_at(2)); + EXPECT_FALSE(values.is_null_at(4)); + const auto& list_column = assert_cast(values.get_nested_column()); + const auto& list_offsets = list_column.get_offsets(); + ASSERT_EQ(list_offsets.size(), 5); + EXPECT_EQ(list_offsets[0], 2); + EXPECT_EQ(list_offsets[1], 2); + EXPECT_EQ(list_offsets[2], 2); + EXPECT_EQ(list_offsets[3], 4); + EXPECT_EQ(list_offsets[4], 5); +} + +TEST_F(ParquetColumnReaderTest, ReadDeepListStructMapListAcrossChunks) { + const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(1, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 1); + st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipDeepListStructMapListThenRead) { + const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(4, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 4); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 4); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + + const auto& outer_array = assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), 4); + EXPECT_EQ(outer_offsets[0], 0); + EXPECT_EQ(outer_offsets[1], 0); + EXPECT_EQ(outer_offsets[2], 2); + EXPECT_EQ(outer_offsets[3], 3); + + const auto& struct_values = assert_cast(outer_array.get_data()); + ASSERT_EQ(struct_values.size(), 3); + EXPECT_FALSE(struct_values.is_null_at(0)); + EXPECT_FALSE(struct_values.is_null_at(1)); + EXPECT_FALSE(struct_values.is_null_at(2)); + const auto& struct_column = assert_cast(struct_values.get_nested_column()); + const auto& map_values = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(map_values.size(), 3); + EXPECT_TRUE(map_values.is_null_at(0)); + EXPECT_FALSE(map_values.is_null_at(1)); + EXPECT_FALSE(map_values.is_null_at(2)); + + const auto& map_column = assert_cast(map_values.get_nested_column()); + const auto& map_offsets = map_column.get_offsets(); + ASSERT_EQ(map_offsets.size(), 3); + EXPECT_EQ(map_offsets[0], 0); + EXPECT_EQ(map_offsets[1], 0); + EXPECT_EQ(map_offsets[2], 2); + const auto& keys = get_nullable_nested_column(map_column.get_keys()); + ASSERT_EQ(keys.size(), 2); + EXPECT_EQ(keys.get_element(0), 3); + EXPECT_EQ(keys.get_element(1), 4); + const auto& lists = assert_cast(map_column.get_values()); + ASSERT_EQ(lists.size(), 2); + EXPECT_TRUE(lists.is_null_at(0)); + EXPECT_FALSE(lists.is_null_at(1)); + const auto& list_column = assert_cast(lists.get_nested_column()); + const auto& list_offsets = list_column.get_offsets(); + ASSERT_EQ(list_offsets.size(), 2); + EXPECT_EQ(list_offsets[0], 0); + EXPECT_EQ(list_offsets[1], 1); +} + +TEST_F(ParquetColumnReaderTest, SelectDeepListStructMapList) { + const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& outer_array = assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), 3); + EXPECT_EQ(outer_offsets[0], 2); + EXPECT_EQ(outer_offsets[1], 4); + EXPECT_EQ(outer_offsets[2], 5); + + const auto& struct_values = assert_cast(outer_array.get_data()); + ASSERT_EQ(struct_values.size(), 5); + EXPECT_FALSE(struct_values.is_null_at(0)); + EXPECT_TRUE(struct_values.is_null_at(1)); + EXPECT_FALSE(struct_values.is_null_at(2)); + EXPECT_FALSE(struct_values.is_null_at(3)); + EXPECT_FALSE(struct_values.is_null_at(4)); + const auto& struct_column = assert_cast(struct_values.get_nested_column()); + const auto& map_values = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(map_values.size(), 5); + EXPECT_FALSE(map_values.is_null_at(0)); + EXPECT_TRUE(map_values.is_null_at(1)); + EXPECT_TRUE(map_values.is_null_at(2)); + EXPECT_FALSE(map_values.is_null_at(3)); + EXPECT_FALSE(map_values.is_null_at(4)); + const auto& map_column = assert_cast(map_values.get_nested_column()); + const auto& map_offsets = map_column.get_offsets(); + ASSERT_EQ(map_offsets.size(), 5); + EXPECT_EQ(map_offsets[0], 2); + EXPECT_EQ(map_offsets[1], 2); + EXPECT_EQ(map_offsets[2], 2); + EXPECT_EQ(map_offsets[3], 2); + EXPECT_EQ(map_offsets[4], 4); +} + +TEST_F(ParquetColumnReaderTest, ReadDeepMapListMapAcrossChunks) { + const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); + auto reader = create_reader(field_idx); + MutableColumnPtr column = reader->type()->create_column(); + + int64_t rows_read = 0; + auto st = reader->read(1, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 1); + st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + st = reader->read(2, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 2); + + _expected_by_field[field_idx](*_fields[field_idx], *column); +} + +TEST_F(ParquetColumnReaderTest, SkipDeepMapListMapThenRead) { + const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); + auto reader = create_reader(field_idx); + auto st = reader->skip(1); + ASSERT_TRUE(st.ok()) << st; + + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + st = reader->read(4, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, 4); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 4); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + const auto& outer_map = assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_map.get_offsets(); + ASSERT_EQ(outer_offsets.size(), 4); + EXPECT_EQ(outer_offsets[0], 0); + EXPECT_EQ(outer_offsets[1], 0); + EXPECT_EQ(outer_offsets[2], 2); + EXPECT_EQ(outer_offsets[3], 3); + const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); + ASSERT_EQ(outer_keys.size(), 3); + EXPECT_EQ(outer_keys.get_element(0), 30); + EXPECT_EQ(outer_keys.get_element(1), 40); + EXPECT_EQ(outer_keys.get_element(2), 50); + + const auto& lists = assert_cast(outer_map.get_values()); + ASSERT_EQ(lists.size(), 3); + EXPECT_TRUE(lists.is_null_at(0)); + EXPECT_FALSE(lists.is_null_at(1)); + EXPECT_FALSE(lists.is_null_at(2)); + const auto& list_column = assert_cast(lists.get_nested_column()); + const auto& list_offsets = list_column.get_offsets(); + ASSERT_EQ(list_offsets.size(), 3); + EXPECT_EQ(list_offsets[0], 0); + EXPECT_EQ(list_offsets[1], 1); + EXPECT_EQ(list_offsets[2], 3); + const auto& inner_maps = assert_cast(list_column.get_data()); + ASSERT_EQ(inner_maps.size(), 3); + EXPECT_FALSE(inner_maps.is_null_at(0)); + EXPECT_TRUE(inner_maps.is_null_at(1)); + EXPECT_FALSE(inner_maps.is_null_at(2)); +} + +TEST_F(ParquetColumnReaderTest, SelectDeepMapListMap) { + const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); + auto reader = create_reader(field_idx); + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 3); + selection.set_index(2, 4); + + MutableColumnPtr column = reader->type()->create_column(); + auto st = reader->select(selection, 3, ROW_COUNT, column); + ASSERT_TRUE(st.ok()) << st; + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 3); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + const auto& outer_map = assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_map.get_offsets(); + ASSERT_EQ(outer_offsets.size(), 3); + EXPECT_EQ(outer_offsets[0], 2); + EXPECT_EQ(outer_offsets[1], 4); + EXPECT_EQ(outer_offsets[2], 5); + const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); + ASSERT_EQ(outer_keys.size(), 5); + EXPECT_EQ(outer_keys.get_element(0), 10); + EXPECT_EQ(outer_keys.get_element(1), 20); + EXPECT_EQ(outer_keys.get_element(2), 30); + EXPECT_EQ(outer_keys.get_element(3), 40); + EXPECT_EQ(outer_keys.get_element(4), 50); + + const auto& lists = assert_cast(outer_map.get_values()); + ASSERT_EQ(lists.size(), 5); + EXPECT_FALSE(lists.is_null_at(0)); + EXPECT_FALSE(lists.is_null_at(1)); + EXPECT_TRUE(lists.is_null_at(2)); + EXPECT_FALSE(lists.is_null_at(3)); + EXPECT_FALSE(lists.is_null_at(4)); + const auto& list_column = assert_cast(lists.get_nested_column()); + const auto& list_offsets = list_column.get_offsets(); + ASSERT_EQ(list_offsets.size(), 5); + EXPECT_EQ(list_offsets[0], 3); + EXPECT_EQ(list_offsets[1], 3); + EXPECT_EQ(list_offsets[2], 3); + EXPECT_EQ(list_offsets[3], 4); + EXPECT_EQ(list_offsets[4], 6); +} + +} // namespace +} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp new file mode 100644 index 00000000000000..0d0f9a2f8567cc --- /dev/null +++ b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp @@ -0,0 +1,506 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/parquet_leaf_reader.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" + +namespace doris::format::parquet { +namespace { + +std::shared_ptr fixed_binary_array(const std::vector& values, + int byte_width) { + auto type = arrow::fixed_size_binary(byte_width); + arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); + } + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +ParquetLeafReader make_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type) { + return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr); +} + +struct CapturedDecodedView { + DecodedValueKind value_kind = DecodedValueKind::INT32; + DecodedTimeUnit time_unit = DecodedTimeUnit::UNKNOWN; + int64_t row_count = 0; + int decimal_precision = -1; + int decimal_scale = -1; + int fixed_length = -1; + bool timestamp_is_adjusted_to_utc = false; + bool enable_strict_mode = false; + const cctz::time_zone* timezone = nullptr; + bool null_map_is_null = true; + std::vector null_map; + std::vector fixed_values; + std::vector binary_values; + std::vector owned_binary_values; +}; + +ParquetLeafReader make_spy_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type, + CapturedDecodedView* captured, + const cctz::time_zone* timezone = nullptr, + bool enable_strict_mode = false) { + auto appender = [captured](MutableColumnPtr&, const DecodedColumnView& view) { + captured->value_kind = view.value_kind; + captured->time_unit = view.time_unit; + captured->row_count = view.row_count; + captured->decimal_precision = view.decimal_precision; + captured->decimal_scale = view.decimal_scale; + captured->fixed_length = view.fixed_length; + captured->timestamp_is_adjusted_to_utc = view.timestamp_is_adjusted_to_utc; + captured->enable_strict_mode = view.enable_strict_mode; + captured->timezone = view.timezone; + captured->null_map_is_null = view.null_map == nullptr; + captured->null_map.clear(); + if (view.null_map != nullptr) { + captured->null_map.assign(view.null_map, view.null_map + view.row_count); + } + captured->fixed_values.clear(); + if (view.values != nullptr && view.value_kind == DecodedValueKind::INT64) { + captured->fixed_values.assign(view.values, view.values + view.row_count * 8); + } else if (view.values != nullptr && view.value_kind == DecodedValueKind::FLOAT) { + captured->fixed_values.assign(view.values, view.values + view.row_count * 4); + } else if (view.values != nullptr && view.value_kind == DecodedValueKind::INT32) { + captured->fixed_values.assign(view.values, view.values + view.row_count * 4); + } + captured->binary_values.clear(); + captured->owned_binary_values.clear(); + if (view.binary_values != nullptr) { + captured->owned_binary_values.reserve(view.binary_values->size()); + for (const auto& value : *view.binary_values) { + captured->owned_binary_values.emplace_back( + value.data == nullptr ? std::string() + : std::string(value.data, value.size)); + } + captured->binary_values.reserve(captured->owned_binary_values.size()); + for (const auto& value : captured->owned_binary_values) { + captured->binary_values.emplace_back(value.data(), value.size()); + } + } + return Status::OK(); + }; + return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr, {}, timezone, + enable_strict_mode, std::move(appender)); +} + +} // namespace + +struct ParquetLeafReaderTestAccess { + static ParquetLeafBatch make_fixed_batch(const std::vector& def_levels, + const std::vector& rep_levels, + const std::vector& values, + bool read_dense_for_nullable = false) { + ParquetLeafBatch batch; + batch._value_kind = DecodedValueKind::INT32; + batch._consumed_level_count = static_cast(def_levels.size()); + batch._decoded_level_count = static_cast(def_levels.size()); + batch._values_written = static_cast(values.size()); + batch._def_levels = def_levels.data(); + batch._rep_levels = rep_levels.data(); + batch._fixed_values = reinterpret_cast(values.data()); + batch._read_dense_for_nullable = read_dense_for_nullable; + return batch; + } + + static Status build_nested_batch(const ParquetLeafReader& reader, + const ParquetLeafBatch& leaf_batch, int64_t records_read, + int16_t value_slot_definition_level, + int16_t value_slot_repetition_level, + ParquetNestedScalarBatch* nested_batch) { + return reader.build_nested_batch_from_leaf_batch(leaf_batch, records_read, + value_slot_definition_level, nested_batch, + value_slot_repetition_level); + } +}; + +std::shared_ptr<::parquet::ColumnDescriptor> int32_column_descriptor(int16_t max_definition_level, + int16_t max_repetition_level) { + auto node = ::parquet::schema::PrimitiveNode::Make("leaf", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32); + return std::make_shared<::parquet::ColumnDescriptor>(node, max_definition_level, + max_repetition_level); +} + +ParquetLeafReader make_nested_leaf_reader( + const std::shared_ptr<::parquet::ColumnDescriptor>& descriptor, DataTypePtr type) { + ParquetTypeDescriptor type_descriptor; + type_descriptor.physical_type = ::parquet::Type::INT32; + type_descriptor.doris_type = type; + return ParquetLeafReader(descriptor.get(), type_descriptor, std::move(type), "nested_leaf", + nullptr); +} + +TEST(ParquetLeafReaderTest, DenseNullableFixedValuesAreSpacedBeforeSerde) { + ParquetTypeDescriptor descriptor; + descriptor.physical_type = ::parquet::Type::INT32; + auto type = make_nullable(std::make_shared()); + auto reader = make_leaf_reader(descriptor, type); + + const std::vector compact_values = {10, 30, 50}; + ParquetLeafBatch batch; + batch._value_kind = DecodedValueKind::INT32; + batch._fixed_values = reinterpret_cast(compact_values.data()); + batch._values_written = compact_values.size(); + batch._read_dense_for_nullable = true; + + const NullMap null_map = {0, 1, 0, 1, 0}; + auto column = type->create_column(); + auto status = reader.append_values(batch, 5, &null_map, column); + ASSERT_TRUE(status.ok()) << status; + + const auto& nullable = assert_cast(*column); + ASSERT_EQ(nullable.size(), 5); + EXPECT_FALSE(nullable.is_null_at(0)); + EXPECT_TRUE(nullable.is_null_at(1)); + EXPECT_FALSE(nullable.is_null_at(2)); + EXPECT_TRUE(nullable.is_null_at(3)); + EXPECT_FALSE(nullable.is_null_at(4)); + const auto& nested = assert_cast(nullable.get_nested_column()); + EXPECT_EQ(nested.get_element(0), 10); + EXPECT_EQ(nested.get_element(2), 30); + EXPECT_EQ(nested.get_element(4), 50); +} + +TEST(ParquetLeafReaderTest, DenseNullableFixedValuesRejectCountMismatch) { + ParquetTypeDescriptor descriptor; + descriptor.physical_type = ::parquet::Type::INT32; + auto type = make_nullable(std::make_shared()); + auto reader = make_leaf_reader(descriptor, type); + + const std::vector compact_values = {10, 30}; + ParquetLeafBatch batch; + batch._value_kind = DecodedValueKind::INT32; + batch._fixed_values = reinterpret_cast(compact_values.data()); + batch._values_written = compact_values.size(); + batch._read_dense_for_nullable = true; + + const NullMap null_map = {0, 1, 0, 1, 0}; + auto column = type->create_column(); + auto status = reader.append_values(batch, 5, &null_map, column); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Invalid dense nullable parquet values"), std::string::npos); +} + +TEST(ParquetLeafReaderTest, Float16BinaryValuesAreConvertedToFloat) { + ParquetTypeDescriptor descriptor; + descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; + descriptor.fixed_length = 2; + auto type = std::make_shared(); + auto reader = make_leaf_reader(descriptor, type); + + auto half = [](uint16_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + ParquetLeafBatch batch; + batch._value_kind = DecodedValueKind::FIXED_BINARY; + batch._binary_chunks = {fixed_binary_array( + {half(0x0000), half(0x8000), half(0x3E00), half(0x0001), half(0x7E00)}, 2)}; + batch._values_written = 5; + + auto column = type->create_column(); + auto status = reader.append_values(batch, 5, nullptr, column); + ASSERT_TRUE(status.ok()) << status; + + const auto& floats = assert_cast(*column); + ASSERT_EQ(floats.size(), 5); + EXPECT_FLOAT_EQ(floats.get_element(0), 0.0F); + EXPECT_TRUE(std::signbit(floats.get_element(1))); + EXPECT_FLOAT_EQ(floats.get_element(2), 1.5F); + EXPECT_NEAR(floats.get_element(3), 5.9604645e-8F, 1e-12F); + EXPECT_TRUE(std::isnan(floats.get_element(4))); +} + +TEST(ParquetLeafReaderTest, BinaryDenseNullableValuesAreSpacedWithNullRefs) { + ParquetTypeDescriptor descriptor; + descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; + auto type = make_nullable(std::make_shared()); + auto reader = make_leaf_reader(descriptor, type); + + arrow::BinaryBuilder builder; + ASSERT_TRUE(builder.Append("aa").ok()); + ASSERT_TRUE(builder.Append("cc").ok()); + ASSERT_TRUE(builder.Append("ee").ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + ParquetLeafBatch batch; + batch._value_kind = DecodedValueKind::BINARY; + batch._binary_chunks = {array}; + batch._values_written = 3; + batch._read_dense_for_nullable = true; + + const NullMap null_map = {0, 1, 0, 1, 0}; + auto column = type->create_column(); + auto status = reader.append_values(batch, 5, &null_map, column); + ASSERT_TRUE(status.ok()) << status; + + const auto& nullable = assert_cast(*column); + const auto& strings = assert_cast(nullable.get_nested_column()); + ASSERT_EQ(nullable.size(), 5); + EXPECT_EQ(strings.get_data_at(0).to_string(), "aa"); + EXPECT_TRUE(nullable.is_null_at(1)); + EXPECT_EQ(strings.get_data_at(2).to_string(), "cc"); + EXPECT_TRUE(nullable.is_null_at(3)); + EXPECT_EQ(strings.get_data_at(4).to_string(), "ee"); +} + +TEST(ParquetLeafReaderTest, BinaryDenseNullableRejectsCountMismatch) { + ParquetTypeDescriptor descriptor; + descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; + auto type = make_nullable(std::make_shared()); + auto reader = make_leaf_reader(descriptor, type); + + arrow::BinaryBuilder builder; + ASSERT_TRUE(builder.Append("only_one").ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + ParquetLeafBatch batch; + batch._value_kind = DecodedValueKind::BINARY; + batch._binary_chunks = {array}; + batch._values_written = 1; + batch._read_dense_for_nullable = true; + + const NullMap null_map = {0, 1, 0}; + auto column = type->create_column(); + auto status = reader.append_values(batch, 3, &null_map, column); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Invalid dense nullable parquet binary values"), + std::string::npos); +} + +TEST(ParquetLeafReaderTest, DecodedColumnViewCarriesDescriptorSessionAndNullMapFields) { + ParquetTypeDescriptor descriptor; + descriptor.physical_type = ::parquet::Type::INT64; + descriptor.time_unit = ParquetTimeUnit::NANOS; + descriptor.decimal_precision = 18; + descriptor.decimal_scale = 4; + descriptor.fixed_length = 12; + descriptor.timestamp_is_adjusted_to_utc = true; + auto type = make_nullable(std::make_shared()); + cctz::time_zone shanghai; + ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); + + CapturedDecodedView captured; + auto reader = make_spy_leaf_reader(descriptor, type, &captured, &shanghai, true); + const std::vector values = {100, 200, 300}; + ParquetLeafBatch batch; + batch._value_kind = DecodedValueKind::INT64; + batch._fixed_values = reinterpret_cast(values.data()); + batch._values_written = values.size(); + + const NullMap null_map = {0, 1, 0}; + auto column = type->create_column(); + ASSERT_TRUE(reader.append_values(batch, 3, &null_map, column).ok()); + EXPECT_EQ(captured.value_kind, DecodedValueKind::INT64); + EXPECT_EQ(captured.time_unit, DecodedTimeUnit::NANOS); + EXPECT_EQ(captured.row_count, 3); + EXPECT_EQ(captured.decimal_precision, 18); + EXPECT_EQ(captured.decimal_scale, 4); + EXPECT_EQ(captured.fixed_length, 12); + EXPECT_TRUE(captured.timestamp_is_adjusted_to_utc); + EXPECT_TRUE(captured.enable_strict_mode); + EXPECT_EQ(captured.timezone, &shanghai); + EXPECT_FALSE(captured.null_map_is_null); + EXPECT_EQ(captured.null_map, std::vector({0, 1, 0})); + + auto required_column = type->create_column(); + ASSERT_TRUE(reader.append_values(batch, 3, nullptr, required_column).ok()); + EXPECT_TRUE(captured.null_map_is_null); + + const NullMap empty_null_map; + ASSERT_TRUE(reader.append_values(batch, 3, &empty_null_map, required_column).ok()); + EXPECT_TRUE(captured.null_map_is_null); +} + +TEST(ParquetLeafReaderTest, DecodedColumnViewCapturesBinaryFixedLengthAndFloat16Override) { + ParquetTypeDescriptor binary_descriptor; + binary_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + binary_descriptor.fixed_length = 4; + auto type = std::make_shared(); + + CapturedDecodedView binary_view; + auto binary_reader = make_spy_leaf_reader(binary_descriptor, type, &binary_view); + ParquetLeafBatch binary_batch; + binary_batch._value_kind = DecodedValueKind::FIXED_BINARY; + binary_batch._binary_chunks = {fixed_binary_array({"abcd", "wxyz"}, 4)}; + binary_batch._values_written = 2; + auto binary_column = type->create_column(); + ASSERT_TRUE(binary_reader.append_values(binary_batch, 2, nullptr, binary_column).ok()); + EXPECT_EQ(binary_view.value_kind, DecodedValueKind::FIXED_BINARY); + EXPECT_EQ(binary_view.fixed_length, 4); + ASSERT_EQ(binary_view.owned_binary_values.size(), 2); + EXPECT_EQ(binary_view.owned_binary_values[0], "abcd"); + EXPECT_EQ(binary_view.owned_binary_values[1], "wxyz"); + + ParquetTypeDescriptor float16_descriptor; + float16_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + float16_descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; + float16_descriptor.fixed_length = 2; + CapturedDecodedView float16_view; + auto float16_reader = make_spy_leaf_reader(float16_descriptor, + std::make_shared(), &float16_view); + auto half = [](uint16_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + ParquetLeafBatch float16_batch; + float16_batch._value_kind = DecodedValueKind::FIXED_BINARY; + float16_batch._binary_chunks = {fixed_binary_array({half(0x3E00), half(0x4000)}, 2)}; + float16_batch._values_written = 2; + auto float16_column = std::make_shared()->create_column(); + ASSERT_TRUE(float16_reader.append_values(float16_batch, 2, nullptr, float16_column).ok()); + EXPECT_EQ(float16_view.value_kind, DecodedValueKind::FLOAT); + ASSERT_EQ(float16_view.fixed_values.size(), sizeof(float) * 2); + const auto* floats = reinterpret_cast(float16_view.fixed_values.data()); + EXPECT_FLOAT_EQ(floats[0], 1.5F); + EXPECT_FLOAT_EQ(floats[1], 2.0F); +} + +TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLevels) { + auto descriptor = int32_column_descriptor(2, 1); + auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); + const std::vector def_levels = {2, 2, 2}; + const std::vector rep_levels = {0, 1, 0}; + const std::vector values = {10, 20, 30}; + const auto leaf_batch = + ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); + + ParquetNestedScalarBatch nested_batch; + auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 2, 2, 1, + &nested_batch); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(nested_batch.records_read, 2); + EXPECT_EQ(nested_batch.levels_written, 3); + EXPECT_EQ(nested_batch.value_indices, std::vector({0, 1, 2})); + const auto& nested_values = assert_cast(*nested_batch.values_column); + ASSERT_EQ(nested_values.size(), 3); + EXPECT_EQ(nested_values.get_element(0), 10); + EXPECT_EQ(nested_values.get_element(2), 30); +} + +TEST(ParquetLeafReaderTest, NestedBatchValueLayoutValueSlots) { + auto descriptor = int32_column_descriptor(2, 1); + auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); + const std::vector def_levels = {2, 1, 2, 0}; + const std::vector rep_levels = {0, 1, 0, 0}; + const std::vector values = {10, 777, 30}; + const auto leaf_batch = + ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); + + ParquetNestedScalarBatch nested_batch; + auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, + &nested_batch); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 2, -1})); +} + +TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLeafValues) { + auto descriptor = int32_column_descriptor(2, 1); + auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); + const std::vector def_levels = {2, 1, 2, 0}; + const std::vector rep_levels = {0, 1, 0, 0}; + const std::vector values = {10, 30}; + const auto leaf_batch = + ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); + + ParquetNestedScalarBatch nested_batch; + auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, + &nested_batch); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 1, -1})); +} + +TEST(ParquetLeafReaderTest, NestedBatchValueLayoutPayloadSlots) { + auto descriptor = int32_column_descriptor(2, 1); + auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); + const std::vector def_levels = {1, 2, 0, 2}; + const std::vector rep_levels = {0, 0, 0, 0}; + const std::vector values = {777, 10, 30}; + const auto leaf_batch = + ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); + + ParquetNestedScalarBatch nested_batch; + auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, + &nested_batch); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(nested_batch.value_indices, std::vector({-1, 1, -1, 2})); +} + +TEST(ParquetLeafReaderTest, NestedBatchRejectsMismatchedValueLayout) { + auto descriptor = int32_column_descriptor(2, 1); + auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); + const std::vector def_levels = {2, 0, 2, 0}; + const std::vector rep_levels = {0, 0, 0, 0}; + const std::vector values = {10, 20, 30}; + const auto leaf_batch = + ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); + + ParquetNestedScalarBatch nested_batch; + const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, + &nested_batch); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("inconsistent value count"), std::string::npos); +} + +TEST(ParquetLeafReaderTest, NestedBatchRejectsDenseNullable) { + auto descriptor = int32_column_descriptor(1, 0); + auto reader = + make_nested_leaf_reader(descriptor, make_nullable(std::make_shared())); + const std::vector def_levels = {1}; + const std::vector rep_levels = {0}; + const std::vector values = {10}; + const auto leaf_batch = + ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values, true); + + ParquetNestedScalarBatch nested_batch; + const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 1, 0, 0, + &nested_batch); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Dense nullable parquet nested reader is not supported"), + std::string::npos); +} + +} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp new file mode 100644 index 00000000000000..fac89b31c422d8 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "format_v2/parquet/parquet_file_context.h" +#include "io/fs/buffered_reader.h" + +namespace doris::format::parquet { +namespace { + +void expect_plan_entry(const ParquetPageCacheReadPlanEntry& entry, + const ParquetPageCacheRange& cached_range, int64_t copy_offset_in_cache, + int64_t output_offset, int64_t copy_size) { + EXPECT_EQ(entry.cached_range.offset, cached_range.offset); + EXPECT_EQ(entry.cached_range.size, cached_range.size); + EXPECT_EQ(entry.copy_offset_in_cache, copy_offset_in_cache); + EXPECT_EQ(entry.output_offset, output_offset); + EXPECT_EQ(entry.copy_size, copy_size); +} + +TEST(ParquetPageCacheRangeTest, SubsetRequestHitsSingleCachedRange) { + const std::vector cached_ranges = { + {100, 100}, + }; + + // Request [120, 150) is fully inside cached [100, 200). The reader should lookup + // the exact cached key [100, 200), then copy from cached offset 20 into output offset 0. + auto plan = detail::plan_page_cache_range_read(120, 30, cached_ranges); + + ASSERT_EQ(plan.size(), 1); + expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); +} + +TEST(ParquetPageCacheRangeTest, SupersetRequestHitsMultipleAdjacentCachedRanges) { + const std::vector cached_ranges = { + {180, 80}, + {100, 80}, + }; + + // Request [100, 260) is larger than either cached entry, but the two cached ranges + // exactly cover it. The copy plan stitches the two exact cache entries together. + auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); + + ASSERT_EQ(plan.size(), 2); + expect_plan_entry(plan[0], {100, 80}, 0, 0, 80); + expect_plan_entry(plan[1], {180, 80}, 0, 80, 80); +} + +TEST(ParquetPageCacheRangeTest, SupersetRequestCanUseOverlappingCachedRanges) { + const std::vector cached_ranges = { + {150, 110}, + {100, 100}, + }; + + // Request [100, 260) is covered by overlapping cached ranges. The first copy uses + // [100, 200); the second resumes at cursor 200 and copies the tail from [150, 260). + auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); + + ASSERT_EQ(plan.size(), 2); + expect_plan_entry(plan[0], {100, 100}, 0, 0, 100); + expect_plan_entry(plan[1], {150, 110}, 50, 100, 60); +} + +TEST(ParquetPageCacheRangeTest, PartialOverlapWithoutFullCoverageMisses) { + const std::vector cached_ranges = { + {100, 80}, + {200, 60}, + }; + + // Cached ranges cover [100, 180) and [200, 260), but [180, 200) is missing. + // The caller must read the whole request from the file instead of returning + // a partially cached result. + auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); + + EXPECT_TRUE(plan.empty()); +} + +TEST(ParquetPageCacheRangeTest, NonCoveringAndInvalidRangesAreIgnored) { + const std::vector cached_ranges = { + {50, 20}, {100, 0}, {100, -1}, {180, 20}, {120, 30}, + }; + + // Only [120, 150) intersects the request, but it does not cover the request start + // [100, 120), so this is still a miss. + auto plan = detail::plan_page_cache_range_read(100, 50, cached_ranges); + + EXPECT_TRUE(plan.empty()); +} + +TEST(ParquetPageCacheRangeTest, InvalidRequestMisses) { + const std::vector cached_ranges = { + {100, 100}, + }; + + EXPECT_TRUE(detail::plan_page_cache_range_read(-1, 10, cached_ranges).empty()); + EXPECT_TRUE(detail::plan_page_cache_range_read(100, 0, cached_ranges).empty()); + EXPECT_TRUE(detail::plan_page_cache_range_read(100, -1, cached_ranges).empty()); +} + +TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) { + const std::vector ranges = { + {100, 50}, + {-1, 50}, + {200, 0}, + {300, -1}, + {std::numeric_limits::max() - 10, 20}, + {400, 60}, + }; + + const auto valid_ranges = detail::valid_prefetch_ranges(ranges); + + ASSERT_EQ(valid_ranges.size(), 2); + EXPECT_EQ(valid_ranges[0].offset, 100); + EXPECT_EQ(valid_ranges[0].size, 50); + EXPECT_EQ(valid_ranges[1].offset, 400); + EXPECT_EQ(valid_ranges[1].size, 60); +} + +TEST(ParquetPageCacheRangeTest, AveragePrefetchRangeSizeUsesOnlyValidRanges) { + const std::vector ranges = { + {0, 512}, + {512, 1536}, + {-1, 1024}, + {2048, 0}, + }; + + EXPECT_EQ(detail::average_prefetch_range_size(ranges), 1024); + EXPECT_EQ(detail::average_prefetch_range_size({{-1, 1024}, {0, 0}}), 0); +} + +TEST(ParquetPageCacheRangeTest, MergeRangeReaderDecisionMatchesV1SmallIoThreshold) { + const std::vector ranges = { + {0, 512 * 1024}, + {512 * 1024, 1024 * 1024}, + }; + + // Two sub-2MB column chunks are the intended merge-reader case: Arrow may issue many page-level + // ReadAt calls inside these chunks, so v2 should route them through MergeRangeFileReader. + EXPECT_TRUE(detail::should_use_merge_range_reader( + ranges, detail::average_prefetch_range_size(ranges), false)); + + // The v1 threshold is strict: a 2MB average chunk is no longer considered "small IO". + EXPECT_FALSE(detail::should_use_merge_range_reader(ranges, io::MergeRangeFileReader::SMALL_IO, + false)); +} + +TEST(ParquetPageCacheRangeTest, MergeRangeReaderDecisionRejectsEmptyInvalidAndInMemoryInputs) { + const std::vector invalid_ranges = { + {-1, 128}, + {0, 0}, + }; + const std::vector valid_ranges = { + {0, 128 * 1024}, + }; + + EXPECT_FALSE(detail::should_use_merge_range_reader({}, 0, false)); + EXPECT_FALSE(detail::should_use_merge_range_reader(invalid_ranges, 128, false)); + EXPECT_FALSE(detail::should_use_merge_range_reader( + valid_ranges, detail::average_prefetch_range_size(valid_ranges), true)); +} + +} // namespace +} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp new file mode 100644 index 00000000000000..a21974bced294d --- /dev/null +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -0,0 +1,1202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/global_rowid_column_reader.h" +#include "format_v2/parquet/reader/list_column_reader.h" +#include "format_v2/parquet/reader/map_column_reader.h" +#include "format_v2/parquet/reader/nested_column_materializer.h" +#include "format_v2/parquet/reader/row_position_column_reader.h" +#include "format_v2/parquet/reader/scalar_column_reader.h" +#include "format_v2/parquet/reader/struct_column_reader.h" +#include "format_v2/parquet/selection_vector.h" +#include "storage/utils.h" + +namespace doris::format::parquet { +namespace { + +ParquetColumnSchema int64_schema(std::string name = "mock") { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = std::move(name); + schema.type = std::make_shared(); + return schema; +} + +ParquetColumnSchema nested_int64_schema(std::string name, int16_t nullable_definition_level, + int16_t definition_level, int16_t repetition_level = 0, + int16_t repeated_ancestor_definition_level = 0) { + ParquetColumnSchema schema = int64_schema(std::move(name)); + schema.type = make_nullable(std::make_shared()); + schema.nullable_definition_level = nullable_definition_level; + schema.definition_level = definition_level; + schema.repetition_level = repetition_level; + schema.repeated_repetition_level = repetition_level; + schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; + return schema; +} + +ParquetColumnSchema nested_struct_schema() { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = "struct"; + schema.kind = ParquetColumnSchemaKind::STRUCT; + schema.nullable_definition_level = 1; + schema.definition_level = 2; + schema.type = make_nullable(std::make_shared( + DataTypes {make_nullable(std::make_shared()), + make_nullable(std::make_shared())}, + Strings {"a", "b"})); + return schema; +} + +ParquetColumnSchema nested_list_schema(std::string name, DataTypePtr element_type, + int16_t nullable_definition_level, int16_t definition_level, + int16_t repetition_level, + int16_t repeated_ancestor_definition_level) { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = std::move(name); + schema.kind = ParquetColumnSchemaKind::LIST; + schema.nullable_definition_level = nullable_definition_level; + schema.definition_level = definition_level; + schema.repetition_level = repetition_level; + schema.repeated_repetition_level = repetition_level; + schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; + schema.type = make_nullable(std::make_shared(std::move(element_type))); + return schema; +} + +ParquetColumnSchema nested_map_schema( + DataTypePtr value_type = make_nullable(std::make_shared())) { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = "map"; + schema.kind = ParquetColumnSchemaKind::MAP; + schema.nullable_definition_level = 1; + schema.definition_level = 2; + schema.repetition_level = 1; + schema.repeated_ancestor_definition_level = 2; + schema.type = make_nullable(std::make_shared( + make_nullable(std::make_shared()), std::move(value_type))); + return schema; +} + +ParquetColumnSchema bare_repeated_int64_list_schema() { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = "repeated"; + schema.kind = ParquetColumnSchemaKind::LIST; + schema.definition_level = 1; + schema.repetition_level = 1; + schema.repeated_repetition_level = 1; + schema.repeated_ancestor_definition_level = 1; + schema.type = std::make_shared(std::make_shared()); + return schema; +} + +std::unique_ptr primitive_child(int local_id, std::string name, + DataTypePtr type) { + auto child = std::make_unique(); + child->local_id = local_id; + child->name = std::move(name); + child->kind = ParquetColumnSchemaKind::PRIMITIVE; + child->leaf_column_id = local_id; + child->type = std::move(type); + child->type_descriptor.physical_type = ::parquet::Type::INT32; + child->type_descriptor.doris_type = child->type; + return child; +} + +ParquetColumnSchema struct_schema_for_projection() { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = "s"; + schema.kind = ParquetColumnSchemaKind::STRUCT; + schema.children.push_back(primitive_child(0, "a", std::make_shared())); + schema.children.push_back(primitive_child(1, "b", std::make_shared())); + DataTypes types = {make_nullable(schema.children[0]->type), + make_nullable(schema.children[1]->type)}; + Strings names = {"a", "b"}; + schema.type = std::make_shared(types, names); + return schema; +} + +ParquetColumnSchema list_schema_for_projection() { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = "xs"; + schema.kind = ParquetColumnSchemaKind::LIST; + schema.children.push_back(primitive_child(0, "element", std::make_shared())); + schema.type = std::make_shared(schema.children[0]->type); + return schema; +} + +ParquetColumnSchema map_schema_for_projection() { + ParquetColumnSchema schema; + schema.local_id = 0; + schema.name = "m"; + schema.kind = ParquetColumnSchemaKind::MAP; + schema.children.push_back(primitive_child(0, "key", std::make_shared())); + schema.children.push_back(primitive_child(1, "value", std::make_shared())); + schema.type = std::make_shared(make_nullable(schema.children[0]->type), + make_nullable(schema.children[1]->type)); + return schema; +} + +class CursorColumnReader final : public ParquetColumnReader { +public: + CursorColumnReader() : ParquetColumnReader(int64_schema(), std::make_shared()) {} + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { + if (column.get() == nullptr || rows_read == nullptr) { + return Status::InvalidArgument("invalid mock read arguments"); + } + auto* values = assert_cast(column.get()); + for (int64_t row = 0; row < rows; ++row) { + values->insert_value(_cursor + row); + } + _read_lengths.push_back(rows); + _cursor += rows; + *rows_read = rows; + return Status::OK(); + } + + Status skip(int64_t rows) override { + _skip_lengths.push_back(rows); + _cursor += rows; + return Status::OK(); + } + + int64_t cursor() const { return _cursor; } + const std::vector& skip_lengths() const { return _skip_lengths; } + const std::vector& read_lengths() const { return _read_lengths; } + +private: + int64_t _cursor = 0; + std::vector _skip_lengths; + std::vector _read_lengths; +}; + +class ScriptedNestedReader final : public ParquetColumnReader { +public: + ScriptedNestedReader(ParquetColumnSchema schema, DataTypePtr type, + std::vector def_levels, std::vector rep_levels, + bool has_repeated_child = false, bool build_nulls = false) + : ParquetColumnReader(schema, std::move(type)), + _def_levels(std::move(def_levels)), + _rep_levels(std::move(rep_levels)), + _has_repeated_child(has_repeated_child), + _build_nulls(build_nulls) {} + + Status read(int64_t, MutableColumnPtr&, int64_t*) override { + return Status::NotSupported("unused"); + } + + Status load_nested_batch(int64_t rows) override { + _load_lengths.push_back(rows); + return Status::OK(); + } + + Status load_nested_levels_batch(int64_t rows) override { + _level_load_lengths.push_back(rows); + return Status::OK(); + } + + Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) override { + _build_lengths.push_back(length_upper_bound); + if (column.get() == nullptr || values_read == nullptr) { + return Status::InvalidArgument("invalid scripted nested build arguments"); + } + for (int64_t row = 0; row < length_upper_bound; ++row) { + insert_value(column, _next_value++, _build_nulls); + } + *values_read = length_upper_bound; + return Status::OK(); + } + + Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { + DORIS_CHECK(values_consumed != nullptr); + _consume_lengths.push_back(length_upper_bound); + set_nested_build_level_cursor(std::min(nested_build_level_cursor() + length_upper_bound, + static_cast(_def_levels.size()))); + *values_consumed = length_upper_bound; + return Status::OK(); + } + + const std::vector& nested_definition_levels() const override { return _def_levels; } + const std::vector& nested_repetition_levels() const override { return _rep_levels; } + int64_t nested_levels_written() const override { + return static_cast(_def_levels.size()); + } + bool is_or_has_repeated_child() const override { return _has_repeated_child; } + + const std::vector& build_lengths() const { return _build_lengths; } + const std::vector& consume_lengths() const { return _consume_lengths; } + const std::vector& level_load_lengths() const { return _level_load_lengths; } + +private: + static void insert_value(MutableColumnPtr& column, int64_t value, bool is_null) { + if (auto* nullable_column = check_and_get_column(*column); + nullable_column != nullptr) { + if (is_null) { + nullable_column->insert_default(); + return; + } + assert_cast(nullable_column->get_nested_column()).insert_value(value); + nullable_column->get_null_map_data().push_back(0); + return; + } + assert_cast(*column).insert_value(value); + } + + std::vector _def_levels; + std::vector _rep_levels; + bool _has_repeated_child = false; + bool _build_nulls = false; + int64_t _next_value = 0; + std::vector _load_lengths; + std::vector _level_load_lengths; + std::vector _build_lengths; + std::vector _consume_lengths; +}; + +class ChunkedNestedLeafReader final : public ParquetColumnReader { +public: + ChunkedNestedLeafReader() + : ParquetColumnReader(nested_int64_schema("element", 0, 1, 1, 1), + std::make_shared()) {} + + Status read(int64_t, MutableColumnPtr&, int64_t*) override { + return Status::NotSupported("unused"); + } + + Status load_nested_batch(int64_t rows) override { + _load_lengths.push_back(rows); + _def_levels.assign(static_cast(rows), 1); + _rep_levels.assign(static_cast(rows), 0); + return Status::OK(); + } + + Status load_nested_levels_batch(int64_t rows) override { + _level_load_lengths.push_back(rows); + _def_levels.assign(static_cast(rows), 1); + _rep_levels.assign(static_cast(rows), 0); + return Status::OK(); + } + + Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, + int64_t* values_read) override { + DORIS_CHECK(column.get() != nullptr); + DORIS_CHECK(values_read != nullptr); + _initial_column_sizes.push_back(column->size()); + _build_lengths.push_back(length_upper_bound); + if (auto* nullable = check_and_get_column(*column); nullable != nullptr) { + auto& values = assert_cast(nullable->get_nested_column()); + for (int64_t row = 0; row < length_upper_bound; ++row) { + values.insert_value(row); + nullable->get_null_map_data().push_back(0); + } + } else { + auto* values = assert_cast(column.get()); + for (int64_t row = 0; row < length_upper_bound; ++row) { + values->insert_value(row); + } + } + *values_read = length_upper_bound; + return Status::OK(); + } + + Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { + DORIS_CHECK(values_consumed != nullptr); + _consume_lengths.push_back(length_upper_bound); + *values_consumed = length_upper_bound; + return Status::OK(); + } + + const std::vector& nested_definition_levels() const override { return _def_levels; } + const std::vector& nested_repetition_levels() const override { return _rep_levels; } + int64_t nested_levels_written() const override { + return static_cast(_def_levels.size()); + } + bool is_or_has_repeated_child() const override { return true; } + + const std::vector& load_lengths() const { return _load_lengths; } + const std::vector& build_lengths() const { return _build_lengths; } + const std::vector& consume_lengths() const { return _consume_lengths; } + const std::vector& level_load_lengths() const { return _level_load_lengths; } + const std::vector& initial_column_sizes() const { return _initial_column_sizes; } + +private: + std::vector _def_levels; + std::vector _rep_levels; + std::vector _load_lengths; + std::vector _level_load_lengths; + std::vector _build_lengths; + std::vector _consume_lengths; + std::vector _initial_column_sizes; +}; + +} // namespace + +struct ScalarColumnReaderTestAccess { + static void set_nested_batch(ScalarColumnReader* reader, + std::unique_ptr batch) { + reader->_nested_batch = std::move(batch); + } + + static int64_t page_filtered_rows_to_skip(const ScalarColumnReader& reader, int64_t rows) { + return reader.page_filtered_rows_to_skip(rows); + } + + static void set_row_group_rows_read(ScalarColumnReader* reader, int64_t rows) { + reader->_row_group_rows_read = rows; + } +}; + +namespace { + +std::unique_ptr make_scripted_scalar_reader( + ParquetColumnSchema schema, std::unique_ptr batch) { + auto reader = std::make_unique(schema, nullptr); + ScalarColumnReaderTestAccess::set_nested_batch(reader.get(), std::move(batch)); + return reader; +} + +std::unique_ptr scalar_batch(std::vector def_levels, + std::vector rep_levels, + std::vector value_indices, + std::vector values) { + auto batch = std::make_unique(); + batch->levels_written = static_cast(def_levels.size()); + batch->def_levels = std::move(def_levels); + batch->rep_levels = std::move(rep_levels); + batch->value_indices = std::move(value_indices); + auto column = ColumnInt64::create(); + for (const auto value : values) { + column->insert_value(value); + } + batch->values_column = std::move(column); + return batch; +} + +class DefaultOnlyReader final : public ParquetColumnReader { +public: + DefaultOnlyReader() + : ParquetColumnReader(int64_schema("default_only"), std::make_shared()) { + } + + Status read(int64_t, MutableColumnPtr&, int64_t*) override { + return Status::NotSupported("unused"); + } +}; + +GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { + const auto ref = column.get_data_at(row); + EXPECT_EQ(ref.size, sizeof(GlobalRowLoacationV2)); + GlobalRowLoacationV2 location(0, 0, 0, 0); + std::memcpy(&location, ref.data, sizeof(GlobalRowLoacationV2)); + return location; +} + +} // namespace + +TEST(SelectionVectorTest, IdentitySelectionToRanges) { + SelectionVector selection; + const auto ranges = selection_to_ranges(selection, 5); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges[0].start, 0); + EXPECT_EQ(ranges[0].length, 5); + EXPECT_TRUE(selection.verify(5, 5).ok()); +} + +TEST(SelectionVectorTest, ExternalBufferSelectionToRanges) { + SelectionVector::Index indices[] = {0, 1, 4, 6, 7}; + SelectionVector selection(indices, std::size(indices)); + const auto ranges = selection_to_ranges(selection, std::size(indices)); + ASSERT_EQ(ranges.size(), 3); + EXPECT_EQ(ranges[0].start, 0); + EXPECT_EQ(ranges[0].length, 2); + EXPECT_EQ(ranges[1].start, 4); + EXPECT_EQ(ranges[1].length, 1); + EXPECT_EQ(ranges[2].start, 6); + EXPECT_EQ(ranges[2].length, 2); + EXPECT_TRUE(selection.verify(std::size(indices), 8).ok()); +} + +TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { + SelectionVector selection(2); + EXPECT_FALSE(selection.verify(3, 3).ok()); + EXPECT_FALSE(selection.verify(1, -1).ok()); + + selection.set_index(0, 2); + selection.set_index(1, 1); + EXPECT_FALSE(selection.verify(2, 3).ok()); + + selection.set_index(0, 0); + selection.set_index(1, 3); + EXPECT_FALSE(selection.verify(2, 3).ok()); +} + +TEST(ParquetColumnReaderControlTest, BaseSelectUsesSkipReadRanges) { + CursorColumnReader reader; + SelectionVector selection(3); + selection.set_index(0, 0); + selection.set_index(1, 2); + selection.set_index(2, 4); + + auto column = std::make_shared()->create_column(); + ASSERT_TRUE(reader.select(selection, 3, 6, column).ok()); + + const auto& values = assert_cast(*column); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(values.get_element(0), 0); + EXPECT_EQ(values.get_element(1), 2); + EXPECT_EQ(values.get_element(2), 4); + EXPECT_EQ(reader.cursor(), 6); + EXPECT_EQ(reader.read_lengths(), std::vector({1, 1, 1})); + EXPECT_EQ(reader.skip_lengths(), std::vector({0, 1, 1, 1})); +} + +TEST(ParquetColumnReaderControlTest, BaseSelectZeroRowsConsumesBatch) { + CursorColumnReader reader; + SelectionVector selection; + auto column = std::make_shared()->create_column(); + ASSERT_TRUE(reader.select(selection, 0, 4, column).ok()); + EXPECT_EQ(column->size(), 0); + EXPECT_EQ(reader.cursor(), 4); + EXPECT_TRUE(reader.read_lengths().empty()); + EXPECT_EQ(reader.skip_lengths(), std::vector({4})); +} + +TEST(ParquetColumnReaderControlTest, BaseNestedDefaultsAndSkipNested) { + DefaultOnlyReader base_reader; + EXPECT_FALSE(base_reader.skip(1).ok()); + EXPECT_FALSE(base_reader.load_nested_batch(1).ok()); + + auto column = std::make_shared()->create_column(); + int64_t values_read = 0; + EXPECT_FALSE(base_reader.build_nested_column(1, column, &values_read).ok()); + + int64_t values_consumed = 0; + EXPECT_FALSE(base_reader.consume_nested_column(1, &values_consumed).ok()); +} + +TEST(ParquetColumnReaderControlTest, NestedSkipConsumesBoundedBatchesWithoutMaterializing) { + auto element_reader = std::make_unique(); + auto* element_reader_ptr = element_reader.get(); + ListColumnReader reader(bare_repeated_int64_list_schema(), + bare_repeated_int64_list_schema().type, std::move(element_reader)); + + ASSERT_TRUE(reader.skip(8193).ok()); + EXPECT_TRUE(element_reader_ptr->load_lengths().empty()); + EXPECT_EQ(element_reader_ptr->level_load_lengths(), std::vector({4096, 4096, 1})); + EXPECT_EQ(element_reader_ptr->consume_lengths(), std::vector({4096, 4096, 1})); + EXPECT_TRUE(element_reader_ptr->build_lengths().empty()); + EXPECT_TRUE(element_reader_ptr->initial_column_sizes().empty()); +} + +TEST(ParquetColumnReaderControlTest, MapSkipConsumesBothStreamsWithoutMaterializing) { + const std::vector def_levels {3, 3, 1, 3}; + const std::vector rep_levels {0, 1, 0, 0}; + auto key_reader = std::make_unique( + nested_int64_schema("key", 2, 3, 1, 2), + make_nullable(std::make_shared()), def_levels, rep_levels); + auto* key_reader_ptr = key_reader.get(); + auto value_reader = std::make_unique( + nested_int64_schema("value", 2, 3, 1, 2), + make_nullable(std::make_shared()), def_levels, rep_levels); + auto* value_reader_ptr = value_reader.get(); + MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), + std::move(value_reader)); + + ASSERT_TRUE(reader.skip(3).ok()); + EXPECT_EQ(key_reader_ptr->level_load_lengths(), std::vector({3})); + EXPECT_EQ(value_reader_ptr->level_load_lengths(), std::vector({3})); + EXPECT_EQ(key_reader_ptr->consume_lengths(), std::vector({3})); + EXPECT_EQ(value_reader_ptr->consume_lengths(), std::vector({3})); + EXPECT_TRUE(key_reader_ptr->build_lengths().empty()); + EXPECT_TRUE(value_reader_ptr->build_lengths().empty()); +} + +TEST(ParquetColumnReaderControlTest, StructSkipConsumesNullSeparatedChildSpans) { + const std::vector def_levels {2, 0, 2}; + const std::vector rep_levels {0, 0, 0}; + auto shape_reader = std::make_unique( + nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), + def_levels, rep_levels); + auto* shape_reader_ptr = shape_reader.get(); + auto child_reader = std::make_unique( + nested_int64_schema("child", 1, 2), make_nullable(std::make_shared()), + def_levels, rep_levels); + auto* child_reader_ptr = child_reader.get(); + std::vector> children; + children.push_back(std::move(shape_reader)); + children.push_back(std::move(child_reader)); + StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, + std::move(children), {-1, 0}); + + ASSERT_TRUE(reader.skip(3).ok()); + EXPECT_EQ(shape_reader_ptr->level_load_lengths(), std::vector({3})); + EXPECT_EQ(child_reader_ptr->level_load_lengths(), std::vector({3})); + EXPECT_EQ(shape_reader_ptr->consume_lengths(), std::vector({1, 1})); + EXPECT_EQ(child_reader_ptr->consume_lengths(), std::vector({1, 1})); + EXPECT_TRUE(shape_reader_ptr->build_lengths().empty()); + EXPECT_TRUE(child_reader_ptr->build_lengths().empty()); +} + +TEST(ParquetColumnReaderControlTest, NestedListSkipConsumesRecursivelyWithoutMaterializing) { + auto leaf_reader = std::make_unique( + nested_int64_schema("leaf", 0, 1, 1, 1), std::make_shared(), + std::vector {1, 1}, std::vector {0, 0}); + auto* leaf_reader_ptr = leaf_reader.get(); + const auto inner_type = std::make_shared(std::make_shared()); + auto inner_reader = std::make_unique(bare_repeated_int64_list_schema(), + inner_type, std::move(leaf_reader)); + const auto outer_type = std::make_shared(inner_type); + ListColumnReader reader(bare_repeated_int64_list_schema(), outer_type, std::move(inner_reader)); + + ASSERT_TRUE(reader.skip(2).ok()); + EXPECT_EQ(leaf_reader_ptr->level_load_lengths(), std::vector({2})); + EXPECT_EQ(leaf_reader_ptr->consume_lengths(), std::vector({2})); + EXPECT_TRUE(leaf_reader_ptr->build_lengths().empty()); +} + +TEST(ParquetColumnReaderControlTest, NestedMaterializerHelpersAppendOffsetsAndParentNulls) { + ColumnArray::Offsets64 offsets; + append_offsets(offsets, {3, 0, 2}); + ASSERT_EQ(offsets.size(), 3); + EXPECT_EQ(offsets[0], 3); + EXPECT_EQ(offsets[1], 3); + EXPECT_EQ(offsets[2], 5); + append_offsets(offsets, {1, 4}); + ASSERT_EQ(offsets.size(), 5); + EXPECT_EQ(offsets[3], 6); + EXPECT_EQ(offsets[4], 10); + + const NullMap parent_nulls = {0, 1, 0}; + append_parent_nulls(nullptr, parent_nulls); + NullMap dst = {1}; + append_parent_nulls(&dst, parent_nulls); + EXPECT_EQ(dst, NullMap({1, 0, 1, 0})); +} + +TEST(ParquetColumnReaderControlTest, PageFilteredRowsToSkipUsesOnlyFullSkippedRanges) { + ParquetPageSkipPlan page_skip_plan; + page_skip_plan.skipped_ranges = {RowRange {0, 3}, RowRange {5, 2}, RowRange {10, 4}}; + + auto schema = nested_int64_schema("page_filtered", 0, 0); + ScalarColumnReader reader(schema, nullptr, &page_skip_plan); + EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 3), 3); + EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 3); + + ScalarColumnReaderTestAccess::set_row_group_rows_read(&reader, 5); + EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 2), 2); + EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 2); +} + +TEST(ParquetColumnReaderControlTest, StructSkipsNullParentForRepeatedChildAndBatchesPresentRows) { + auto repeated_child = std::make_unique( + nested_int64_schema("repeated_shape", 1, 2, 1), + make_nullable(std::make_shared()), std::vector {2, 2, 2, 2}, + std::vector {0, 0, 0, 0}, true); + auto* repeated_child_ptr = repeated_child.get(); + auto scalar_child = make_scripted_scalar_reader( + nested_int64_schema("scalar_child", 1, 2), + scalar_batch({2, 0, 2, 2}, {0, 0, 0, 0}, {0, -1, 1, 2}, {10, 20, 30})); + auto* scalar_child_ptr = scalar_child.get(); + + std::vector> children; + children.push_back(std::move(repeated_child)); + children.push_back(std::move(scalar_child)); + StructColumnReader reader(nested_struct_schema(), + make_nullable(std::make_shared( + DataTypes {make_nullable(std::make_shared()), + make_nullable(std::make_shared())}, + Strings {"a", "b"})), + std::move(children), {0, 1}); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + auto status = reader.build_nested_column(4, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows_read, 4); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 4); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_FALSE(nullable_column.is_null_at(3)); + EXPECT_EQ(repeated_child_ptr->build_lengths(), std::vector({1, 2})); + EXPECT_EQ(scalar_child_ptr->nested_build_level_cursor(), 4); +} + +TEST(ParquetColumnReaderControlTest, StructFallsBackToFirstChildWhenAllChildrenAreRepeated) { + auto first_child = std::make_unique( + nested_int64_schema("first", 1, 2, 1), make_nullable(std::make_shared()), + std::vector {2, 0}, std::vector {0, 0}, true); + auto second_child = std::make_unique( + nested_int64_schema("second", 1, 2, 1), + make_nullable(std::make_shared()), std::vector {2, 2}, + std::vector {0, 0}, true); + + std::vector> children; + children.push_back(std::move(first_child)); + children.push_back(std::move(second_child)); + StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, + std::move(children), {0, 1}); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + auto status = reader.build_nested_column(2, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(rows_read, 2); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); +} + +TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesComplexChildShapeOnly) { + auto shape_child = std::make_unique( + nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), + std::vector {2, 2, 0, 0, 2, 2}, std::vector {0, 0, 0, 0, 0, 0}); + + ParquetColumnSchema map_schema = nested_map_schema(); + map_schema.nullable_definition_level = 2; + map_schema.definition_level = 3; + map_schema.repeated_ancestor_definition_level = 0; + auto key_reader = std::make_unique( + nested_int64_schema("key", 3, 3, 1, 0), + make_nullable(std::make_shared()), + std::vector {3, 3, 0, 0, 3, 3}, std::vector {0, 0, 0, 0, 0, 0}); + auto value_reader = + make_scripted_scalar_reader(nested_int64_schema("value", 4, 4, 1, 0), + scalar_batch({4, 4, 0, 0, 4, 4}, {0, 0, 0, 0, 0, 0}, + {0, 1, -1, -1, 2, 3}, {10, 20, 30, 40})); + auto map_reader = std::make_unique( + map_schema, map_schema.type, std::move(key_reader), std::move(value_reader)); + + std::vector> children; + children.push_back(std::move(shape_child)); + children.push_back(std::move(map_reader)); + auto struct_type = make_nullable(std::make_shared(DataTypes {map_schema.type}, + Strings {"partitionValues"})); + StructColumnReader reader(nested_struct_schema(), struct_type, std::move(children), {-1, 0}); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + auto status = reader.build_nested_column(6, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows_read, 6); + + const auto& nullable_struct = assert_cast(*column); + ASSERT_EQ(nullable_struct.size(), 6); + EXPECT_FALSE(nullable_struct.is_null_at(0)); + EXPECT_FALSE(nullable_struct.is_null_at(1)); + EXPECT_TRUE(nullable_struct.is_null_at(2)); + EXPECT_TRUE(nullable_struct.is_null_at(3)); + EXPECT_FALSE(nullable_struct.is_null_at(4)); + EXPECT_FALSE(nullable_struct.is_null_at(5)); + + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& map_nullable = assert_cast(struct_column.get_column(0)); + ASSERT_EQ(map_nullable.size(), 6); + EXPECT_FALSE(map_nullable.is_null_at(0)); + EXPECT_FALSE(map_nullable.is_null_at(1)); + EXPECT_TRUE(map_nullable.is_null_at(2)); + EXPECT_TRUE(map_nullable.is_null_at(3)); + EXPECT_FALSE(map_nullable.is_null_at(4)); + EXPECT_FALSE(map_nullable.is_null_at(5)); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), 6); + EXPECT_EQ(map_column.get_offsets()[0], 1); + EXPECT_EQ(map_column.get_offsets()[1], 2); + EXPECT_EQ(map_column.get_offsets()[2], 2); + EXPECT_EQ(map_column.get_offsets()[3], 2); + EXPECT_EQ(map_column.get_offsets()[4], 3); + EXPECT_EQ(map_column.get_offsets()[5], 4); +} + +TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesNestedStructDescendants) { + auto shape_child = std::make_unique( + nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), + std::vector {2, 0, 2}, std::vector {0, 0, 0}); + + auto id_batch = scalar_batch({4, 3, 4}, {0, 0, 0}, {0, -1, 1}, {10, 20}); + id_batch->value_slot_definition_level = 3; + auto id_reader = + make_scripted_scalar_reader(nested_int64_schema("id", 3, 4), std::move(id_batch)); + + ParquetColumnSchema inner_schema; + inner_schema.local_id = 0; + inner_schema.name = "stats_parsed"; + inner_schema.kind = ParquetColumnSchemaKind::STRUCT; + inner_schema.nullable_definition_level = 2; + inner_schema.definition_level = 3; + inner_schema.type = make_nullable(std::make_shared( + DataTypes {make_nullable(std::make_shared())}, Strings {"id"})); + + std::vector> inner_children; + inner_children.push_back(std::move(id_reader)); + auto inner_reader = std::make_unique( + inner_schema, inner_schema.type, std::move(inner_children), std::vector {0}); + + std::vector> outer_children; + outer_children.push_back(std::move(shape_child)); + outer_children.push_back(std::move(inner_reader)); + auto outer_type = make_nullable(std::make_shared(DataTypes {inner_schema.type}, + Strings {"stats_parsed"})); + StructColumnReader reader(nested_struct_schema(), outer_type, std::move(outer_children), + {-1, 0}); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + auto status = reader.build_nested_column(3, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows_read, 3); + + const auto& outer_nullable = assert_cast(*column); + ASSERT_EQ(outer_nullable.size(), 3); + EXPECT_FALSE(outer_nullable.is_null_at(0)); + EXPECT_TRUE(outer_nullable.is_null_at(1)); + EXPECT_FALSE(outer_nullable.is_null_at(2)); + + const auto& outer_struct = assert_cast(outer_nullable.get_nested_column()); + const auto& inner_nullable = assert_cast(outer_struct.get_column(0)); + ASSERT_EQ(inner_nullable.size(), 3); + EXPECT_FALSE(inner_nullable.is_null_at(0)); + EXPECT_TRUE(inner_nullable.is_null_at(1)); + EXPECT_FALSE(inner_nullable.is_null_at(2)); + + const auto& inner_struct = assert_cast(inner_nullable.get_nested_column()); + const auto& id_nullable = assert_cast(inner_struct.get_column(0)); + const auto& id_values = assert_cast(id_nullable.get_nested_column()); + EXPECT_EQ(id_values.get_element(0), 10); + EXPECT_EQ(id_values.get_element(2), 20); +} + +TEST(ParquetColumnReaderControlTest, ListKeepsEmptyBareRepeatedPrimitiveRows) { + auto element_reader = std::make_unique( + nested_int64_schema("element", 0, 1, 1, 1), std::make_shared(), + std::vector {0, 1, 1, 0}, std::vector {0, 0, 1, 0}); + auto* element_reader_ptr = element_reader.get(); + ListColumnReader reader(bare_repeated_int64_list_schema(), + bare_repeated_int64_list_schema().type, std::move(element_reader)); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + auto status = reader.build_nested_column(3, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows_read, 3); + + const auto& array_column = assert_cast(*column); + ASSERT_EQ(array_column.get_offsets().size(), 3); + EXPECT_EQ(array_column.get_offsets()[0], 0); + EXPECT_EQ(array_column.get_offsets()[1], 2); + EXPECT_EQ(array_column.get_offsets()[2], 2); + EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({2})); +} + +TEST(ParquetColumnReaderControlTest, NestedListSkipsAncestorEmptyRowsButKeepsNullElements) { + auto element_reader = + std::make_unique(nested_int64_schema("element", 5, 5, 2, 4), + make_nullable(std::make_shared()), + std::vector {1, 5, 5, 5, 2, 5, 2, 0}, + std::vector {0, 0, 2, 1, 0, 1, 1, 0}); + auto* element_reader_ptr = element_reader.get(); + + const auto inner_type = make_nullable( + std::make_shared(make_nullable(std::make_shared()))); + auto inner_reader = std::make_unique( + nested_list_schema("inner", make_nullable(std::make_shared()), 3, 4, 2, + 2), + inner_type, std::move(element_reader)); + auto outer_type = make_nullable(std::make_shared(inner_type)); + ListColumnReader reader(nested_list_schema("outer", inner_type, 1, 2, 1, 2), outer_type, + std::move(inner_reader)); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + auto status = reader.build_nested_column(4, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows_read, 4); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 4); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_TRUE(nullable_column.is_null_at(3)); + + const auto& outer_array = assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), 4); + EXPECT_EQ(outer_offsets[0], 0); + EXPECT_EQ(outer_offsets[1], 2); + EXPECT_EQ(outer_offsets[2], 5); + EXPECT_EQ(outer_offsets[3], 5); + + const auto& inner_nullable = assert_cast(outer_array.get_data()); + ASSERT_EQ(inner_nullable.size(), 5); + EXPECT_FALSE(inner_nullable.is_null_at(0)); + EXPECT_FALSE(inner_nullable.is_null_at(1)); + EXPECT_TRUE(inner_nullable.is_null_at(2)); + EXPECT_FALSE(inner_nullable.is_null_at(3)); + EXPECT_TRUE(inner_nullable.is_null_at(4)); + + const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); + const auto& inner_offsets = inner_array.get_offsets(); + ASSERT_EQ(inner_offsets.size(), 5); + EXPECT_EQ(inner_offsets[0], 2); + EXPECT_EQ(inner_offsets[1], 3); + EXPECT_EQ(inner_offsets[2], 3); + EXPECT_EQ(inner_offsets[3], 4); + EXPECT_EQ(inner_offsets[4], 4); + EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({4})); +} + +TEST(ParquetColumnReaderControlTest, MapKeepsEmptyMapRows) { + auto key_reader = std::make_unique( + nested_int64_schema("key", 1, 2, 1, 2), + make_nullable(std::make_shared()), std::vector {1}, + std::vector {0}); + auto value_reader = std::make_unique( + nested_int64_schema("value", 2, 3, 1, 2), + make_nullable(std::make_shared()), std::vector {1}, + std::vector {0}); + auto* value_reader_ptr = value_reader.get(); + MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), + std::move(value_reader)); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + auto status = reader.build_nested_column(1, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows_read, 1); + + const auto& nullable_map = assert_cast(*column); + EXPECT_FALSE(nullable_map.is_null_at(0)); + const auto& map_column = assert_cast(nullable_map.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), 1); + EXPECT_EQ(map_column.get_offsets()[0], 0); + EXPECT_EQ(value_reader_ptr->build_lengths(), std::vector({0})); +} + +TEST(ParquetColumnReaderControlTest, ListMapSkipsAncestorEmptyRowsBeforeScalarValues) { + auto key_reader = std::make_unique( + nested_int64_schema("key", 4, 4, 2, 4), + make_nullable(std::make_shared()), std::vector {1, 4}, + std::vector {0, 0}); + auto value_reader = make_scripted_scalar_reader(nested_int64_schema("value", 5, 5, 2, 4), + scalar_batch({1, 5}, {0, 0}, {-1, 0}, {100})); + + const auto map_type = make_nullable( + std::make_shared(make_nullable(std::make_shared()), + make_nullable(std::make_shared()))); + auto map_reader = std::make_unique( + nested_map_schema(make_nullable(std::make_shared())), map_type, + std::move(key_reader), std::move(value_reader)); + auto outer_type = make_nullable(std::make_shared(map_type)); + ListColumnReader reader(nested_list_schema("outer", map_type, 1, 2, 1, 2), outer_type, + std::move(map_reader)); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + auto status = reader.build_nested_column(2, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows_read, 2); + + const auto& nullable_column = assert_cast(*column); + ASSERT_EQ(nullable_column.size(), 2); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_FALSE(nullable_column.is_null_at(1)); + + const auto& outer_array = assert_cast(nullable_column.get_nested_column()); + const auto& outer_offsets = outer_array.get_offsets(); + ASSERT_EQ(outer_offsets.size(), 2); + EXPECT_EQ(outer_offsets[0], 0); + EXPECT_EQ(outer_offsets[1], 1); + + const auto& map_nullable = assert_cast(outer_array.get_data()); + ASSERT_EQ(map_nullable.size(), 1); + EXPECT_FALSE(map_nullable.is_null_at(0)); + const auto& map_column = assert_cast(map_nullable.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), 1); + EXPECT_EQ(map_column.get_offsets()[0], 1); + + const auto& values = assert_cast(map_column.get_values()); + const auto& value_data = assert_cast(values.get_nested_column()); + ASSERT_EQ(values.size(), 1); + EXPECT_FALSE(values.is_null_at(0)); + EXPECT_EQ(value_data.get_element(0), 100); +} + +TEST(ParquetColumnReaderControlTest, MapRejectsNullKeysAndMisalignedScalarValueRepLevels) { + auto key_reader = std::make_unique( + nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), + std::vector {2}, std::vector {0}, false, true); + auto value_reader = std::make_unique( + nested_int64_schema("value", 1, 2, 1), make_nullable(std::make_shared()), + std::vector {2}, std::vector {0}); + MapColumnReader null_key_reader(nested_map_schema(), nested_map_schema().type, + std::move(key_reader), std::move(value_reader)); + auto column = null_key_reader.type()->create_column(); + int64_t rows_read = 0; + auto status = null_key_reader.build_nested_column(1, column, &rows_read); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains null key"), std::string::npos); + + auto aligned_key_reader = std::make_unique( + nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), + std::vector {2, 2}, std::vector {0, 1}); + auto misaligned_value_reader = + make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), + scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); + MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, + std::move(aligned_key_reader), + std::move(misaligned_value_reader)); + column = misaligned_reader.type()->create_column(); + status = misaligned_reader.build_nested_column(1, column, &rows_read); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); +} + +TEST(ParquetColumnReaderControlTest, MapConsumePreservesKeyAndValueCorruptionChecks) { + auto null_key_reader = + make_scripted_scalar_reader(nested_int64_schema("key", 2, 3, 1, 2), + scalar_batch({2}, {0}, {-1}, std::vector {})); + auto value_reader = std::make_unique( + nested_int64_schema("value", 2, 3, 1, 2), + make_nullable(std::make_shared()), std::vector {2}, + std::vector {0}); + MapColumnReader null_key_reader_map(nested_map_schema(), nested_map_schema().type, + std::move(null_key_reader), std::move(value_reader)); + int64_t values_consumed = 0; + auto status = null_key_reader_map.consume_nested_column(1, &values_consumed); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains null"), std::string::npos); + + auto key_reader = std::make_unique( + nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), + std::vector {2, 2}, std::vector {0, 1}); + auto misaligned_value_reader = + make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), + scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); + MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, + std::move(key_reader), std::move(misaligned_value_reader)); + status = misaligned_reader.consume_nested_column(1, &values_consumed); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); +} + +TEST(ParquetColumnReaderControlTest, MapBuildsScalarAndComplexValuePaths) { + auto key_reader = std::make_unique( + nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), + std::vector {2, 2}, std::vector {0, 1}); + auto scalar_value_reader = + make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), + scalar_batch({3, 3}, {0, 1}, {0, 1}, {100, 200})); + MapColumnReader scalar_reader(nested_map_schema(), nested_map_schema().type, + std::move(key_reader), std::move(scalar_value_reader)); + auto column = scalar_reader.type()->create_column(); + int64_t rows_read = 0; + auto status = scalar_reader.build_nested_column(1, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + const auto& nullable_map = assert_cast(*column); + const auto& map_column = assert_cast(nullable_map.get_nested_column()); + ASSERT_EQ(map_column.get_offsets().size(), 1); + EXPECT_EQ(map_column.get_offsets()[0], 2); + const auto& values = assert_cast(map_column.get_values()); + const auto& value_data = assert_cast(values.get_nested_column()); + ASSERT_EQ(values.size(), 2); + EXPECT_EQ(value_data.get_element(0), 100); + EXPECT_EQ(value_data.get_element(1), 200); + + auto complex_key_reader = std::make_unique( + nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), + std::vector {2, 2}, std::vector {0, 1}); + auto complex_value_reader = std::make_unique( + nested_int64_schema("complex_value", 2, 3, 1), + make_nullable(std::make_shared()), std::vector {3, 3}, + std::vector {0, 1}); + auto* complex_value_reader_ptr = complex_value_reader.get(); + MapColumnReader complex_reader(nested_map_schema(), nested_map_schema().type, + std::move(complex_key_reader), std::move(complex_value_reader)); + column = complex_reader.type()->create_column(); + status = complex_reader.build_nested_column(1, column, &rows_read); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(complex_value_reader_ptr->build_lengths(), std::vector({2})); +} + +TEST(ParquetVirtualColumnReaderTest, RowPositionReadSkipAndInvalidArgs) { + RowPositionColumnReader reader(100); + EXPECT_EQ(reader.file_column_id(), format::ROW_POSITION_COLUMN_ID); + EXPECT_EQ(reader.parquet_leaf_column_id(), -1); + EXPECT_EQ(reader.name(), format::ROW_POSITION_COLUMN_NAME); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + ASSERT_TRUE(reader.read(2, column, &rows_read).ok()); + ASSERT_EQ(rows_read, 2); + ASSERT_TRUE(reader.skip(3).ok()); + ASSERT_TRUE(reader.read(2, column, &rows_read).ok()); + + const auto& values = assert_cast(*column); + ASSERT_EQ(values.size(), 4); + EXPECT_EQ(values.get_element(0), 100); + EXPECT_EQ(values.get_element(1), 101); + EXPECT_EQ(values.get_element(2), 105); + EXPECT_EQ(values.get_element(3), 106); + + MutableColumnPtr null_column; + EXPECT_FALSE(reader.read(1, null_column, &rows_read).ok()); + EXPECT_FALSE(reader.read(-1, column, &rows_read).ok()); + EXPECT_FALSE(reader.read(1, column, nullptr).ok()); +} + +TEST(ParquetVirtualColumnReaderTest, GlobalRowIdReadSkipSelectAndInvalidArgs) { + format::GlobalRowIdContext context {.version = 7, .backend_id = 123456789, .file_id = 42}; + GlobalRowIdColumnReader reader(context, 10); + EXPECT_EQ(reader.file_column_id(), format::GLOBAL_ROWID_COLUMN_ID); + EXPECT_EQ(reader.parquet_leaf_column_id(), -1); + EXPECT_EQ(reader.name(), BeConsts::GLOBAL_ROWID_COL); + + auto column = reader.type()->create_column(); + int64_t rows_read = 0; + ASSERT_TRUE(reader.read(2, column, &rows_read).ok()); + ASSERT_TRUE(reader.skip(2).ok()); + ASSERT_TRUE(reader.read(1, column, &rows_read).ok()); + + const auto& strings = assert_cast(*column); + ASSERT_EQ(strings.size(), 3); + const auto first = decode_rowid(strings, 0); + EXPECT_EQ(first.version, context.version); + EXPECT_EQ(first.backend_id, context.backend_id); + EXPECT_EQ(first.file_id, context.file_id); + EXPECT_EQ(first.row_id, 10); + EXPECT_EQ(decode_rowid(strings, 1).row_id, 11); + EXPECT_EQ(decode_rowid(strings, 2).row_id, 14); + + GlobalRowIdColumnReader select_reader(context, 20); + SelectionVector selection(2); + selection.set_index(0, 1); + selection.set_index(1, 3); + auto selected_column = select_reader.type()->create_column(); + ASSERT_TRUE(select_reader.select(selection, 2, 5, selected_column).ok()); + const auto& selected_strings = assert_cast(*selected_column); + ASSERT_EQ(selected_strings.size(), 2); + EXPECT_EQ(decode_rowid(selected_strings, 0).row_id, 21); + EXPECT_EQ(decode_rowid(selected_strings, 1).row_id, 23); + + MutableColumnPtr null_column; + EXPECT_FALSE(reader.read(1, null_column, &rows_read).ok()); + EXPECT_FALSE(reader.read(-1, column, &rows_read).ok()); + EXPECT_FALSE(reader.read(1, column, nullptr).ok()); +} + +TEST(ParquetColumnReaderFactoryTest, RejectsInvalidLeafIdBeforeCreatingRecordReader) { + ParquetColumnSchema schema = int64_schema("bad_leaf"); + schema.kind = ParquetColumnSchemaKind::PRIMITIVE; + schema.leaf_column_id = 3; + schema.type_descriptor.physical_type = ::parquet::Type::INT64; + schema.type_descriptor.doris_type = schema.type; + + ParquetColumnReaderFactory factory(nullptr, 1); + std::unique_ptr reader; + const auto status = factory.create(schema, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); +} + +TEST(ParquetColumnReaderFactoryTest, RejectsStructInvalidAndEmptyProjection) { + auto schema = struct_schema_for_projection(); + ParquetColumnReaderFactory factory(nullptr, 0); + std::unique_ptr reader; + + auto invalid_projection = format::LocalColumnIndex::partial_local(0); + invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); + auto status = factory.create(schema, &invalid_projection, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); + + auto empty_projection = format::LocalColumnIndex::partial_local(0); + status = factory.create(schema, &empty_projection, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); +} + +TEST(ParquetColumnReaderFactoryTest, RejectsListProjectionWithoutElement) { + auto schema = list_schema_for_projection(); + ParquetColumnReaderFactory factory(nullptr, 0); + std::unique_ptr reader; + + auto projection = format::LocalColumnIndex::partial_local(0); + const auto status = factory.create(schema, &projection, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); +} + +TEST(ParquetColumnReaderFactoryTest, RejectsMapInvalidAndKeyOnlyProjection) { + auto schema = map_schema_for_projection(); + ParquetColumnReaderFactory factory(nullptr, 0); + std::unique_ptr reader; + + auto invalid_projection = format::LocalColumnIndex::partial_local(0); + invalid_projection.children.push_back(format::LocalColumnIndex::local(1)); + invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); + auto status = factory.create(schema, &invalid_projection, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); + + auto key_only_projection = format::LocalColumnIndex::partial_local(0); + key_only_projection.children.push_back(format::LocalColumnIndex::local(0)); + status = factory.create(schema, &key_only_projection, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains no value"), std::string::npos); +} + +} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp new file mode 100644 index 00000000000000..71d1cc291754ae --- /dev/null +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -0,0 +1,2906 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_reader.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" +#include "exprs/vcompound_pred.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "format_v2/column_mapper.h" +#include "format_v2/expr/delete_predicate.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_scan.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/Types_types.h" +#include "io/io_common.h" +#include "runtime/runtime_state.h" +#include "storage/index/zone_map/zonemap_eval_context.h" +#include "storage/index/zone_map/zonemap_filter_result.h" +#include "storage/segment/condition_cache.h" +#include "storage/utils.h" + +namespace doris { +namespace { + +constexpr int64_t ROW_COUNT = 5; + +format::LocalColumnIndex field_projection(int32_t column_id) { + return format::LocalColumnIndex {.index = column_id}; +} + +template +const ColumnType& nullable_nested_column(const Block& block, size_t position) { + const IColumn* column = block.get_by_position(position).column.get(); + int nullable_depth = 0; + while (const auto* nullable = check_and_get_column(*column)) { + const auto& null_map = nullable->get_null_map_data(); + for (size_t row = 0; row < null_map.size(); ++row) { + EXPECT_EQ(null_map[row], 0) << "Unexpected null at row " << row << ", column position " + << position << ", nullable depth " << nullable_depth; + } + column = &nullable->get_nested_column(); + ++nullable_depth; + } + EXPECT_GT(nullable_depth, 0) << "Expected a nullable file-local column at position " + << position; + return assert_cast(*column); +} + +class Int32GreaterThanExpr final : public VExpr { +public: + Int32GreaterThanExpr(int column_id, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& input = nullable_nested_column(*block, _column_id); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = input.get_element(input_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool can_evaluate_zonemap_filter() const override { return true; } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + auto zone_map = ctx.zone_map(_column_id); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + const auto literal = Field::create_field(_value); + return zone_map->max_value <= literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "Int32GreaterThanExpr"; +}; + +class Int32SumGreaterThanExpr final : public VExpr { +public: + Int32SumGreaterThanExpr(int left_column_id, int right_column_id, int32_t value) + : VExpr(std::make_shared(), false), + _left_column_id(left_column_id), + _right_column_id(right_column_id), + _value(value) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& left_input = nullable_nested_column(*block, _left_column_id); + const auto& right_input = nullable_nested_column(*block, _right_column_id); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + left_input.get_element(input_row) + right_input.get_element(input_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_left_column_id); + column_ids.insert(_right_column_id); + } + +private: + const int _left_column_id; + const int _right_column_id; + const int32_t _value; + const std::string _expr_name = "Int32SumGreaterThanExpr"; +}; + +class NonDeterministicCountingInt32Expr final : public VExpr { +public: + NonDeterministicCountingInt32Expr(int column_id, std::vector* executed_rows) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _executed_rows(executed_rows) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(_executed_rows != nullptr); + DORIS_CHECK(block != nullptr); + (void)nullable_nested_column(*block, _column_id); + _executed_rows->push_back(count); + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool is_deterministic() const override { return false; } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + std::vector* const _executed_rows; + const std::string _expr_name = "NonDeterministicCountingInt32Expr"; +}; + +class SelectedRowsUnsafeCountingInt32Expr final : public VExpr { +public: + SelectedRowsUnsafeCountingInt32Expr(int column_id, std::vector* executed_rows) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _executed_rows(executed_rows) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(_executed_rows != nullptr); + DORIS_CHECK(block != nullptr); + (void)nullable_nested_column(*block, _column_id); + _executed_rows->push_back(count); + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool is_safe_to_execute_on_selected_rows() const override { return false; } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + std::vector* const _executed_rows; + const std::string _expr_name = "SelectedRowsUnsafeCountingInt32Expr"; +}; + +class StringInExpr final : public VExpr { +public: + StringInExpr(int column_id, std::vector values) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _values(std::move(values)) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& input = nullable_nested_column(*block, _column_id); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + const auto value = input.get_data_at(input_row).to_string(); + result_data[row] = std::find(_values.begin(), _values.end(), value) != _values.end(); + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + const auto* dictionary = ctx.slot(_column_id); + if (dictionary == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + for (const auto& value : _values) { + const auto field = Field::create_field(value); + for (const auto& dictionary_value : dictionary->values) { + if (dictionary_value == field) { + return ZoneMapFilterResult::kMayMatch; + } + } + } + return ZoneMapFilterResult::kNoMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const std::vector _values; + const std::string _expr_name = "StringInExpr"; +}; + +class StringEqualsExpr final : public VExpr { +public: + StringEqualsExpr(int column_id, std::string row_value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _row_value(std::move(row_value)) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& input = nullable_nested_column(*block, _column_id); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = input.get_data_at(input_row).to_string() == _row_value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const std::string _row_value; + const std::string _expr_name = "StringEqualsExpr"; +}; + +class StringEqualsOrLengthEqualsExpr final : public VExpr { +public: + StringEqualsOrLengthEqualsExpr(int column_id, std::string row_value, size_t length) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _row_value(std::move(row_value)), + _length(length) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& input = nullable_nested_column(*block, _column_id); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + const auto value = input.get_data_at(input_row); + result_data[row] = value.to_string() == _row_value || value.size == _length; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const std::string _row_value; + const size_t _length; + const std::string _expr_name = "StringEqualsOrLengthEqualsExpr"; +}; + +VExprContextSPtr create_int32_greater_than_conjunct(int column_id, int32_t value) { + auto ctx = + VExprContext::create_shared(std::make_shared(column_id, value)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +VExprContextSPtr create_int32_sum_greater_than_conjunct(int left_column_id, int right_column_id, + int32_t value) { + auto ctx = VExprContext::create_shared( + std::make_shared(left_column_id, right_column_id, value)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +VExprContextSPtr create_non_deterministic_counting_int32_conjunct( + int column_id, std::vector* executed_rows) { + auto ctx = VExprContext::create_shared( + std::make_shared(column_id, executed_rows)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +VExprContextSPtr create_selected_rows_unsafe_counting_int32_conjunct( + int column_id, std::vector* executed_rows) { + auto ctx = VExprContext::create_shared( + std::make_shared(column_id, executed_rows)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +VExprContextSPtr create_string_in_conjunct(int column_id, std::vector values) { + auto ctx = VExprContext::create_shared( + std::make_shared(column_id, std::move(values))); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +TExprNode make_compound_node(TExprOpcode::type opcode, int num_children) { + TExprNode node; + node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); + node.__set_node_type(TExprNodeType::COMPOUND_PRED); + node.__set_opcode(opcode); + node.__set_num_children(num_children); + node.__set_is_nullable(false); + return node; +} + +VExprContextSPtr create_string_dictionary_and_residual_conjunct( + int column_id, std::vector dictionary_values, std::string row_value) { + auto compound = VCompoundPred::create_shared(make_compound_node(TExprOpcode::COMPOUND_AND, 2)); + compound->add_child(std::make_shared(column_id, std::move(dictionary_values))); + compound->add_child(std::make_shared(column_id, std::move(row_value))); + auto ctx = VExprContext::create_shared(std::move(compound)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +VExprContextSPtr create_nested_or_dictionary_and_residual_conjunct(int column_id) { + auto root = VCompoundPred::create_shared(make_compound_node(TExprOpcode::COMPOUND_AND, 2)); + root->add_child( + std::make_shared(column_id, std::vector {"az", "za"})); + root->add_child(std::make_shared(column_id, "az", 1)); + + auto ctx = VExprContext::create_shared(std::move(root)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { + std::shared_ptr array; + EXPECT_TRUE(builder->Finish(&array).ok()); + return array; +} + +std::shared_ptr build_int32_array(const std::vector& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_string_array(const std::vector& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_timestamp_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_struct_array(const std::vector& ids, + const std::vector& names) { + auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false), + arrow::field("name", arrow::utf8(), false)}); + std::vector> field_builders; + auto id_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(id_builder))); + auto name_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(name_builder))); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* struct_id_builder = assert_cast(builder.field_builder(0)); + auto* struct_name_builder = assert_cast(builder.field_builder(1)); + for (size_t row = 0; row < ids.size(); ++row) { + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_id_builder->Append(ids[row]).ok()); + EXPECT_TRUE(struct_name_builder->Append(names[row]).ok()); + } + return finish_array(&builder); +} + +void write_parquet_file(const std::string& file_path, int64_t row_group_size = ROW_COUNT) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), + }); + auto table = arrow::Table::Make(schema, + {build_int32_array({1, 2, 3, 4, 5}), + build_string_array({"one", "two", "three", "four", "five"})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + row_group_size, builder.build())); +} + +std::shared_ptr build_nullable_int_string_map_array() { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, map_type); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(10).ok()); + EXPECT_TRUE(value_builder->Append("small").ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(20).ok()); + EXPECT_TRUE(value_builder->Append(std::string(4096, 'x')).ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(30).ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + return finish_array(&builder); +} + +std::shared_ptr build_nullable_string_list_array() { + auto value_builder = std::make_shared(); + arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, + arrow::list(arrow::field("element", arrow::utf8(), true))); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->Append("small").ok()); + EXPECT_TRUE(value_builder->Append(std::string(4096, 'a')).ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(value_builder->Append(std::string(4096, 'b')).ok()); + return finish_array(&builder); +} + +std::shared_ptr build_nullable_string_struct_array() { + auto struct_type = arrow::struct_({arrow::field("payload", arrow::utf8(), true), + arrow::field("id", arrow::int32(), false)}); + std::vector> field_builders; + auto payload_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(payload_builder))); + auto id_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(id_builder))); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* struct_payload_builder = assert_cast(builder.field_builder(0)); + auto* struct_id_builder = assert_cast(builder.field_builder(1)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_payload_builder->Append("small").ok()); + EXPECT_TRUE(struct_id_builder->Append(1).ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_payload_builder->Append(std::string(4096, 'c')).ok()); + EXPECT_TRUE(struct_id_builder->Append(2).ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_payload_builder->AppendNull().ok()); + EXPECT_TRUE(struct_id_builder->Append(3).ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_payload_builder->Append(std::string(4096, 'd')).ok()); + EXPECT_TRUE(struct_id_builder->Append(4).ok()); + return finish_array(&builder); +} + +std::shared_ptr build_nullable_struct_with_list_array(bool list_first) { + auto list_type = arrow::list(arrow::field("element", arrow::int32(), false)); + auto scalar_field = arrow::field("scalar", arrow::int32(), false); + auto list_field = arrow::field("items", list_type, true); + auto struct_type = arrow::struct_(list_first ? arrow::FieldVector {list_field, scalar_field} + : arrow::FieldVector {scalar_field, list_field}); + + auto scalar_builder = std::make_shared(); + auto list_value_builder = std::make_shared(); + auto list_builder = std::make_shared(arrow::default_memory_pool(), + list_value_builder, list_type); + std::vector> field_builders = + list_first ? std::vector> {list_builder, + scalar_builder} + : std::vector> {scalar_builder, + list_builder}; + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(scalar_builder->Append(1).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(list_value_builder->Append(10).ok()); + EXPECT_TRUE(list_value_builder->Append(11).ok()); + + EXPECT_TRUE(builder.AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(scalar_builder->Append(2).ok()); + EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(scalar_builder->Append(3).ok()); + EXPECT_TRUE(list_builder->AppendNull().ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(scalar_builder->Append(4).ok()); + EXPECT_TRUE(list_builder->Append().ok()); + EXPECT_TRUE(list_value_builder->Append(20).ok()); + return finish_array(&builder); +} + +void write_nullable_map_parquet_file(const std::string& file_path) { + auto array = build_nullable_int_string_map_array(); + auto field = arrow::field("arr", array->type(), true); + auto table = arrow::Table::Make(arrow::schema({field}), {array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + +void write_nullable_string_list_parquet_file(const std::string& file_path) { + auto array = build_nullable_string_list_array(); + auto field = arrow::field("arr", array->type(), true); + auto table = arrow::Table::Make(arrow::schema({field}), {array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + +void write_nullable_string_struct_parquet_file(const std::string& file_path) { + auto array = build_nullable_string_struct_array(); + auto field = arrow::field("s", array->type(), true); + auto table = arrow::Table::Make(arrow::schema({field}), {array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + +void write_nullable_struct_with_list_parquet_file(const std::string& file_path) { + auto scalar_first = build_nullable_struct_with_list_array(false); + auto list_first = build_nullable_struct_with_list_array(true); + auto table = arrow::Table::Make( + arrow::schema({arrow::field("scalar_first", scalar_first->type(), true), + arrow::field("list_first", list_first->type(), true)}), + {scalar_first, list_first}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + +void write_int96_timestamp_parquet_file(const std::string& file_path) { + auto field = arrow::field("ts_tz", arrow::timestamp(arrow::TimeUnit::MICRO), true); + auto array = + build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), + {1735660800000000LL, 1735660800123456LL, 1735689600000000LL}); + auto table = arrow::Table::Make(arrow::schema({field}), {array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + ::parquet::ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_force_write_int96_timestamps(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, writer_builder.build(), + arrow_builder.build())); +} + +void write_int_pair_parquet_file(const std::string& file_path, int64_t row_group_size = ROW_COUNT) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("score", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), + }); + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5}), build_int32_array({1, 2, 3, 4, 5}), + build_string_array({"one", "two", "three", "four", "five"})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + row_group_size, builder.build())); +} + +void write_condition_cache_parquet_file(const std::string& file_path) { + constexpr int64_t row_count = ConditionCacheContext::GRANULE_SIZE * 2; + std::vector ids(row_count); + std::iota(ids.begin(), ids.end(), 0); + + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + row_count, builder.build())); +} + +void write_struct_filter_parquet_file(const std::string& file_path) { + auto id_field = arrow::field("id", arrow::int32(), false); + auto name_field = arrow::field("name", arrow::utf8(), false); + auto struct_type = arrow::struct_({id_field, name_field}); + auto schema = arrow::schema({ + arrow::field("s", struct_type, false), + }); + auto table = arrow::Table::Make( + schema, {build_struct_array({1, 2, 10, 11}, {"one", "two", "ten", "eleven"})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + +void write_dictionary_filter_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), + }); + auto table = + arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_string_array({"aa", "az", "lm", "lz", "za", "zz"})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.enable_dictionary("value"); + builder.disable_dictionary("id"); + builder.disable_statistics(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_single_row_group_dictionary_filter_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), + }); + auto table = + arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_string_array({"aa", "az", "lm", "lz", "za", "zz"})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.enable_dictionary("value"); + builder.disable_dictionary("id"); + builder.disable_statistics(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, + builder.build())); +} + +void write_dictionary_filter_with_trailing_column_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), + arrow::field("payload", arrow::int32(), false), + }); + auto table = + arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_string_array({"aa", "az", "lm", "lz", "za", "zz"}), + build_int32_array({10, 20, 30, 40, 50, 60})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary("id"); + builder.enable_dictionary("value"); + builder.disable_dictionary("payload"); + builder.disable_statistics(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, + builder.build())); +} + +void write_nested_dictionary_filter_parquet_file(const std::string& file_path) { + auto id_field = arrow::field("id", arrow::int32(), false); + auto name_field = arrow::field("name", arrow::utf8(), false); + auto struct_type = arrow::struct_({id_field, name_field}); + auto schema = arrow::schema({ + arrow::field("s", struct_type, false), + }); + auto table = arrow::Table::Make( + schema, {build_struct_array({1, 2, 3, 4, 5, 6}, {"aa", "az", "lm", "lz", "za", "zz"})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.enable_dictionary("s.name"); + builder.disable_dictionary("s.identifier.field_id"); + builder.disable_statistics(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_dictionary_edge_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), + }); + auto table = arrow::Table::Make( + schema, + {build_int32_array({1, 2, 3, 4, 5, 6, 7, 8}), + build_string_array({"", "same", "other", "long-value", "", "tail", "same", "last"})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.enable_dictionary("value"); + builder.disable_dictionary("id"); + builder.disable_statistics(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + +void write_nested_page_index_filter_parquet_file(const std::string& file_path) { + std::vector ids(128); + std::iota(ids.begin(), ids.end(), 0); + std::vector names; + names.reserve(ids.size()); + for (const auto id : ids) { + names.push_back("name-" + std::to_string(id)); + } + auto id_field = arrow::field("id", arrow::int32(), false); + auto name_field = arrow::field("name", arrow::utf8(), false); + auto struct_type = arrow::struct_({id_field, name_field}); + auto schema = arrow::schema({ + arrow::field("s", struct_type, false), + }); + auto table = arrow::Table::Make(schema, {build_struct_array(ids, names)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary(); + builder.enable_write_page_index(); + builder.write_batch_size(8); + builder.data_pagesize(10); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ids.size(), builder.build())); +} + +void write_page_index_filter_parquet_file(const std::string& file_path) { + std::vector ids(128); + std::iota(ids.begin(), ids.end(), 0); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary(); + builder.enable_write_page_index(); + builder.write_batch_size(8); + builder.data_pagesize(10); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ids.size(), builder.build())); +} + +void write_page_index_filter_pair_parquet_file(const std::string& file_path) { + std::vector ids(128); + std::iota(ids.begin(), ids.end(), 0); + std::vector payloads; + payloads.reserve(ids.size()); + for (const auto id : ids) { + payloads.push_back(id + 1000); + } + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("payload", arrow::int32(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(payloads)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary(); + builder.enable_write_page_index(); + builder.write_batch_size(8); + builder.data_pagesize(10); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ids.size(), builder.build())); +} + +Block build_file_block(const std::vector& schema) { + Block block; + for (const auto& field : schema) { + block.insert({field.type->create_column(), field.type, field.name}); + } + return block; +} + +Block build_file_block_with_row_position(const std::vector& schema) { + auto block = build_file_block(schema); + const auto row_position_field = format::row_position_column_definition(); + block.insert({row_position_field.type->create_column(), row_position_field.type, + row_position_field.name}); + return block; +} + +void use_schema_order_positions(format::FileScanRequest* request, + const std::vector& schema) { + DORIS_CHECK(request != nullptr); + for (size_t idx = 0; idx < schema.size(); ++idx) { + request->local_positions.emplace(format::LocalColumnId(schema[idx].local_id), + format::LocalIndex(idx)); + } +} + +int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { + return column_metadata.has_dictionary_page() + ? static_cast(column_metadata.dictionary_page_offset()) + : static_cast(column_metadata.data_page_offset()); +} + +std::pair row_group_mid_range(const std::string& file_path, int row_group_idx) { + auto reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); + auto metadata = reader->metadata(); + auto row_group_metadata = metadata->RowGroup(row_group_idx); + auto first_column = row_group_metadata->ColumnChunk(0); + auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); + const int64_t row_group_start_offset = parquet_column_start_offset(*first_column); + const int64_t row_group_end_offset = + parquet_column_start_offset(*last_column) + last_column->total_compressed_size(); + const int64_t row_group_mid_offset = + row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; + return {row_group_mid_offset, 1}; +} + +GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { + const auto ref = column.get_data_at(row); + EXPECT_EQ(ref.size, sizeof(GlobalRowLoacationV2)); + GlobalRowLoacationV2 location(0, 0, 0, 0); + std::memcpy(&location, ref.data, sizeof(GlobalRowLoacationV2)); + return location; +} + +class TestFileReader final : public format::FileReader { +public: + TestFileReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx) + : format::FileReader(system_properties, file_description, io_ctx, nullptr) {} + + Status get_schema(std::vector* file_schema) const override { + file_schema->clear(); + format::ColumnDefinition field; + field.identifier = Field::create_field(0); + field.name = "id"; + field.type = std::make_shared(); + file_schema->push_back(std::move(field)); + return Status::OK(); + } + + bool has_request() const { return _request != nullptr; } + + bool eof() const { return _eof; } + + bool has_io_context() const { return _io_ctx != nullptr; } + + long io_context_use_count() const { return _io_ctx.use_count(); } +}; + +TEST(FileReaderTest, OpenStoresRequestAndCloseKeepsRequest) { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto file_description = std::make_unique(); + auto io_ctx = std::make_shared(); + TestFileReader reader(system_properties, file_description, io_ctx); + + auto request = std::make_shared(); + request->non_predicate_columns.push_back(field_projection(0)); + ASSERT_TRUE(reader.open(request).ok()); + EXPECT_NE(request, nullptr); + EXPECT_TRUE(reader.has_request()); + + ASSERT_TRUE(reader.close().ok()); + EXPECT_TRUE(reader.has_request()); + EXPECT_TRUE(reader.eof()); +} + +TEST(FileReaderTest, CloseReleasesSharedIOContext) { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto file_description = std::make_unique(); + auto io_ctx = std::make_shared(); + std::weak_ptr weak_io_ctx = io_ctx; + TestFileReader reader(system_properties, file_description, io_ctx); + + EXPECT_TRUE(reader.has_io_context()); + EXPECT_EQ(reader.io_context_use_count(), 2); + io_ctx.reset(); + EXPECT_FALSE(weak_io_ctx.expired()); + EXPECT_EQ(reader.io_context_use_count(), 1); + + ASSERT_TRUE(reader.close().ok()); + EXPECT_FALSE(reader.has_io_context()); + EXPECT_TRUE(weak_io_ctx.expired()); +} + +class NewParquetReaderTest : public testing::Test { +protected: + void SetUp() override { + _test_dir = std::filesystem::temp_directory_path() / "doris_format_v2_parquet_reader_test"; + std::filesystem::remove_all(_test_dir); + std::filesystem::create_directories(_test_dir); + _file_path = (_test_dir / "reader.parquet").string(); + write_parquet_file(_file_path); + } + + void TearDown() override { std::filesystem::remove_all(_test_dir); } + + std::unique_ptr create_reader( + int64_t range_start_offset = 0, int64_t range_size = -1, + RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, + std::shared_ptr io_ctx = nullptr, + std::optional global_rowid_context = std::nullopt) const { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto file_description = std::make_unique(); + file_description->path = _file_path; + file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); + file_description->range_start_offset = range_start_offset; + file_description->range_size = range_size; + return std::make_unique( + system_properties, file_description, std::move(io_ctx), profile, + global_rowid_context, enable_mapping_timestamp_tz); + } + + std::filesystem::path _test_dir; + std::string _file_path; +}; + +TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(schema[0].local_id, 0); + EXPECT_EQ(schema[0].name, "id"); + ASSERT_TRUE(schema[0].type->is_nullable()); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_INT); + EXPECT_EQ(schema[1].local_id, 1); + EXPECT_EQ(schema[1].name, "value"); + ASSERT_TRUE(schema[1].type->is_nullable()); + EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); +} + +// Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and +// file-layer pruning hints. The reader declares those scan-request capabilities by choosing +// ParquetColumnMapper itself. +TEST_F(NewParquetReaderTest, CreatesParquetColumnMapper) { + auto reader = create_reader(); + auto mapper = + reader->create_column_mapper({.mode = format::TableColumnMappingMode::BY_FIELD_ID}); + + ASSERT_NE(dynamic_cast(mapper.get()), nullptr); +} + +TEST_F(NewParquetReaderTest, CountComplexColumnUsesShapeOnlyPath) { + write_nullable_map_parquet_file(_file_path); + RuntimeProfile profile("count_map_shape_only_path"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + ASSERT_TRUE(reader->open(std::make_shared()).ok()); + + format::FileAggregateRequest request; + request.agg_type = TPushAggOp::type::COUNT; + request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))}); + format::FileAggregateResult result; + ASSERT_TRUE(reader->get_aggregate_result(request, &result).ok()); + + // Rows are: non-empty map, NULL map, empty map, non-empty map with large value string, + // non-empty map with NULL value. COUNT(arr) excludes only the top-level NULL map. + EXPECT_EQ(result.count, 4); + ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + EXPECT_EQ(profile.get_counter("MaterializationTime")->value(), 0); +} + +TEST_F(NewParquetReaderTest, CountArrayColumnUsesLevelsOnlyPath) { + write_nullable_string_list_parquet_file(_file_path); + RuntimeProfile profile("count_array_levels_only_path"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + ASSERT_TRUE(reader->open(std::make_shared()).ok()); + + format::FileAggregateRequest request; + request.agg_type = TPushAggOp::type::COUNT; + request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))}); + format::FileAggregateResult result; + ASSERT_TRUE(reader->get_aggregate_result(request, &result).ok()); + + // Rows are: non-empty array with a large string, NULL array, empty array, non-empty array + // with NULL element, non-empty array with a large string. Only the top-level NULL is excluded. + EXPECT_EQ(result.count, 4); + ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + EXPECT_EQ(profile.get_counter("MaterializationTime")->value(), 0); +} + +TEST_F(NewParquetReaderTest, CountStructColumnUsesLevelsOnlyPath) { + write_nullable_string_struct_parquet_file(_file_path); + RuntimeProfile profile("count_struct_levels_only_path"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + ASSERT_TRUE(reader->open(std::make_shared()).ok()); + + format::FileAggregateRequest request; + request.agg_type = TPushAggOp::type::COUNT; + request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))}); + format::FileAggregateResult result; + ASSERT_TRUE(reader->get_aggregate_result(request, &result).ok()); + + // The representative STRUCT leaf is the first child, a nullable STRING payload. A row with + // NULL payload but non-NULL struct still counts; only the top-level NULL struct is excluded. + EXPECT_EQ(result.count, 4); + ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + EXPECT_EQ(profile.get_counter("MaterializationTime")->value(), 0); +} + +TEST_F(NewParquetReaderTest, CountStructWithRepeatedChildUsesTopLevelRowBoundaries) { + write_nullable_struct_with_list_parquet_file(_file_path); + + for (int32_t column_id = 0; column_id < 2; ++column_id) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + ASSERT_TRUE(reader->open(std::make_shared()).ok()); + + format::FileAggregateRequest request; + request.agg_type = TPushAggOp::type::COUNT; + request.columns.push_back({.projection = format::LocalColumnIndex::top_level( + format::LocalColumnId(column_id))}); + format::FileAggregateResult result; + ASSERT_TRUE(reader->get_aggregate_result(request, &result).ok()); + + // Rows are: non-empty ARRAY, NULL STRUCT, empty ARRAY, NULL ARRAY, non-empty ARRAY. + // COUNT(struct) excludes only the NULL STRUCT regardless of child field order. + EXPECT_EQ(result.count, 4); + } +} + +TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { + write_struct_filter_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + EXPECT_EQ(schema[0].name, "s"); + ASSERT_TRUE(schema[0].type->is_nullable()); + ASSERT_EQ(schema[0].children.size(), 2); + EXPECT_EQ(schema[0].children[0].name, "id"); + ASSERT_TRUE(schema[0].children[0].type->is_nullable()); + EXPECT_EQ(remove_nullable(schema[0].children[0].type)->get_primitive_type(), TYPE_INT); + EXPECT_EQ(schema[0].children[1].name, "name"); + ASSERT_TRUE(schema[0].children[1].type->is_nullable()); + EXPECT_EQ(remove_nullable(schema[0].children[1].type)->get_primitive_type(), TYPE_STRING); + + const auto* struct_type = + assert_cast(remove_nullable(schema[0].type).get()); + ASSERT_EQ(struct_type->get_elements().size(), 2); + EXPECT_TRUE(struct_type->get_element(0)->is_nullable()); + EXPECT_TRUE(struct_type->get_element(1)->is_nullable()); +} + +TEST_F(NewParquetReaderTest, GetSchemaMapsInt96ToTimestampTzWhenTimestampTzMappingEnabled) { + write_int96_timestamp_parquet_file(_file_path); + auto reader = create_reader(0, -1, nullptr, true); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + EXPECT_EQ(schema[0].name, "ts_tz"); + ASSERT_TRUE(schema[0].type->is_nullable()); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_TIMESTAMPTZ); + EXPECT_EQ(remove_nullable(schema[0].type)->get_scale(), 6); +} + +TEST_F(NewParquetReaderTest, ReadSingleRowGroupThenEof) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, ROW_COUNT); + + const auto& ids = nullable_nested_column(block, 0); + const auto& values = nullable_nested_column(block, 1); + ASSERT_EQ(ids.size(), ROW_COUNT); + ASSERT_EQ(values.size(), ROW_COUNT); + EXPECT_EQ(ids.get_element(0), 1); + EXPECT_EQ(ids.get_element(4), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "one"); + EXPECT_EQ(values.get_data_at(4).to_string(), "five"); + + rows = 0; + eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); +} + +TEST_F(NewParquetReaderTest, RespectsConfiguredBatchSize) { + auto reader = create_reader(); + reader->set_batch_size(1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + for (int32_t expected_id = 1; expected_id <= ROW_COUNT; ++expected_id) { + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 1); + const auto& ids = nullable_nested_column(block, 0); + ASSERT_EQ(ids.size(), 1); + EXPECT_EQ(ids.get_element(0), expected_id); + } + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); +} + +TEST_F(NewParquetReaderTest, ConditionCacheMissMarksSurvivingGranules) { + write_condition_cache_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + create_int32_greater_than_conjunct(0, ConditionCacheContext::GRANULE_SIZE - 1)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + auto ctx = std::make_shared(); + ctx->is_hit = false; + ctx->filter_result = std::make_shared>(3, false); + reader->set_condition_cache_context(ctx); + + std::vector ids; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + } + } + + ASSERT_EQ(ids.size(), ConditionCacheContext::GRANULE_SIZE); + EXPECT_EQ(ids.front(), ConditionCacheContext::GRANULE_SIZE); + EXPECT_EQ(ids.back(), ConditionCacheContext::GRANULE_SIZE * 2 - 1); + EXPECT_FALSE((*ctx->filter_result)[0]); + EXPECT_TRUE((*ctx->filter_result)[1]); + EXPECT_FALSE((*ctx->filter_result)[2]); +} + +TEST_F(NewParquetReaderTest, ConditionCacheHitSkipsFalseGranulesBeforeColumnRead) { + write_condition_cache_parquet_file(_file_path); + auto io_ctx = std::make_shared(); + auto reader = create_reader(0, -1, nullptr, false, io_ctx); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + create_int32_greater_than_conjunct(0, ConditionCacheContext::GRANULE_SIZE - 1)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + auto ctx = std::make_shared(); + ctx->is_hit = true; + ctx->filter_result = + std::make_shared>(std::vector {false, true, false}); + reader->set_condition_cache_context(ctx); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, ConditionCacheContext::GRANULE_SIZE); + EXPECT_EQ(io_ctx->condition_cache_filtered_rows, ConditionCacheContext::GRANULE_SIZE); + + const auto& ids = nullable_nested_column(block, 0); + EXPECT_EQ(ids.get_element(0), ConditionCacheContext::GRANULE_SIZE); + EXPECT_EQ(ids.get_element(rows - 1), ConditionCacheContext::GRANULE_SIZE * 2 - 1); + + block = build_file_block(schema); + rows = 0; + eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); +} + +TEST_F(NewParquetReaderTest, ReadMultipleRowGroups) { + write_parquet_file(_file_path, 2); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({1, 2, 3, 4, 5})); + EXPECT_EQ(values, std::vector({"one", "two", "three", "four", "five"})); +} + +TEST_F(NewParquetReaderTest, RewriteSameLocalPathDoesNotReuseUnknownMtimePageCache) { + RuntimeProfile first_profile("new_parquet_reader_first_unknown_mtime"); + { + auto reader = create_reader(0, -1, &first_profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + } + } + + ASSERT_NE(first_profile.get_counter("PageReadCount"), nullptr); + ASSERT_NE(first_profile.get_counter("PageCacheWriteCount"), nullptr); + EXPECT_EQ(first_profile.get_counter("PageReadCount")->value(), 0); + EXPECT_EQ(first_profile.get_counter("PageCacheWriteCount")->value(), 0); + + // LocalFileReader reports mtime as 0. Rewriting the same path must not reuse page-cache bytes + // from the previous physical file, even when the query option enables parquet file page cache. + write_int_pair_parquet_file(_file_path); + RuntimeProfile second_profile("new_parquet_reader_second_unknown_mtime"); + auto reader = create_reader(0, -1, &second_profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector scores; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& score_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + scores.push_back(score_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({1, 2, 3, 4, 5})); + EXPECT_EQ(scores, std::vector({1, 2, 3, 4, 5})); + ASSERT_NE(second_profile.get_counter("PageReadCount"), nullptr); + ASSERT_NE(second_profile.get_counter("PageCacheWriteCount"), nullptr); + EXPECT_EQ(second_profile.get_counter("PageReadCount")->value(), 0); + EXPECT_EQ(second_profile.get_counter("PageCacheWriteCount")->value(), 0); +} + +TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { + RuntimeProfile profile("new_parquet_reader_filter_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids = nullable_nested_column(block, 0); + const auto& values = nullable_nested_column(block, 1); + ASSERT_EQ(ids.size(), 3); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 4); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "three"); + EXPECT_EQ(values.get_data_at(1).to_string(), "four"); + EXPECT_EQ(values.get_data_at(2).to_string(), "five"); + + ASSERT_NE(profile.get_counter("FileReaderCreateTime"), nullptr); + ASSERT_NE(profile.get_counter("FileNum"), nullptr); + ASSERT_NE(profile.get_counter("RawRowsRead"), nullptr); + ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); + ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); + ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); + ASSERT_NE(profile.get_counter("ReaderReadRows"), nullptr); + ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); + ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); + ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + ASSERT_GT(profile.get_counter("FileReaderCreateTime")->value(), 0); + EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); + EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); + EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 3); + EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 2); + EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("ReaderReadRows")->value(), ROW_COUNT + 3); + EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 2); + EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 3); + EXPECT_GT(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + EXPECT_GT(profile.get_counter("MaterializationTime")->value(), 0); + + rows = 0; + eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); +} + +TEST_F(NewParquetReaderTest, GlobalRowIdSchemaAndSelectionUseFileRowPosition) { + format::GlobalRowIdContext context {.version = 7, .backend_id = 123456789, .file_id = 42}; + auto reader = create_reader(0, -1, nullptr, false, nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + EXPECT_EQ(schema[2].local_id, format::GLOBAL_ROWID_COLUMN_ID); + EXPECT_EQ(schema[2].column_type, format::GLOBAL_ROWID); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1), + field_projection(format::GLOBAL_ROWID_COLUMN_ID)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids = nullable_nested_column(block, 0); + const auto& values = nullable_nested_column(block, 1); + const auto& rowids = assert_cast(*block.get_by_position(2).column); + ASSERT_EQ(ids.size(), 3); + ASSERT_EQ(values.size(), 3); + ASSERT_EQ(rowids.size(), 3); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 4); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "three"); + EXPECT_EQ(values.get_data_at(1).to_string(), "four"); + EXPECT_EQ(values.get_data_at(2).to_string(), "five"); + + for (size_t row = 0; row < rows; ++row) { + const auto location = decode_rowid(rowids, row); + EXPECT_EQ(location.version, context.version); + EXPECT_EQ(location.backend_id, context.backend_id); + EXPECT_EQ(location.file_id, context.file_id); + EXPECT_EQ(location.row_id, static_cast(row + 2)); + } +} + +TEST_F(NewParquetReaderTest, ScanWithoutConjunctDoesNotFilterRowsInsideRowGroup) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, ROW_COUNT); + + const auto& ids = nullable_nested_column(block, 0); + const auto& values = nullable_nested_column(block, 1); + ASSERT_EQ(ids.size(), ROW_COUNT); + ASSERT_EQ(values.size(), ROW_COUNT); + EXPECT_EQ(ids.get_element(0), 1); + EXPECT_EQ(ids.get_element(4), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "one"); + EXPECT_EQ(values.get_data_at(4).to_string(), "five"); +} + +TEST_F(NewParquetReaderTest, EmptySelectionUpdatesProfileCounters) { + RuntimeProfile profile("new_parquet_reader_empty_selection_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(create_int32_sum_greater_than_conjunct(0, 0, 10)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); + + ASSERT_NE(profile.get_counter("RawRowsRead"), nullptr); + ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); + ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); + ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); + EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); + EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 0); + EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), ROW_COUNT); + EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 1); +} + +TEST_F(NewParquetReaderTest, ReadMultiPredicateColumnsBeforeExpressionFilter) { + write_int_pair_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0), field_projection(1)}; + request->non_predicate_columns = {}; + request->conjuncts.push_back(create_int32_sum_greater_than_conjunct(0, 1, 7)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 2); + + const auto& ids = nullable_nested_column(block, 0); + const auto& scores = nullable_nested_column(block, 1); + ASSERT_EQ(ids.size(), 2); + ASSERT_EQ(scores.size(), 2); + EXPECT_EQ(ids.get_element(0), 4); + EXPECT_EQ(ids.get_element(1), 5); + EXPECT_EQ(scores.get_element(0), 4); + EXPECT_EQ(scores.get_element(1), 5); +} + +TEST_F(NewParquetReaderTest, NonDeterministicPredicateKeepsFullBatchEvaluation) { + write_int_pair_parquet_file(_file_path); + RuntimeProfile profile("new_parquet_reader_non_deterministic_predicate_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + std::vector non_deterministic_executed_rows; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0), field_projection(1)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + request->conjuncts.push_back( + create_non_deterministic_counting_int32_conjunct(1, &non_deterministic_executed_rows)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids = nullable_nested_column(block, 0); + const auto& scores = nullable_nested_column(block, 1); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 4); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(scores.get_element(0), 3); + EXPECT_EQ(scores.get_element(1), 4); + EXPECT_EQ(scores.get_element(2), 5); + + // A non-deterministic predicate must stay on the old full-batch path. If it were left as a + // remaining conjunct while earlier deterministic predicates compacted later predicate columns, + // this expression would only see the three surviving rows instead of the original five. + EXPECT_EQ(non_deterministic_executed_rows, + std::vector({static_cast(ROW_COUNT)})); + ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); + EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 0); +} + +TEST_F(NewParquetReaderTest, SelectedRowsUnsafePredicateKeepsFullBatchEvaluation) { + write_int_pair_parquet_file(_file_path); + RuntimeProfile profile("new_parquet_reader_selected_rows_unsafe_predicate_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + std::vector unsafe_executed_rows; + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0), field_projection(1)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + request->conjuncts.push_back( + create_selected_rows_unsafe_counting_int32_conjunct(1, &unsafe_executed_rows)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids = nullable_nested_column(block, 0); + const auto& scores = nullable_nested_column(block, 1); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 4); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(scores.get_element(0), 3); + EXPECT_EQ(scores.get_element(1), 4); + EXPECT_EQ(scores.get_element(2), 5); + + // Error-preserving functions such as assert_true are deterministic, but moving them after an + // earlier predicate's compacted selection can hide errors from rows filtered by that earlier + // predicate. Such conjuncts therefore keep the old full-batch execution path. + EXPECT_EQ(unsafe_executed_rows, std::vector({static_cast(ROW_COUNT)})); + ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); + EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 0); +} + +TEST_F(NewParquetReaderTest, PredicateColumnFiltersBeforeNonPredicateRead) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids = nullable_nested_column(block, 0); + const auto& values = nullable_nested_column(block, 1); + ASSERT_EQ(ids.size(), 3); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 4); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(values.get_data_at(0).to_string(), "three"); + EXPECT_EQ(values.get_data_at(1).to_string(), "four"); + EXPECT_EQ(values.get_data_at(2).to_string(), "five"); +} + +TEST_F(NewParquetReaderTest, NonPredicateColumnKeepsSelectionFromPredicateColumn) { + write_int_pair_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& ids = nullable_nested_column(block, 0); + const auto& scores = nullable_nested_column(block, 1); + ASSERT_EQ(ids.size(), 3); + ASSERT_EQ(scores.size(), 3); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 4); + EXPECT_EQ(ids.get_element(2), 5); + EXPECT_EQ(scores.get_element(0), 3); + EXPECT_EQ(scores.get_element(1), 4); + EXPECT_EQ(scores.get_element(2), 5); +} + +TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByStatistics) { + write_parquet_file(_file_path, 2); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({3, 4, 5})); + EXPECT_EQ(values, std::vector({"three", "four", "five"})); +} + +TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { + write_dictionary_filter_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); + for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { + auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); + ASSERT_NE(row_group, nullptr); + auto value_chunk = row_group->ColumnChunk(1); + ASSERT_NE(value_chunk, nullptr); + ASSERT_TRUE(value_chunk->has_dictionary_page()); + ASSERT_TRUE(value_chunk->statistics() == nullptr || + !value_chunk->statistics()->HasMinMax()); + } + + std::vector> file_schema; + auto schema_descriptor = parquet_file_reader->metadata()->schema(); + ASSERT_NE(schema_descriptor, nullptr); + ASSERT_TRUE( + format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); + ASSERT_EQ(file_schema.size(), 2); + + format::FileScanRequest plan_request; + plan_request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); + plan_request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); + + format::parquet::RowGroupScanPlan plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + plan_request, scan_range, false, &plan) + .ok()); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 5); + EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 5); + EXPECT_EQ(plan.pruning_stats.selected_row_ranges, 1); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({3})); + EXPECT_EQ(values, std::vector({"lm"})); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { + write_single_row_group_dictionary_filter_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); + auto row_group = parquet_file_reader->metadata()->RowGroup(0); + ASSERT_NE(row_group, nullptr); + ASSERT_TRUE(row_group->ColumnChunk(1)->has_dictionary_page()); + + RuntimeProfile profile("new_parquet_reader_dictionary_filter_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"az", "za"})); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({2, 5})); + EXPECT_EQ(values, std::vector({"az", "za"})); + EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 4); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + EXPECT_EQ(profile.get_counter("DictFilterCandidateColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterUnsupportedColumns")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterReadFailures")->value(), 0); + ASSERT_NE(profile.get_counter("DictFilterExprRewriteTime"), nullptr); + ASSERT_NE(profile.get_counter("DictFilterReadDictTime"), nullptr); + ASSERT_NE(profile.get_counter("DictFilterBuildTime"), nullptr); + EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 2); + EXPECT_GE(profile.get_counter("ReaderSelectRows")->value(), 8); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) { + write_dictionary_filter_with_trailing_column_parquet_file(_file_path); + + RuntimeProfile profile("new_parquet_reader_dictionary_filter_merge_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0), field_projection(2)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"az", "za"})); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + std::vector payloads; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + const auto& payload_column = nullable_nested_column(block, 2); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + payloads.push_back(payload_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({2, 5})); + EXPECT_EQ(values, std::vector({"az", "za"})); + EXPECT_EQ(payloads, std::vector({20, 50})); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + ASSERT_NE(profile.get_counter("MergedIO"), nullptr); + ASSERT_NE(profile.get_counter("MergedBytes"), nullptr); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateWorksWithoutRuntimeProfile) { + write_single_row_group_dictionary_filter_parquet_file(_file_path); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"az", "za"})); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({2, 5})); + EXPECT_EQ(values, std::vector({"az", "za"})); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateSkipsRemainingPredicateColumnsWhenEmpty) { + write_single_row_group_dictionary_filter_parquet_file(_file_path); + + RuntimeProfile profile("new_parquet_reader_dictionary_filter_empty_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1), field_projection(0)}; + request->conjuncts.push_back( + create_string_dictionary_and_residual_conjunct(1, {"az"}, "not_present")); + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 0)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + + EXPECT_EQ(total_rows, 0); + EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 6); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 5); + EXPECT_EQ(profile.get_counter("DictFilterCandidateColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterUnsupportedColumns")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterReadFailures")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 0); + // The first dictionary predicate column is read once to produce a compact row filter. The + // second predicate column is skipped after the selection becomes empty, which verifies the + // StarRocks-style round-by-round policy: only rows surviving previous predicates are read. + EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 6); + EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 6); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateRunsResidualConjunctOnSurvivors) { + write_single_row_group_dictionary_filter_parquet_file(_file_path); + + RuntimeProfile profile("new_parquet_reader_dictionary_prefilter_residual_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + create_string_dictionary_and_residual_conjunct(1, {"az", "za"}, "za")); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({5})); + EXPECT_EQ(values, std::vector({"za"})); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 5); + EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 1); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateKeepsNestedOrResidualConjunct) { + write_single_row_group_dictionary_filter_parquet_file(_file_path); + + RuntimeProfile profile("new_parquet_reader_dictionary_nested_or_residual_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_nested_or_dictionary_and_residual_conjunct(1)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({2})); + EXPECT_EQ(values, std::vector({"az"})); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 5); + EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 1); +} + +TEST_F(NewParquetReaderTest, ScanRangeFiltersRowGroupsBeforeDictionaryPruning) { + write_dictionary_filter_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); + + std::vector> file_schema; + auto schema_descriptor = parquet_file_reader->metadata()->schema(); + ASSERT_NE(schema_descriptor, nullptr); + ASSERT_TRUE( + format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); + + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); + request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); + + const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 2); + format::parquet::ParquetScanRange scan_range; + scan_range.start_offset = range_start_offset; + scan_range.size = range_size; + scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); + + format::parquet::RowGroupScanPlan plan; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + ASSERT_EQ(plan.row_groups.size(), 1); + EXPECT_EQ(plan.row_groups[0].row_group_id, 2); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); + EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); +} + +TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByStatistics) { + write_struct_filter_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 2); + + std::vector> file_schema; + auto schema_descriptor = parquet_file_reader->metadata()->schema(); + ASSERT_NE(schema_descriptor, nullptr); + ASSERT_TRUE( + format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); + ASSERT_EQ(file_schema.size(), 1); + ASSERT_EQ(file_schema[0]->children.size(), 2); + ASSERT_EQ(file_schema[0]->children[0]->name, "id"); + + format::FileScanRequest request; + + format::parquet::RowGroupScanPlan plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + ASSERT_EQ(plan.row_groups.size(), 2); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 2); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 2); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); + EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); +} + +TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByDictionary) { + write_nested_dictionary_filter_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); + for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { + auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); + ASSERT_NE(row_group, nullptr); + auto name_chunk = row_group->ColumnChunk(1); + ASSERT_NE(name_chunk, nullptr); + ASSERT_TRUE(name_chunk->has_dictionary_page()); + ASSERT_TRUE(name_chunk->statistics() == nullptr || !name_chunk->statistics()->HasMinMax()); + } + + std::vector> file_schema; + auto schema_descriptor = parquet_file_reader->metadata()->schema(); + ASSERT_NE(schema_descriptor, nullptr); + ASSERT_TRUE( + format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); + ASSERT_EQ(file_schema.size(), 1); + ASSERT_EQ(file_schema[0]->children.size(), 2); + ASSERT_EQ(file_schema[0]->children[1]->name, "name"); + + format::FileScanRequest request; + + format::parquet::RowGroupScanPlan plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + ASSERT_EQ(plan.row_groups.size(), 6); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 6); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); + EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); +} + +TEST_F(NewParquetReaderTest, PlannerNarrowsRowRangesByPageIndex) { + write_page_index_filter_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); + auto page_index_reader = parquet_file_reader->GetPageIndexReader(); + ASSERT_NE(page_index_reader, nullptr); + auto row_group_index_reader = page_index_reader->RowGroup(0); + ASSERT_NE(row_group_index_reader, nullptr); + auto offset_index = row_group_index_reader->GetOffsetIndex(0); + ASSERT_NE(offset_index, nullptr); + ASSERT_GT(offset_index->page_locations().size(), 1); + + std::vector> file_schema; + auto schema_descriptor = parquet_file_reader->metadata()->schema(); + ASSERT_NE(schema_descriptor, nullptr); + ASSERT_TRUE( + format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); + ASSERT_EQ(file_schema.size(), 1); + + format::FileScanRequest request; + request.predicate_columns = {field_projection(0)}; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); + + format::parquet::RowGroupScanPlan plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + ASSERT_EQ(plan.row_groups.size(), 1); + ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); + EXPECT_GT(plan.row_groups[0].selected_ranges.front().start, 0); + EXPECT_LT(plan.row_groups[0].selected_ranges.front().length, 128); + auto skip_plan_it = plan.row_groups[0].page_skip_plans.find(0); + ASSERT_NE(skip_plan_it, plan.row_groups[0].page_skip_plans.end()); + EXPECT_EQ(skip_plan_it->second.leaf_column_id, 0); + EXPECT_GT(skip_plan_it->second.skipped_ranges.size(), 0); + EXPECT_GT(skip_plan_it->second.skipped_pages.size(), 1); + ASSERT_EQ(skip_plan_it->second.skipped_pages.size(), + skip_plan_it->second.skipped_page_compressed_sizes.size()); + int64_t skipped_compressed_bytes = 0; + for (size_t page_idx = 0; page_idx < skip_plan_it->second.skipped_pages.size(); ++page_idx) { + if (skip_plan_it->second.should_skip_page(page_idx)) { + skipped_compressed_bytes += skip_plan_it->second.skipped_page_compressed_size(page_idx); + } + } + EXPECT_GT(skipped_compressed_bytes, 0); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); + EXPECT_GT(plan.pruning_stats.filtered_page_rows, 0); + EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); +} + +TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotNarrowRowRangesByPageIndex) { + write_nested_page_index_filter_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); + auto page_index_reader = parquet_file_reader->GetPageIndexReader(); + ASSERT_NE(page_index_reader, nullptr); + auto row_group_index_reader = page_index_reader->RowGroup(0); + ASSERT_NE(row_group_index_reader, nullptr); + auto offset_index = row_group_index_reader->GetOffsetIndex(0); + ASSERT_NE(offset_index, nullptr); + ASSERT_GT(offset_index->page_locations().size(), 1); + + std::vector> file_schema; + auto schema_descriptor = parquet_file_reader->metadata()->schema(); + ASSERT_NE(schema_descriptor, nullptr); + ASSERT_TRUE( + format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); + ASSERT_EQ(file_schema.size(), 1); + ASSERT_EQ(file_schema[0]->children.size(), 2); + ASSERT_EQ(file_schema[0]->children[0]->name, "id"); + + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); + + format::parquet::RowGroupScanPlan plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + ASSERT_EQ(plan.row_groups.size(), 1); + ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); + EXPECT_EQ(plan.row_groups[0].selected_ranges.front().start, 0); + EXPECT_EQ(plan.row_groups[0].selected_ranges.front().length, + parquet_file_reader->metadata()->RowGroup(0)->num_rows()); + EXPECT_TRUE(plan.row_groups[0].page_skip_plans.empty()); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); + EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 0); + EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); +} + +TEST_F(NewParquetReaderTest, PageIndexFilteredPagesDoNotDoubleSkipOutputColumns) { + write_page_index_filter_pair_parquet_file(_file_path); + RuntimeProfile profile("new_parquet_reader_page_skip"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector payloads; + bool eof = false; + while (!eof) { + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& payload_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + payloads.push_back(payload_column.get_element(row)); + } + } + + ASSERT_NE(profile.get_counter("PagesSkippedByDataPageFilter"), nullptr); + ASSERT_NE(profile.get_counter("DataPageFilterSkipBytes"), nullptr); + ASSERT_NE(profile.get_counter("RawRowsRead"), nullptr); + ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); + ASSERT_NE(profile.get_counter("RangeGapSkippedRows"), nullptr); + ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupFilterTime"), nullptr); + ASSERT_NE(profile.get_counter("PageIndexFilterTime"), nullptr); + ASSERT_NE(profile.get_counter("PageIndexReadTime"), nullptr); + EXPECT_GT(profile.get_counter("PagesSkippedByDataPageFilter")->value(), 0); + EXPECT_GT(profile.get_counter("DataPageFilterSkipBytes")->value(), 0); + EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), 64); + EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 64); + EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); + EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 0); + EXPECT_GT(profile.get_counter("RowGroupFilterTime")->value(), 0); + EXPECT_GT(profile.get_counter("PageIndexFilterTime")->value(), 0); + EXPECT_GT(profile.get_counter("PageIndexReadTime")->value(), 0); + + ASSERT_EQ(ids.size(), 64); + ASSERT_EQ(payloads.size(), ids.size()); + for (size_t row = 0; row < ids.size(); ++row) { + EXPECT_EQ(ids[row], static_cast(row + 64)); + EXPECT_EQ(payloads[row], ids[row] + 1000); + } +} + +TEST_F(NewParquetReaderTest, InPredicateFiltersRowGroupsByDictionary) { + write_dictionary_filter_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"az", "za"})); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({2, 5})); + EXPECT_EQ(values, std::vector({"az", "za"})); +} + +TEST_F(NewParquetReaderTest, DictionaryPageV2StringEdgesSurviveSelection) { + write_dictionary_edge_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 4); + for (int row_group_idx = 0; row_group_idx < 4; ++row_group_idx) { + auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); + ASSERT_NE(row_group, nullptr); + ASSERT_TRUE(row_group->ColumnChunk(1)->has_dictionary_page()); + } + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"", "same"})); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({1, 2, 5, 7})); + EXPECT_EQ(values, std::vector({"", "same", "", "same"})); +} + +TEST_F(NewParquetReaderTest, StatisticsPruningSkipsPrefixRowGroupsAndReadsLaterGroups) { + write_parquet_file(_file_path, 1); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 5); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 3)); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(ids, std::vector({4, 5})); + EXPECT_EQ(values, std::vector({"four", "five"})); +} + +TEST_F(NewParquetReaderTest, RowPositionReaderReturnsFileLocalPositions) { + write_parquet_file(_file_path, 2); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(format::ROW_POSITION_COLUMN_ID), + field_projection(0)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(format::ROW_POSITION_COLUMN_ID), format::LocalIndex(2)}, + }; + ASSERT_TRUE(reader->open(request).ok()); + + std::vector row_positions; + std::vector ids; + bool eof = false; + while (!eof) { + Block block = build_file_block_with_row_position(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& row_position_column = + assert_cast(*block.get_by_position(2).column); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + row_positions.push_back(row_position_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({1, 2, 3, 4, 5})); + EXPECT_EQ(row_positions, std::vector({0, 1, 2, 3, 4})); +} + +TEST_F(NewParquetReaderTest, RowPositionReaderKeepsPositionsAfterSelection) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block_with_row_position(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(format::ROW_POSITION_COLUMN_ID)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(format::ROW_POSITION_COLUMN_ID), format::LocalIndex(2)}, + }; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& id_column = nullable_nested_column(block, 0); + const auto& row_position_column = + assert_cast(*block.get_by_position(2).column); + EXPECT_EQ(id_column.get_element(0), 3); + EXPECT_EQ(id_column.get_element(1), 4); + EXPECT_EQ(id_column.get_element(2), 5); + EXPECT_EQ(row_position_column.get_element(0), 2); + EXPECT_EQ(row_position_column.get_element(1), 3); + EXPECT_EQ(row_position_column.get_element(2), 4); +} + +TEST_F(NewParquetReaderTest, DeletePredicateFiltersRowPositions) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block_with_row_position(schema); + + static const std::vector deleted_rows {1, 3}; + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->add_child(VSlotRef::create_shared(2, 2, -1, std::make_shared(), + format::ROW_POSITION_COLUMN_NAME)); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(format::ROW_POSITION_COLUMN_ID)}; + request->non_predicate_columns = {field_projection(0)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(format::ROW_POSITION_COLUMN_ID), format::LocalIndex(2)}, + }; + request->delete_conjuncts.push_back(VExprContext::create_shared(std::move(delete_predicate))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 3); + + const auto& id_column = nullable_nested_column(block, 0); + const auto& row_position_column = + assert_cast(*block.get_by_position(2).column); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 3); + EXPECT_EQ(id_column.get_element(2), 5); + EXPECT_EQ(row_position_column.get_element(0), 0); + EXPECT_EQ(row_position_column.get_element(1), 2); + EXPECT_EQ(row_position_column.get_element(2), 4); +} + +TEST_F(NewParquetReaderTest, QueryPredicateAndDeletePredicateFilterRowPositions) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block_with_row_position(schema); + + static const std::vector deleted_rows {3}; + auto delete_predicate = std::make_shared(deleted_rows); + delete_predicate->add_child(VSlotRef::create_shared(2, 2, -1, std::make_shared(), + format::ROW_POSITION_COLUMN_NAME)); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0), + field_projection(format::ROW_POSITION_COLUMN_ID)}; + request->non_predicate_columns = {}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(format::ROW_POSITION_COLUMN_ID), format::LocalIndex(2)}, + }; + request->conjuncts.push_back(create_int32_greater_than_conjunct(0, 2)); + request->delete_conjuncts.push_back(VExprContext::create_shared(std::move(delete_predicate))); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_FALSE(eof); + ASSERT_EQ(rows, 2); + + const auto& id_column = nullable_nested_column(block, 0); + const auto& row_position_column = + assert_cast(*block.get_by_position(2).column); + EXPECT_EQ(id_column.get_element(0), 3); + EXPECT_EQ(id_column.get_element(1), 5); + EXPECT_EQ(row_position_column.get_element(0), 2); + EXPECT_EQ(row_position_column.get_element(1), 4); +} + +TEST_F(NewParquetReaderTest, RowPositionReaderUsesFileLocalPositionsForScanRange) { + write_parquet_file(_file_path, 2); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); + + const std::vector> expected_ids = {{1, 2}, {3, 4}, {5}}; + const std::vector> expected_row_positions = {{0, 1}, {2, 3}, {4}}; + for (int row_group_idx = 0; row_group_idx < 3; ++row_group_idx) { + const auto [range_start_offset, range_size] = + row_group_mid_range(_file_path, row_group_idx); + auto reader = create_reader(range_start_offset, range_size); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(format::ROW_POSITION_COLUMN_ID), + field_projection(0)}; + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(format::ROW_POSITION_COLUMN_ID), format::LocalIndex(2)}, + }; + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector row_positions; + bool eof = false; + while (!eof) { + Block block = build_file_block_with_row_position(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& row_position_column = + assert_cast(*block.get_by_position(2).column); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + row_positions.push_back(row_position_column.get_element(row)); + } + } + + EXPECT_EQ(ids, expected_ids[row_group_idx]); + EXPECT_EQ(row_positions, expected_row_positions[row_group_idx]); + } +} + +} // namespace +} // namespace doris diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp new file mode 100644 index 00000000000000..04d46097c98648 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -0,0 +1,1158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_scan.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "gen_cpp/Types_types.h" +#include "io/io_common.h" +#include "runtime/runtime_state.h" +#include "storage/index/zone_map/zonemap_eval_context.h" +#include "storage/index/zone_map/zonemap_filter_result.h" +#include "storage/utils.h" + +namespace doris { +namespace { + +format::LocalColumnIndex field_projection(int32_t column_id) { + return format::LocalColumnIndex {.index = column_id}; +} + +const ColumnInt32& int32_data_column(const IColumn& column) { + if (const auto* nullable_column = check_and_get_column(&column)) { + return assert_cast(nullable_column->get_nested_column()); + } + return assert_cast(column); +} + +const ColumnString& string_data_column(const IColumn& column) { + if (const auto* nullable_column = check_and_get_column(&column)) { + return assert_cast(nullable_column->get_nested_column()); + } + return assert_cast(column); +} + +class Int32ZoneMapExpr final : public VExpr { +public: + enum class Op { GE, GT, LT }; + + Int32ZoneMapExpr(int column_id, Op op, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _op(op), + _value(value) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(selector == nullptr); + DORIS_CHECK(_column_id >= 0 && _column_id < static_cast(block->columns())); + const auto& data_column = int32_data_column(*block->get_by_position(_column_id).column); + DORIS_CHECK(data_column.size() >= count); + + auto result = ColumnUInt8::create(count, 0); + auto& result_data = result->get_data(); + for (size_t row = 0; row < count; ++row) { + const auto value = data_column.get_element(row); + if (_op == Op::GE) { + result_data[row] = value >= _value; + } else if (_op == Op::GT) { + result_data[row] = value > _value; + } else { + result_data[row] = value < _value; + } + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_evaluate_zonemap_filter() const override { return true; } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + auto zone_map = ctx.zone_map(_column_id); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + const auto literal = Field::create_field(_value); + if (_op == Op::GE) { + return zone_map->max_value < literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + if (_op == Op::GT) { + return zone_map->max_value <= literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + return zone_map->min_value >= literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + +private: + int _column_id; + Op _op; + int32_t _value; + const std::string _expr_name = "Int32ZoneMapExpr"; +}; + +class Int32PairSumExpr final : public VExpr { +public: + Int32PairSumExpr(int left_column_id, int right_column_id, int32_t upper_bound) + : VExpr(std::make_shared(), false), + _left_column_id(left_column_id), + _right_column_id(right_column_id), + _upper_bound(upper_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(selector == nullptr); + DORIS_CHECK(_left_column_id >= 0 && _left_column_id < static_cast(block->columns())); + DORIS_CHECK(_right_column_id >= 0 && _right_column_id < static_cast(block->columns())); + const auto& left_column = + int32_data_column(*block->get_by_position(_left_column_id).column); + const auto& right_column = + int32_data_column(*block->get_by_position(_right_column_id).column); + DORIS_CHECK(left_column.size() >= count); + DORIS_CHECK(right_column.size() >= count); + + auto result = ColumnUInt8::create(count, 0); + auto& result_data = result->get_data(); + for (size_t row = 0; row < count; ++row) { + result_data[row] = + left_column.get_element(row) + right_column.get_element(row) < _upper_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_left_column_id); + column_ids.insert(_right_column_id); + } + +private: + int _left_column_id; + int _right_column_id; + int32_t _upper_bound; + const std::string _expr_name = "Int32PairSumExpr"; +}; + +VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr::Op op, + int32_t value) { + return VExprContext::create_shared(std::make_shared(column_id, op, value)); +} + +VExprContextSPtr create_int32_pair_sum_conjunct(int left_column_id, int right_column_id, + int32_t upper_bound) { + return VExprContext::create_shared( + std::make_shared(left_column_id, right_column_id, upper_bound)); +} + +int64_t counter_value(RuntimeProfile& profile, const std::string& name) { + auto* counter = profile.get_counter(name); + DORIS_CHECK(counter != nullptr); + return counter->value(); +} + +std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { + std::shared_ptr array; + EXPECT_TRUE(builder->Finish(&array).ok()); + return array; +} + +std::shared_ptr build_int32_array(const std::vector& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_string_array(const std::vector& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_fixed_binary_array(const std::vector& values, + int byte_width) { + auto type = arrow::fixed_size_binary(byte_width); + arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); + for (const auto& value : values) { + EXPECT_EQ(value.size(), byte_width); + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_struct_array(const std::vector& ids, + const std::vector& names) { + auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false), + arrow::field("name", arrow::utf8(), false)}); + std::vector> field_builders; + field_builders.push_back(std::shared_ptr( + std::make_unique().release())); + field_builders.push_back(std::shared_ptr( + std::make_unique().release())); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* id_builder = assert_cast(builder.field_builder(0)); + auto* name_builder = assert_cast(builder.field_builder(1)); + for (size_t row = 0; row < ids.size(); ++row) { + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(id_builder->Append(ids[row]).ok()); + EXPECT_TRUE(name_builder->Append(names[row]).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_list_array() { + auto value_builder = std::make_unique(); + arrow::ListBuilder builder(arrow::default_memory_pool(), std::move(value_builder)); + auto* int_builder = assert_cast(builder.value_builder()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(int_builder->Append(1).ok()); + EXPECT_TRUE(int_builder->Append(2).ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(int_builder->Append(3).ok()); + EXPECT_TRUE(builder.Append().ok()); + return finish_array(&builder); +} + +void write_table(const std::string& file_path, const std::shared_ptr& table, + int64_t row_group_size, bool enable_dictionary = false, + bool enable_page_index = false, bool enable_statistics = true) { + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + if (enable_dictionary) { + builder.enable_dictionary(); + } else { + builder.disable_dictionary(); + } + if (enable_page_index) { + builder.enable_write_page_index(); + builder.write_batch_size(8); + builder.data_pagesize(10); + } + if (!enable_statistics) { + builder.disable_statistics(); + } + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + row_group_size, builder.build())); +} + +void write_int_pair_parquet_file(const std::string& file_path, int64_t row_group_size = 2, + bool enable_statistics = true) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("score", arrow::int32(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_int32_array({10, 20, 30, 40, 50, 60})}); + write_table(file_path, table, row_group_size, false, false, enable_statistics); +} + +void write_binary_minmax_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({ + arrow::field("text", arrow::utf8(), false), + arrow::field("fixed", arrow::fixed_size_binary(4), false), + }); + auto table = arrow::Table::Make(schema, {build_string_array({"alpha", "omega"}), + build_fixed_binary_array({"aaaa", "zzzz"}, 4)}); + write_table(file_path, table, 2); +} + +void write_struct_parquet_file(const std::string& file_path) { + auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false), + arrow::field("name", arrow::utf8(), false)}); + auto schema = arrow::schema({ + arrow::field("s", struct_type, false), + }); + auto table = arrow::Table::Make( + schema, {build_struct_array({1, 2, 10, 11}, {"one", "two", "ten", "eleven"})}); + write_table(file_path, table, 2); +} + +void write_list_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({ + arrow::field("xs", arrow::list(arrow::int32()), false), + }); + auto table = arrow::Table::Make(schema, {build_list_array()}); + write_table(file_path, table, 2); +} + +void write_page_index_parquet_file(const std::string& file_path) { + std::vector ids(128); + std::iota(ids.begin(), ids.end(), 0); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + write_table(file_path, table, ids.size(), false, true); +} + +void write_page_index_pair_parquet_file(const std::string& file_path) { + std::vector ids(128); + std::iota(ids.begin(), ids.end(), 0); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("score", arrow::int32(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(ids)}); + write_table(file_path, table, ids.size(), false, true); +} + +int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { + return column_metadata.has_dictionary_page() + ? static_cast(column_metadata.dictionary_page_offset()) + : static_cast(column_metadata.data_page_offset()); +} + +std::pair row_group_mid_range(const std::string& file_path, int row_group_idx) { + auto reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); + auto metadata = reader->metadata(); + auto row_group_metadata = metadata->RowGroup(row_group_idx); + auto first_column = row_group_metadata->ColumnChunk(0); + auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); + const int64_t row_group_start_offset = parquet_column_start_offset(*first_column); + const int64_t row_group_end_offset = + parquet_column_start_offset(*last_column) + last_column->total_compressed_size(); + const int64_t row_group_mid_offset = + row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; + return {row_group_mid_offset, 1}; +} + +Block build_file_block(const std::vector& schema) { + Block block; + for (const auto& field : schema) { + block.insert({field.type->create_column(), field.type, field.name}); + } + return block; +} + +GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { + const auto ref = column.get_data_at(row); + EXPECT_EQ(ref.size, sizeof(GlobalRowLoacationV2)); + GlobalRowLoacationV2 location(0, 0, 0, 0); + std::memcpy(&location, ref.data, sizeof(GlobalRowLoacationV2)); + return location; +} + +void use_schema_order_positions(format::FileScanRequest* request, + const std::vector& schema) { + DORIS_CHECK(request != nullptr); + for (size_t idx = 0; idx < schema.size(); ++idx) { + request->local_positions.emplace(format::LocalColumnId(schema[idx].local_id), + format::LocalIndex(idx)); + } +} + +std::vector> build_file_schema( + const ::parquet::ParquetFileReader& reader) { + std::vector> file_schema; + auto schema_descriptor = reader.metadata()->schema(); + EXPECT_NE(schema_descriptor, nullptr); + EXPECT_TRUE( + format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); + return file_schema; +} + +int64_t count_range_rows(const std::vector& ranges) { + int64_t rows = 0; + for (const auto& range : ranges) { + rows += range.length; + } + return rows; +} + +class ParquetScanTest : public testing::Test { +protected: + void SetUp() override { + _test_dir = std::filesystem::temp_directory_path() / "doris_format_v2_parquet_scan_test"; + std::filesystem::remove_all(_test_dir); + std::filesystem::create_directories(_test_dir); + _file_path = (_test_dir / "scan.parquet").string(); + } + + void TearDown() override { std::filesystem::remove_all(_test_dir); } + + std::unique_ptr create_reader( + int64_t range_start_offset = 0, int64_t range_size = -1, + RuntimeProfile* profile = nullptr, + std::optional global_rowid_context = std::nullopt, + std::shared_ptr io_ctx = nullptr) const { + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto file_description = std::make_unique(); + file_description->path = _file_path; + file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); + file_description->range_start_offset = range_start_offset; + file_description->range_size = range_size; + return std::make_unique(system_properties, file_description, + std::move(io_ctx), profile, + global_rowid_context); + } + + std::shared_ptr open_all_row_groups( + format::parquet::ParquetReader* reader) { + auto request = std::make_shared(); + EXPECT_TRUE(reader->open(request).ok()); + return request; + } + + std::filesystem::path _test_dir; + std::string _file_path; +}; + +TEST(ParquetScanSelectionTest, CompactFilterShrinksCurrentSelection) { + format::parquet::SelectionVector selection(4); + selection.set_index(0, 0); + selection.set_index(1, 2); + selection.set_index(2, 4); + selection.set_index(3, 5); + + const IColumn::Filter compact_filter {1, 0, 1, 0}; + const auto selected_rows = + format::parquet::apply_compact_filter_to_selection(compact_filter, &selection, 4); + + ASSERT_EQ(selected_rows, 2); + EXPECT_EQ(selection.get_index(0), 0); + EXPECT_EQ(selection.get_index(1), 4); + EXPECT_TRUE(selection.verify(selected_rows, 6).ok()); +} + +TEST_F(ParquetScanTest, PlanRowGroupsAppliesScanRangeBeforeStatistics) { + write_int_pair_parquet_file(_file_path, 2); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); + auto file_schema = build_file_schema(*parquet_file_reader); + + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); + + const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 1); + format::parquet::ParquetScanRange scan_range; + scan_range.start_offset = range_start_offset; + scan_range.size = range_size; + scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); + + format::parquet::RowGroupScanPlan plan; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + EXPECT_TRUE(plan.row_groups.empty()); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 3); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 1); + EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 2); +} + +TEST_F(ParquetScanTest, PlanRowGroupsPreservesFirstFileRowAcrossPrunedRowGroups) { + write_int_pair_parquet_file(_file_path, 2); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); + auto file_schema = build_file_schema(*parquet_file_reader); + + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); + + format::parquet::RowGroupScanPlan plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + ASSERT_EQ(plan.row_groups.size(), 1); + EXPECT_EQ(plan.row_groups[0].row_group_id, 2); + EXPECT_EQ(plan.row_groups[0].first_file_row, 4); + EXPECT_EQ(plan.row_groups[0].row_group_rows, 2); + ASSERT_EQ(plan.row_groups[0].selected_ranges.size(), 1); + EXPECT_EQ(plan.row_groups[0].selected_ranges[0].start, 0); + EXPECT_EQ(plan.row_groups[0].selected_ranges[0].length, 2); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 2); + EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 4); +} + +TEST_F(ParquetScanTest, PlanRowGroupsSelectsAllRowGroupsWithoutFilters) { + write_int_pair_parquet_file(_file_path, 2); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); + auto file_schema = build_file_schema(*parquet_file_reader); + + format::FileScanRequest request; + format::parquet::RowGroupScanPlan plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + + ASSERT_EQ(plan.row_groups.size(), 3); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 3); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 3); + for (size_t row_group_idx = 0; row_group_idx < plan.row_groups.size(); ++row_group_idx) { + EXPECT_EQ(plan.row_groups[row_group_idx].row_group_id, row_group_idx); + EXPECT_EQ(plan.row_groups[row_group_idx].first_file_row, + static_cast(row_group_idx * 2)); + ASSERT_EQ(plan.row_groups[row_group_idx].selected_ranges.size(), 1); + EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].start, 0); + EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].length, 2); + EXPECT_TRUE(plan.row_groups[row_group_idx].page_skip_plans.empty()); + } +} + +TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater) { + format::parquet::RowGroupScanPlan plan; + plan.row_groups.push_back( + {.row_group_id = 1, + .first_file_row = ConditionCacheContext::GRANULE_SIZE, + .row_group_rows = ConditionCacheContext::GRANULE_SIZE, + .selected_ranges = {{.start = 0, .length = ConditionCacheContext::GRANULE_SIZE}}, + .page_skip_plans = {}}); + + format::parquet::ParquetScanScheduler scheduler; + scheduler.set_plan(std::move(plan)); + auto ctx = std::make_shared(); + ctx->is_hit = true; + ctx->base_granule = 0; + ctx->filter_result = std::make_shared>(std::vector {false}); + scheduler.set_condition_cache_context(ctx); + + EXPECT_FALSE(scheduler.empty()); + EXPECT_EQ(scheduler.condition_cache_filtered_rows(), 0); + EXPECT_EQ(ctx->base_granule, 0); +} + +TEST_F(ParquetScanTest, PageIndexIntersectsMultipleFiltersAndBuildsSkipPlan) { + write_page_index_pair_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); + auto file_schema = build_file_schema(*parquet_file_reader); + + format::FileScanRequest single_filter_request; + format::FileScanRequestBuilder single_filter_builder(&single_filter_request); + ASSERT_TRUE(single_filter_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + single_filter_request.conjuncts.push_back( + create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); + format::parquet::RowGroupScanPlan single_filter_plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups( + *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, + single_filter_request, scan_range, false, &single_filter_plan) + .ok()); + ASSERT_EQ(single_filter_plan.row_groups.size(), 1); + const int64_t single_filter_rows = + count_range_rows(single_filter_plan.row_groups[0].selected_ranges); + + format::FileScanRequest intersect_request; + format::FileScanRequestBuilder intersect_builder(&intersect_request); + ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + intersect_request.conjuncts.push_back( + create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); + intersect_request.conjuncts.push_back( + create_int32_zonemap_conjunct(1, Int32ZoneMapExpr::Op::LT, 96)); + format::parquet::RowGroupScanPlan intersect_plan; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups( + *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, + intersect_request, scan_range, false, &intersect_plan) + .ok()); + ASSERT_EQ(intersect_plan.row_groups.size(), 1); + ASSERT_FALSE(intersect_plan.row_groups[0].selected_ranges.empty()); + const int64_t intersect_rows = count_range_rows(intersect_plan.row_groups[0].selected_ranges); + EXPECT_GT(single_filter_rows, intersect_rows); + EXPECT_GT(intersect_plan.row_groups[0].selected_ranges.front().start, 0); + const auto& last_range = intersect_plan.row_groups[0].selected_ranges.back(); + EXPECT_LT(last_range.start + last_range.length, 128); + EXPECT_GT(intersect_plan.pruning_stats.filtered_page_rows, 0); + EXPECT_EQ(intersect_plan.pruning_stats.selected_row_ranges, + intersect_plan.row_groups[0].selected_ranges.size()); + + auto id_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(0); + ASSERT_NE(id_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); + EXPECT_EQ(id_skip_plan->second.leaf_column_id, 0); + EXPECT_FALSE(id_skip_plan->second.empty()); + auto score_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(1); + ASSERT_NE(score_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); + EXPECT_EQ(score_skip_plan->second.leaf_column_id, 1); + EXPECT_FALSE(score_skip_plan->second.empty()); +} + +TEST_F(ParquetScanTest, PageIndexCanFullyFilterRowGroupAfterRangeIntersection) { + write_page_index_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); + auto file_schema = build_file_schema(*parquet_file_reader); + + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); + request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::LT, 32)); + + format::parquet::RowGroupScanPlan plan; + format::parquet::ParquetScanRange scan_range; + ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), + parquet_file_reader.get(), file_schema, + request, scan_range, false, &plan) + .ok()); + EXPECT_TRUE(plan.row_groups.empty()); + EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); + EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); + EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 1); + EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 128); +} + +TEST_F(ParquetScanTest, PageIndexFullRangeWhenDisabledOrUnavailable) { + write_page_index_parquet_file(_file_path); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + auto file_schema = build_file_schema(*parquet_file_reader); + + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); + + const bool old_enable_page_index = config::enable_parquet_page_index; + config::enable_parquet_page_index = false; + std::vector selected_ranges; + std::map page_skip_plans; + format::parquet::ParquetPruningStats pruning_stats; + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( + parquet_file_reader.get(), file_schema, request, 0, 128, &selected_ranges, + &page_skip_plans, &pruning_stats) + .ok()); + config::enable_parquet_page_index = old_enable_page_index; + ASSERT_EQ(selected_ranges.size(), 1); + EXPECT_EQ(selected_ranges[0].start, 0); + EXPECT_EQ(selected_ranges[0].length, 128); + EXPECT_TRUE(page_skip_plans.empty()); + EXPECT_EQ(pruning_stats.page_index_read_calls, 0); + + write_int_pair_parquet_file(_file_path, 6); + auto no_index_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + auto no_index_schema = build_file_schema(*no_index_reader); + format::FileScanRequest no_index_request; + no_index_request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + no_index_request.conjuncts.push_back( + create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 3)); + selected_ranges.clear(); + page_skip_plans.clear(); + pruning_stats = {}; + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( + no_index_reader.get(), no_index_schema, no_index_request, 0, 6, + &selected_ranges, &page_skip_plans, &pruning_stats) + .ok()); + ASSERT_EQ(selected_ranges.size(), 1); + EXPECT_EQ(selected_ranges[0].start, 0); + EXPECT_EQ(selected_ranges[0].length, 6); + EXPECT_TRUE(page_skip_plans.empty()); +} + +TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { + write_int_pair_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + open_all_row_groups(reader.get()); + + format::FileAggregateResult count_result; + format::FileAggregateRequest count_request; + count_request.agg_type = TPushAggOp::COUNT; + ASSERT_TRUE(reader->get_aggregate_result(count_request, &count_result).ok()); + EXPECT_EQ(count_result.count, 6); + EXPECT_TRUE(count_result.columns.empty()); + + format::FileAggregateResult minmax_result; + format::FileAggregateRequest minmax_request; + minmax_request.agg_type = TPushAggOp::MINMAX; + minmax_request.columns.push_back({.projection = field_projection(0)}); + minmax_request.columns.push_back({.projection = field_projection(1)}); + ASSERT_TRUE(reader->get_aggregate_result(minmax_request, &minmax_result).ok()); + EXPECT_EQ(minmax_result.count, 6); + ASSERT_EQ(minmax_result.columns.size(), 2); + EXPECT_TRUE(minmax_result.columns[0].has_min); + EXPECT_TRUE(minmax_result.columns[0].has_max); + EXPECT_EQ(minmax_result.columns[0].min_value.get(), 1); + EXPECT_EQ(minmax_result.columns[0].max_value.get(), 6); + EXPECT_EQ(minmax_result.columns[1].min_value.get(), 10); + EXPECT_EQ(minmax_result.columns[1].max_value.get(), 60); +} + +TEST_F(ParquetScanTest, AggregateMinMaxRejectsInexactBinaryStatistics) { + write_binary_minmax_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + open_all_row_groups(reader.get()); + + for (int32_t column_id = 0; column_id < 2; ++column_id) { + format::FileAggregateRequest request; + request.agg_type = TPushAggOp::MINMAX; + request.columns.push_back({.projection = field_projection(column_id)}); + format::FileAggregateResult result; + const auto status = reader->get_aggregate_result(request, &result); + EXPECT_TRUE(status.is()) << status; + } +} + +TEST_F(ParquetScanTest, AggregateRespectsStatisticsPrunedRowGroups) { + write_int_pair_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::MINMAX; + aggregate_request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult result; + ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, &result).ok()); + EXPECT_EQ(result.count, 2); + ASSERT_EQ(result.columns.size(), 1); + EXPECT_EQ(result.columns[0].min_value.get(), 5); + EXPECT_EQ(result.columns[0].max_value.get(), 6); +} + +TEST_F(ParquetScanTest, AggregateCountKeepsRowGroupRowsAfterPageIndexPruning) { + write_page_index_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::COUNT; + format::FileAggregateResult result; + ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, &result).ok()); + EXPECT_EQ(result.count, 128); +} + +TEST_F(ParquetScanTest, AggregateMinMaxSupportsNestedSingleLeafProjection) { + write_struct_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + open_all_row_groups(reader.get()); + + format::LocalColumnIndex nested_id = format::LocalColumnIndex::partial_local(0); + nested_id.children.push_back(field_projection(0)); + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::MINMAX; + aggregate_request.columns.push_back({.projection = nested_id}); + format::FileAggregateResult result; + ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, &result).ok()); + EXPECT_EQ(result.count, 4); + ASSERT_EQ(result.columns.size(), 1); + EXPECT_EQ(result.columns[0].min_value.get(), 1); + EXPECT_EQ(result.columns[0].max_value.get(), 11); +} + +TEST_F(ParquetScanTest, AggregateCountOnStructRecordsSelectedRowsRead) { + write_struct_parquet_file(_file_path); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + auto reader = create_reader(0, -1, nullptr, std::nullopt, io_ctx); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + open_all_row_groups(reader.get()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::COUNT; + aggregate_request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult result; + ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, &result).ok()); + EXPECT_EQ(result.count, 4); + EXPECT_EQ(file_reader_stats.read_rows, 4); +} + +TEST_F(ParquetScanTest, AggregateCountOnStructReturnsEndOfFileWhenStopped) { + write_struct_parquet_file(_file_path); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + auto reader = create_reader(0, -1, nullptr, std::nullopt, io_ctx); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + open_all_row_groups(reader.get()); + io_ctx->should_stop = true; + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::COUNT; + aggregate_request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult result; + const auto status = reader->get_aggregate_result(aggregate_request, &result); + EXPECT_TRUE(status.is()) << status; + EXPECT_EQ(file_reader_stats.read_rows, 0); +} + +TEST_F(ParquetScanTest, AggregateRejectsRepeatedMissingStatisticsAndInvalidRequests) { + write_list_parquet_file(_file_path); + auto repeated_reader = create_reader(); + RuntimeState repeated_state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(repeated_reader->init(&repeated_state).ok()); + open_all_row_groups(repeated_reader.get()); + + format::FileAggregateRequest repeated_request; + repeated_request.agg_type = TPushAggOp::MINMAX; + repeated_request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult repeated_result; + EXPECT_FALSE(repeated_reader->get_aggregate_result(repeated_request, &repeated_result).ok()); + + write_int_pair_parquet_file(_file_path, 2, false); + auto no_stats_reader = create_reader(); + RuntimeState no_stats_state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(no_stats_reader->init(&no_stats_state).ok()); + open_all_row_groups(no_stats_reader.get()); + format::FileAggregateRequest no_stats_request; + no_stats_request.agg_type = TPushAggOp::MINMAX; + no_stats_request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult no_stats_result; + EXPECT_FALSE(no_stats_reader->get_aggregate_result(no_stats_request, &no_stats_result).ok()); + + format::FileAggregateRequest invalid_type_request; + invalid_type_request.agg_type = TPushAggOp::MIX; + format::FileAggregateResult invalid_type_result; + EXPECT_FALSE( + no_stats_reader->get_aggregate_result(invalid_type_request, &invalid_type_result).ok()); + + format::FileAggregateRequest invalid_column_request; + invalid_column_request.agg_type = TPushAggOp::MINMAX; + invalid_column_request.columns.push_back({.projection = field_projection(100)}); + format::FileAggregateResult invalid_column_result; + EXPECT_FALSE( + no_stats_reader->get_aggregate_result(invalid_column_request, &invalid_column_result) + .ok()); +} + +TEST_F(ParquetScanTest, GlobalRowIdUsesFileLocalPositionForScanRange) { + write_int_pair_parquet_file(_file_path, 2); + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); + const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 1); + format::GlobalRowIdContext context {.version = 7, .backend_id = 123456789, .file_id = 42}; + auto reader = create_reader(range_start_offset, range_size, nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), + field_projection(format::GLOBAL_ROWID_COLUMN_ID)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector row_ids; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = int32_data_column(*block.get_by_position(0).column); + const auto& rowid_column = string_data_column(*block.get_by_position(2).column); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + const auto location = decode_rowid(rowid_column, row); + EXPECT_EQ(location.version, context.version); + EXPECT_EQ(location.backend_id, context.backend_id); + EXPECT_EQ(location.file_id, context.file_id); + row_ids.push_back(location.row_id); + } + } + + EXPECT_EQ(ids, std::vector({3, 4})); + EXPECT_EQ(row_ids, std::vector({2, 3})); +} + +TEST_F(ParquetScanTest, EmptyScanPlanReturnsEofWithoutReadingColumns) { + write_int_pair_parquet_file(_file_path, 2); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 100)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); +} + +TEST_F(ParquetScanTest, NoRequestedColumnsReturnsRowsOnlyAcrossRowGroups) { + write_int_pair_parquet_file(_file_path, 2); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block; + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(block.columns(), 0); + total_rows += rows; + } + EXPECT_EQ(total_rows, 6); +} + +TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 2)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(1, Int32ZoneMapExpr::Op::LT, 50)); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector scores; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = int32_data_column(*block.get_by_position(0).column); + const auto& score_column = int32_data_column(*block.get_by_position(1).column); + ASSERT_EQ(id_column.size(), rows); + ASSERT_EQ(score_column.size(), rows); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + scores.push_back(score_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({3, 4})); + EXPECT_EQ(scores, std::vector({30, 40})); + EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); + EXPECT_EQ(counter_value(profile, "SelectedRows"), 2); + EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 4); + EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 10); + EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 4); + EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 2); +} + +TEST_F(ParquetScanTest, PredicateColumnsSkipUnreadColumnsWhenFirstPredicateFiltersAll) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 100)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(1, Int32ZoneMapExpr::Op::LT, 50)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + + EXPECT_EQ(total_rows, 0); + EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); + EXPECT_EQ(counter_value(profile, "SelectedRows"), 0); + EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 6); + EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 1); + EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 6); + EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 0); + EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 6); +} + +TEST_F(ParquetScanTest, MultiColumnPredicateWaitsForAllPredicateColumns) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_pair_sum_conjunct(0, 1, 45)); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector scores; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = int32_data_column(*block.get_by_position(0).column); + const auto& score_column = int32_data_column(*block.get_by_position(1).column); + ASSERT_EQ(id_column.size(), rows); + ASSERT_EQ(score_column.size(), rows); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + scores.push_back(score_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({1, 2, 3, 4})); + EXPECT_EQ(scores, std::vector({10, 20, 30, 40})); + EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); + EXPECT_EQ(counter_value(profile, "SelectedRows"), 4); + EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 12); + EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 0); +} + +TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { + write_page_index_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + + EXPECT_EQ(total_rows, 64); + ASSERT_NE(profile.get_counter("RowGroupsTotalNum"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsReadNum"), nullptr); + ASSERT_NE(profile.get_counter("FilteredRowsByPage"), nullptr); + ASSERT_NE(profile.get_counter("SelectedRowRanges"), nullptr); + ASSERT_NE(profile.get_counter("PageIndexReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("RawRowsRead"), nullptr); + ASSERT_NE(profile.get_counter("RangeGapSkippedRows"), nullptr); + EXPECT_EQ(profile.get_counter("RowGroupsTotalNum")->value(), 1); + EXPECT_EQ(profile.get_counter("RowGroupsReadNum")->value(), 1); + EXPECT_GT(profile.get_counter("FilteredRowsByPage")->value(), 0); + EXPECT_GT(profile.get_counter("SelectedRowRanges")->value(), 0); + EXPECT_GT(profile.get_counter("PageIndexReadCalls")->value(), 0); + EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), 64); + EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); +} + +} // namespace +} // namespace doris diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp new file mode 100644 index 00000000000000..e620ed718efbf2 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -0,0 +1,527 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include + +#include "core/assert_cast.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/primitive_type.h" +#include "format_v2/parquet/parquet_column_schema.h" + +namespace doris::format::parquet { +namespace { + +std::vector> build_fields( + const std::vector<::parquet::schema::NodePtr>& nodes) { + auto schema = + ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); + ::parquet::SchemaDescriptor descriptor; + descriptor.Init(schema); + std::vector> fields; + EXPECT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); + return fields; +} + +Status build_status(const std::vector<::parquet::schema::NodePtr>& nodes) { + auto schema = + ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); + ::parquet::SchemaDescriptor descriptor; + descriptor.Init(schema); + std::vector> fields; + return build_parquet_column_schema(descriptor, &fields); +} + +} // namespace + +TEST(ParquetSchemaTest, PrimitiveStateAndFieldIdArePreserved) { + const auto fields = build_fields({ + ::parquet::schema::PrimitiveNode::Make("required_i32", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32), + ::parquet::schema::PrimitiveNode::Make("optional_i64", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT64, + ::parquet::ConvertedType::NONE, -1, -1, -1, 42), + }); + + ASSERT_EQ(fields.size(), 2); + EXPECT_EQ(fields[0]->local_id, 0); + EXPECT_EQ(fields[0]->name, "required_i32"); + EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); + EXPECT_EQ(fields[0]->leaf_column_id, 0); + EXPECT_EQ(fields[0]->nullable_definition_level, 0); + EXPECT_FALSE(fields[0]->type->is_nullable()); + + EXPECT_EQ(fields[1]->local_id, 1); + EXPECT_EQ(fields[1]->parquet_field_id, 42); + EXPECT_EQ(fields[1]->leaf_column_id, 1); + EXPECT_EQ(fields[1]->nullable_definition_level, 1); + EXPECT_TRUE(fields[1]->type->is_nullable()); +} + +TEST(ParquetSchemaTest, PrimitiveTypeDescriptorCoversLogicalConvertedAndPhysicalFallback) { + const auto fields = build_fields({ + ::parquet::schema::PrimitiveNode::Make( + "ts", ::parquet::Repetition::OPTIONAL, + ::parquet::LogicalType::Timestamp(false, + ::parquet::LogicalType::TimeUnit::MICROS), + ::parquet::Type::INT64), + ::parquet::schema::PrimitiveNode::Make("i8", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32, + ::parquet::ConvertedType::INT_8), + ::parquet::schema::PrimitiveNode::Make("plain", ::parquet::Repetition::REQUIRED, + ::parquet::Type::DOUBLE), + }); + + ASSERT_EQ(fields.size(), 3); + EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_EQ(fields[0]->type_descriptor.time_unit, ParquetTimeUnit::MICROS); + EXPECT_EQ(fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); + EXPECT_TRUE(fields[0]->type_descriptor.is_timestamp); + EXPECT_FALSE(fields[0]->type_descriptor.timestamp_is_adjusted_to_utc); + + EXPECT_EQ(remove_nullable(fields[1]->type)->get_primitive_type(), TYPE_TINYINT); + EXPECT_EQ(fields[1]->type_descriptor.integer_bit_width, 8); + EXPECT_FALSE(fields[1]->type_descriptor.is_unsigned_integer); + + EXPECT_EQ(remove_nullable(fields[2]->type)->get_primitive_type(), TYPE_DOUBLE); + EXPECT_EQ(fields[2]->type_descriptor.physical_type, ::parquet::Type::DOUBLE); + EXPECT_EQ(fields[2]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::NONE); +} + +TEST(ParquetSchemaTest, StructMakesDataTypeChildrenNullableAndPropagatesLevels) { + const auto fields = build_fields({::parquet::schema::GroupNode::Make( + "s", ::parquet::Repetition::OPTIONAL, + { + ::parquet::schema::PrimitiveNode::Make("a", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32), + ::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::BYTE_ARRAY, + ::parquet::ConvertedType::UTF8), + })}); + + ASSERT_EQ(fields.size(), 1); + const auto& struct_schema = *fields[0]; + EXPECT_EQ(struct_schema.kind, ParquetColumnSchemaKind::STRUCT); + EXPECT_EQ(struct_schema.nullable_definition_level, 1); + ASSERT_EQ(struct_schema.children.size(), 2); + EXPECT_EQ(struct_schema.children[0]->definition_level, 1); + EXPECT_EQ(struct_schema.children[1]->definition_level, 2); + EXPECT_EQ(struct_schema.max_definition_level, 2); + + const auto& struct_type = + assert_cast(*remove_nullable(struct_schema.type)); + ASSERT_EQ(struct_type.get_elements().size(), 2); + EXPECT_TRUE(struct_type.get_elements()[0]->is_nullable()); + EXPECT_TRUE(struct_type.get_elements()[1]->is_nullable()); +} + +TEST(ParquetSchemaTest, ListCompatibilityRulesAndLevels) { + const auto standard_list = ::parquet::schema::GroupNode::Make( + "xs", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "list", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32)})}, + ::parquet::ConvertedType::LIST); + const auto structural_array = ::parquet::schema::GroupNode::Make( + "ys", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "array", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, + ::parquet::ConvertedType::LIST); + + const auto fields = build_fields({standard_list, structural_array}); + ASSERT_EQ(fields.size(), 2); + + const auto& xs = *fields[0]; + EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); + EXPECT_EQ(xs.definition_level, 2); + EXPECT_EQ(xs.repetition_level, 1); + ASSERT_EQ(xs.children.size(), 1); + EXPECT_EQ(xs.children[0]->name, "element"); + EXPECT_EQ(xs.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); + EXPECT_TRUE(xs.children[0]->type->is_nullable()); + const auto& xs_type = assert_cast(*remove_nullable(xs.type)); + EXPECT_TRUE(xs_type.get_nested_type()->is_nullable()); + + const auto& ys = *fields[1]; + EXPECT_EQ(ys.kind, ParquetColumnSchemaKind::LIST); + ASSERT_EQ(ys.children.size(), 1); + EXPECT_EQ(ys.children[0]->kind, ParquetColumnSchemaKind::STRUCT); + EXPECT_EQ(remove_nullable(ys.children[0]->type)->get_primitive_type(), TYPE_STRUCT); +} + +TEST(ParquetSchemaTest, LegacyListElementResolutionRulesArePreserved) { + const auto two_level_list = ::parquet::schema::GroupNode::Make( + "two_level", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::REPEATED, + ::parquet::Type::INT32)}, + ::parquet::ConvertedType::LIST); + const auto tuple_list = ::parquet::schema::GroupNode::Make( + "tuple_list", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "tuple_list_tuple", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, + ::parquet::ConvertedType::LIST); + const auto multi_field_list = ::parquet::schema::GroupNode::Make( + "records", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "list", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32), + ::parquet::schema::PrimitiveNode::Make("name", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::BYTE_ARRAY, + ::parquet::ConvertedType::UTF8)})}, + ::parquet::ConvertedType::LIST); + const auto fields = build_fields({two_level_list, tuple_list, multi_field_list}); + ASSERT_EQ(fields.size(), 3); + + const auto& two_level = *fields[0]; + EXPECT_EQ(two_level.kind, ParquetColumnSchemaKind::LIST); + EXPECT_EQ(two_level.definition_level, 2); + EXPECT_EQ(two_level.repetition_level, 1); + ASSERT_EQ(two_level.children.size(), 1); + EXPECT_EQ(two_level.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); + EXPECT_EQ(two_level.children[0]->name, "element"); + EXPECT_EQ(remove_nullable(two_level.children[0]->type)->get_primitive_type(), TYPE_INT); + + const auto& tuple = *fields[1]; + ASSERT_EQ(tuple.children.size(), 1); + EXPECT_EQ(tuple.children[0]->kind, ParquetColumnSchemaKind::STRUCT); + EXPECT_EQ(tuple.children[0]->name, "element"); + ASSERT_EQ(tuple.children[0]->children.size(), 1); + EXPECT_EQ(tuple.children[0]->children[0]->name, "value"); + + const auto& multi_field = *fields[2]; + ASSERT_EQ(multi_field.children.size(), 1); + EXPECT_EQ(multi_field.children[0]->kind, ParquetColumnSchemaKind::STRUCT); + ASSERT_EQ(multi_field.children[0]->children.size(), 2); + EXPECT_EQ(multi_field.children[0]->children[0]->name, "id"); + EXPECT_EQ(multi_field.children[0]->children[1]->name, "name"); +} + +TEST(ParquetSchemaTest, NestedRepeatedInsideListElementIsWrappedOnce) { + const auto list_with_repeated_child = ::parquet::schema::GroupNode::Make( + "outer", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "list", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make( + "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})}, + ::parquet::ConvertedType::LIST); + + const auto fields = build_fields({list_with_repeated_child}); + ASSERT_EQ(fields.size(), 1); + const auto& outer = *fields[0]; + EXPECT_EQ(outer.kind, ParquetColumnSchemaKind::LIST); + ASSERT_EQ(outer.children.size(), 1); + const auto& element = *outer.children[0]; + EXPECT_EQ(element.kind, ParquetColumnSchemaKind::STRUCT); + ASSERT_EQ(element.children.size(), 1); + EXPECT_EQ(element.children[0]->kind, ParquetColumnSchemaKind::LIST); + EXPECT_EQ(element.children[0]->name, "items"); + ASSERT_EQ(element.children[0]->children.size(), 1); + EXPECT_EQ(element.children[0]->children[0]->name, "element"); +} + +TEST(ParquetSchemaTest, ListWrapperWithLogicalAnnotationIsPreservedAsElement) { + const auto annotated_repeated_group = ::parquet::schema::GroupNode::Make( + "xs", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "list", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)}, + ::parquet::ConvertedType::LIST)}, + ::parquet::ConvertedType::LIST); + + EXPECT_FALSE(build_status({annotated_repeated_group}).ok()); + + const auto nested_list_wrapper = ::parquet::schema::GroupNode::Make( + "xs", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "list", ::parquet::Repetition::REPEATED, + {::parquet::schema::GroupNode::Make( + "list", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make("value", + ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32)})}, + ::parquet::ConvertedType::LIST)}, + ::parquet::ConvertedType::LIST); + + const auto fields = build_fields({nested_list_wrapper}); + ASSERT_EQ(fields.size(), 1); + const auto& xs = *fields[0]; + EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); + ASSERT_EQ(xs.children.size(), 1); + const auto& element = *xs.children[0]; + EXPECT_EQ(element.kind, ParquetColumnSchemaKind::LIST); + EXPECT_EQ(element.name, "element"); + ASSERT_EQ(element.children.size(), 1); + EXPECT_EQ(element.children[0]->name, "element"); + EXPECT_EQ(remove_nullable(element.children[0]->type)->get_primitive_type(), TYPE_INT); +} + +TEST(ParquetSchemaTest, MapWrapperIsFoldedAndOptionalKeyIsAllowed) { + const auto fields = build_fields({::parquet::schema::GroupNode::Make( + "m", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "key_value", ::parquet::Repetition::REPEATED, + { + ::parquet::schema::PrimitiveNode::Make( + "key", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), + ::parquet::schema::PrimitiveNode::Make("value", + ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32), + })}, + ::parquet::ConvertedType::MAP)}); + + ASSERT_EQ(fields.size(), 1); + const auto& map_schema = *fields[0]; + EXPECT_EQ(map_schema.kind, ParquetColumnSchemaKind::MAP); + EXPECT_EQ(map_schema.definition_level, 2); + EXPECT_EQ(map_schema.repetition_level, 1); + ASSERT_EQ(map_schema.children.size(), 2); + EXPECT_EQ(map_schema.children[0]->name, "key"); + EXPECT_EQ(map_schema.children[1]->name, "value"); + EXPECT_TRUE(map_schema.children[0]->type->is_nullable()); + + const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); + EXPECT_TRUE(map_type.get_key_type()->is_nullable()); + EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +} + +TEST(ParquetSchemaTest, StandardMapLevelsAndDataTypesAreBuiltFromEntryContext) { + const auto fields = build_fields({::parquet::schema::GroupNode::Make( + "m", ::parquet::Repetition::REQUIRED, + {::parquet::schema::GroupNode::Make( + "key_value", ::parquet::Repetition::REPEATED, + { + ::parquet::schema::PrimitiveNode::Make( + "key", ::parquet::Repetition::REQUIRED, + ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), + ::parquet::schema::PrimitiveNode::Make("value", + ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32), + })}, + ::parquet::ConvertedType::MAP)}); + + ASSERT_EQ(fields.size(), 1); + const auto& map_schema = *fields[0]; + EXPECT_FALSE(map_schema.type->is_nullable()); + EXPECT_EQ(map_schema.definition_level, 1); + EXPECT_EQ(map_schema.repetition_level, 1); + EXPECT_EQ(map_schema.repeated_repetition_level, 1); + EXPECT_EQ(map_schema.max_definition_level, 2); + EXPECT_EQ(map_schema.max_repetition_level, 1); + ASSERT_EQ(map_schema.children.size(), 2); + EXPECT_EQ(map_schema.children[0]->definition_level, 1); + EXPECT_EQ(map_schema.children[0]->repetition_level, 1); + EXPECT_EQ(map_schema.children[1]->definition_level, 2); + EXPECT_EQ(map_schema.children[1]->nullable_definition_level, 2); + + const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); + EXPECT_TRUE(map_type.get_key_type()->is_nullable()); + EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +} + +TEST(ParquetSchemaTest, BareRepeatedFieldsAreWrappedAsLists) { + const auto fields = build_fields({ + ::parquet::schema::PrimitiveNode::Make("items", ::parquet::Repetition::REPEATED, + ::parquet::Type::INT32), + ::parquet::schema::GroupNode::Make( + "links", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make("url", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::BYTE_ARRAY, + ::parquet::ConvertedType::UTF8), + ::parquet::schema::PrimitiveNode::Make("rank", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32)}), + }); + + ASSERT_EQ(fields.size(), 2); + EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::LIST); + ASSERT_EQ(fields[0]->children.size(), 1); + EXPECT_EQ(fields[0]->children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); + EXPECT_EQ(fields[0]->children[0]->name, "element"); + + EXPECT_EQ(fields[1]->kind, ParquetColumnSchemaKind::LIST); + ASSERT_EQ(fields[1]->children.size(), 1); + EXPECT_EQ(fields[1]->children[0]->kind, ParquetColumnSchemaKind::STRUCT); + EXPECT_EQ(fields[1]->children[0]->name, "element"); +} + +TEST(ParquetSchemaTest, DeepLevelChainPropagatesDefinitionAndRepetitionLevels) { + const auto fields = build_fields({::parquet::schema::GroupNode::Make( + "s", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "inner", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::PrimitiveNode::Make( + "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})})}); + + ASSERT_EQ(fields.size(), 1); + const auto& s = *fields[0]; + EXPECT_EQ(s.definition_level, 1); + EXPECT_EQ(s.nullable_definition_level, 1); + ASSERT_EQ(s.children.size(), 1); + const auto& inner = *s.children[0]; + EXPECT_EQ(inner.definition_level, 2); + EXPECT_EQ(inner.nullable_definition_level, 2); + ASSERT_EQ(inner.children.size(), 1); + const auto& items = *inner.children[0]; + EXPECT_EQ(items.kind, ParquetColumnSchemaKind::LIST); + EXPECT_EQ(items.definition_level, 3); + EXPECT_EQ(items.repetition_level, 1); + EXPECT_EQ(items.repeated_ancestor_definition_level, 3); + EXPECT_EQ(items.repeated_repetition_level, 1); + EXPECT_EQ(items.max_definition_level, 3); + EXPECT_EQ(items.max_repetition_level, 1); + ASSERT_EQ(items.children.size(), 1); + EXPECT_EQ(items.children[0]->definition_level, 3); + EXPECT_EQ(items.children[0]->repetition_level, 1); +} + +TEST(ParquetSchemaTest, BuildEntryValidatesNullPointerAndEmptyRoot) { + auto empty_root = ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, + ::parquet::schema::NodeVector {}); + ::parquet::SchemaDescriptor descriptor; + descriptor.Init(empty_root); + + EXPECT_FALSE(build_parquet_column_schema(descriptor, nullptr).ok()); + + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); + EXPECT_TRUE(fields.empty()); +} + +TEST(ParquetSchemaTest, RejectInvalidListMapAndUnsupportedTime) { + const auto bad_list = ::parquet::schema::GroupNode::Make( + "bad_list", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32)}, + ::parquet::ConvertedType::LIST); + EXPECT_FALSE(build_status({bad_list}).ok()); + + const auto bad_map = ::parquet::schema::GroupNode::Make( + "bad_map", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::PrimitiveNode::Make("entry", ::parquet::Repetition::REPEATED, + ::parquet::Type::INT32)}, + ::parquet::ConvertedType::MAP); + EXPECT_FALSE(build_status({bad_map}).ok()); + + const auto converted_time = ::parquet::schema::PrimitiveNode::Make( + "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, + ::parquet::ConvertedType::TIME_MILLIS); + const auto status = build_status({converted_time}); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), + std::string::npos); +} + +TEST(ParquetSchemaTest, RejectAdditionalInvalidListAndMapLayouts) { + const auto zero_child_list = ::parquet::schema::GroupNode::Make( + "zero_child_list", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make("list", ::parquet::Repetition::REPEATED, + ::parquet::schema::NodeVector {})}, + ::parquet::ConvertedType::LIST); + EXPECT_FALSE(build_status({zero_child_list}).ok()); + + const auto repeated_list = ::parquet::schema::GroupNode::Make( + "repeated_list", ::parquet::Repetition::REPEATED, + {::parquet::schema::GroupNode::Make( + "list", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32)})}, + ::parquet::ConvertedType::LIST); + EXPECT_FALSE(build_status({repeated_list}).ok()); + + const auto map_with_two_fields = ::parquet::schema::GroupNode::Make( + "bad_map", ::parquet::Repetition::OPTIONAL, + { + ::parquet::schema::GroupNode::Make( + "entry1", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make( + "key", ::parquet::Repetition::REQUIRED, + ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), + ::parquet::schema::PrimitiveNode::Make("value", + ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32)}), + ::parquet::schema::GroupNode::Make( + "entry2", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make( + "key", ::parquet::Repetition::REQUIRED, + ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), + ::parquet::schema::PrimitiveNode::Make("value", + ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32)}), + }, + ::parquet::ConvertedType::MAP); + EXPECT_FALSE(build_status({map_with_two_fields}).ok()); + + const auto non_repeated_map_entry = ::parquet::schema::GroupNode::Make( + "bad_map", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "key_value", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, + ::parquet::Type::BYTE_ARRAY, + ::parquet::ConvertedType::UTF8), + ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, + ::parquet::ConvertedType::MAP); + EXPECT_FALSE(build_status({non_repeated_map_entry}).ok()); + + const auto map_entry_with_one_child = ::parquet::schema::GroupNode::Make( + "bad_map", ::parquet::Repetition::OPTIONAL, + {::parquet::schema::GroupNode::Make( + "key_value", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, + ::parquet::Type::BYTE_ARRAY, + ::parquet::ConvertedType::UTF8)})}, + ::parquet::ConvertedType::MAP); + EXPECT_FALSE(build_status({map_entry_with_one_child}).ok()); + + const auto repeated_map = ::parquet::schema::GroupNode::Make( + "repeated_map", ::parquet::Repetition::REPEATED, + {::parquet::schema::GroupNode::Make( + "key_value", ::parquet::Repetition::REPEATED, + {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, + ::parquet::Type::BYTE_ARRAY, + ::parquet::ConvertedType::UTF8), + ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, + ::parquet::ConvertedType::MAP); + EXPECT_FALSE(build_status({repeated_map}).ok()); +} + +TEST(ParquetSchemaTest, LogicalUtcTimeIsRejected) { + const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( + "time_ms", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32); + const auto status = build_status({adjusted_time}); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), + std::string::npos); +} + +} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp b/be/test/format_v2/parquet/parquet_serde_reader_test.cpp new file mode 100644 index 00000000000000..c35138e3263723 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_serde_reader_test.cpp @@ -0,0 +1,459 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_decimal.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/types.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/reader/column_reader.h" + +namespace doris::format::parquet { +namespace { + +constexpr int64_t ROW_COUNT = 5; + +std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { + std::shared_ptr array; + EXPECT_TRUE(builder->Finish(&array).ok()); + return array; +} + +class ParquetSerdeReaderTest : public testing::Test { +protected: + void SetUp() override { + _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_serde_reader_test"; + std::filesystem::remove_all(_test_dir); + std::filesystem::create_directories(_test_dir); + _file_path = (_test_dir / "serde.parquet").string(); + write_parquet_file(); + open_file(_file_path); + } + + void TearDown() override { std::filesystem::remove_all(_test_dir); } + + template + std::shared_ptr build_required_array(const std::vector& values) { + Builder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_nullable_int32_array() { + arrow::Int32Builder builder; + EXPECT_TRUE(builder.Append(1).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.Append(3).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.Append(5).ok()); + return finish_array(&builder); + } + + std::shared_ptr build_nullable_float16_array() { + arrow::HalfFloatBuilder builder; + EXPECT_TRUE(builder.AppendNull().ok()); + EXPECT_TRUE(builder.Append(0x0000).ok()); + EXPECT_TRUE(builder.Append(0x8000).ok()); + EXPECT_TRUE(builder.Append(0x3E00).ok()); + EXPECT_TRUE(builder.Append(0x7E00).ok()); + return finish_array(&builder); + } + + std::shared_ptr build_binary_array(const std::vector& values) { + arrow::BinaryBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data()), + static_cast(value.size())) + .ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_string_array(const std::vector& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_fixed_binary_array( + const std::shared_ptr& type, const std::vector& values) { + arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_timestamp_array( + const std::shared_ptr& type, const std::vector& values) { + arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); + } + + std::shared_ptr build_decimal_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); + } + return finish_array(&builder); + } + + void add_field(const std::shared_ptr& field, + std::shared_ptr array) { + _arrow_fields.push_back(field); + _arrays.push_back(std::move(array)); + } + + void write_table(const std::string& file_path, const std::shared_ptr& table, + std::shared_ptr<::parquet::ArrowWriterProperties> arrow_properties = nullptr) { + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + if (arrow_properties == nullptr) { + ::parquet::ArrowWriterProperties::Builder arrow_builder; + arrow_properties = arrow_builder.build(); + } + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( + *table, arrow::default_memory_pool(), *file_result, ROW_COUNT, + writer_builder.build(), std::move(arrow_properties))); + } + + void write_parquet_file() { + add_field(arrow::field("bool_col", arrow::boolean(), false), + build_required_array( + {true, false, true, false, true})); + add_field(arrow::field("int32_col", arrow::int32(), false), + build_required_array({10, 20, 30, 40, 50})); + add_field(arrow::field("int64_col", arrow::int64(), false), + build_required_array( + {10000000000L, -9L, 42L, 77L, 123L})); + add_field(arrow::field("uint32_col", arrow::uint32(), false), + build_required_array( + {0U, 1U, 1U << 31, std::numeric_limits::max(), 42U})); + add_field(arrow::field("uint64_col", arrow::uint64(), false), + build_required_array( + {0ULL, 1ULL, 1ULL << 63, std::numeric_limits::max(), 42ULL})); + add_field(arrow::field("float_col", arrow::float32(), false), + build_required_array( + {1.5F, -2.25F, 3.0F, 4.5F, 5.75F})); + add_field(arrow::field("double_col", arrow::float64(), false), + build_required_array({3.5, -4.75, 6.0, 7.25, 8.5})); + add_field(arrow::field("nullable_float16_col", arrow::float16(), true), + build_nullable_float16_array()); + add_field(arrow::field("binary_col", arrow::binary(), false), + build_binary_array({"bin_a", "bin_b", "bin_c", "bin_d", "bin_e"})); + add_field(arrow::field("string_col", arrow::utf8(), false), + build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"})); + add_field(arrow::field("fixed_binary_col", arrow::fixed_size_binary(4), false), + build_fixed_binary_array(arrow::fixed_size_binary(4), + {"aaaa", "bbbb", "cccc", "dddd", "eeee"})); + add_field(arrow::field("date_col", arrow::date32(), false), + build_required_array({0, 1, 18628, 18629, 18630})); + add_field(arrow::field("timestamp_millis_col", arrow::timestamp(arrow::TimeUnit::MILLI), + false), + build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MILLI), + {0, 1234, 1609459200000, 1609459201000, -1})); + add_field(arrow::field("timestamp_micros_col", arrow::timestamp(arrow::TimeUnit::MICRO), + false), + build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), + {0, 1234567, 1609459200000000, 1609459201000000, -1})); + add_field(arrow::field("timestamp_micros_utc_col", + arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), + build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + {0, 1234567, 1609459200000000, 1609459201000000, -1})); + add_field(arrow::field("decimal_fixed_binary_9_2_col", arrow::decimal128(9, 2), false), + build_decimal_array(arrow::decimal128(9, 2), {12345, -67, 0, 987, 1000})); + add_field(arrow::field("decimal_fixed_binary_18_6_col", arrow::decimal128(18, 6), false), + build_decimal_array(arrow::decimal128(18, 6), + {1234567, -670000, 0, 9870000, 1000000})); + add_field(arrow::field("nullable_int_col", arrow::int32(), true), + build_nullable_int32_array()); + + write_table(_file_path, arrow::Table::Make(arrow::schema(_arrow_fields), _arrays)); + } + + void open_file(const std::string& file_path) { + _file_reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); + ASSERT_NE(_file_reader, nullptr); + ASSERT_EQ(_file_reader->metadata()->num_row_groups(), 1); + _row_group = _file_reader->RowGroup(0); + ASSERT_NE(_row_group, nullptr); + auto schema_descriptor = _file_reader->metadata()->schema(); + ASSERT_NE(schema_descriptor, nullptr); + auto st = build_parquet_column_schema(*schema_descriptor, &_fields); + ASSERT_TRUE(st.ok()) << st; + } + + size_t find_field_idx(const std::string& name) const { + for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { + if (_fields[field_idx]->name == name) { + return field_idx; + } + } + ADD_FAILURE() << "Cannot find parquet serde test field " << name; + return _fields.size(); + } + + std::unique_ptr create_reader(size_t field_idx) const { + ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); + std::unique_ptr reader; + auto st = factory.create(*_fields[field_idx], &reader); + EXPECT_TRUE(st.ok()) << st; + return reader; + } + + template + void read_and_validate(const std::string& name, Validator validator) const { + const auto field_idx = find_field_idx(name); + ASSERT_TRUE(supports_record_reader(_fields[field_idx]->type_descriptor)); + auto reader = create_reader(field_idx); + ASSERT_NE(reader, nullptr); + MutableColumnPtr column = reader->type()->create_column(); + int64_t rows_read = 0; + auto st = reader->read(ROW_COUNT, column, &rows_read); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(rows_read, ROW_COUNT); + ASSERT_EQ(column->size(), ROW_COUNT); + validator(*_fields[field_idx], *column); + } + + std::filesystem::path _test_dir; + std::string _file_path; + std::unique_ptr<::parquet::ParquetFileReader> _file_reader; + std::shared_ptr<::parquet::RowGroupReader> _row_group; + std::vector> _fields; + std::vector> _arrow_fields; + std::vector> _arrays; +}; + +TEST_F(ParquetSerdeReaderTest, ReadAllSupportedPhysicalAndLogicalTypes) { + read_and_validate("bool_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BOOLEAN); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_element(0), 1); + EXPECT_EQ(values.get_element(1), 0); + EXPECT_EQ(values.get_element(4), 1); + }); + read_and_validate("int32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_element(0), 10); + EXPECT_EQ(values.get_element(4), 50); + }); + read_and_validate("int64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_element(0), 10000000000L); + EXPECT_EQ(values.get_element(1), -9L); + }); + read_and_validate("uint32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); + EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); + EXPECT_EQ(schema.type_descriptor.integer_bit_width, 32); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_BIGINT); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_element(2), 2147483648L); + EXPECT_EQ(values.get_element(3), + static_cast(std::numeric_limits::max())); + }); + read_and_validate("uint64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); + EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); + EXPECT_EQ(schema.type_descriptor.integer_bit_width, 64); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_LARGEINT); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_element(2), static_cast(1) << 63); + EXPECT_EQ(values.get_element(3), + static_cast(std::numeric_limits::max())); + }); + read_and_validate("float_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FLOAT); + const auto& values = assert_cast(column); + EXPECT_FLOAT_EQ(values.get_element(0), 1.5F); + EXPECT_FLOAT_EQ(values.get_element(1), -2.25F); + }); + read_and_validate("double_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::DOUBLE); + const auto& values = assert_cast(column); + EXPECT_DOUBLE_EQ(values.get_element(0), 3.5); + EXPECT_DOUBLE_EQ(values.get_element(1), -4.75); + }); + read_and_validate("nullable_float16_col", [](const ParquetColumnSchema& schema, + const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); + EXPECT_EQ(schema.type_descriptor.fixed_length, 2); + EXPECT_EQ(schema.type_descriptor.extra_type_info, ParquetExtraTypeInfo::FLOAT16); + EXPECT_FALSE(schema.type_descriptor.is_string_like); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_FLOAT); + const auto& nullable_column = assert_cast(column); + const auto& values = assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_TRUE(nullable_column.is_null_at(0)); + EXPECT_FLOAT_EQ(values.get_element(1), 0.0F); + EXPECT_FALSE(std::signbit(values.get_element(1))); + EXPECT_FLOAT_EQ(values.get_element(2), -0.0F); + EXPECT_TRUE(std::signbit(values.get_element(2))); + EXPECT_FLOAT_EQ(values.get_element(3), 1.5F); + EXPECT_TRUE(std::isnan(values.get_element(4))); + }); + read_and_validate("binary_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BYTE_ARRAY); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_data_at(0).to_string(), "bin_a"); + EXPECT_EQ(values.get_data_at(3).to_string(), "bin_d"); + }); + read_and_validate("string_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type_descriptor.is_string_like); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); + EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); + }); + read_and_validate("fixed_binary_col", [](const ParquetColumnSchema& schema, + const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); + EXPECT_EQ(schema.type_descriptor.fixed_length, 4); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_data_at(0).to_string(), "aaaa"); + EXPECT_EQ(values.get_data_at(2).to_string(), "cccc"); + }); + read_and_validate("date_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATEV2); + EXPECT_EQ(schema.type->to_string(column, 0), "1970-01-01"); + EXPECT_EQ(schema.type->to_string(column, 2), "2021-01-01"); + }); + read_and_validate( + "timestamp_millis_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234"); + EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999"); + }); + read_and_validate( + "timestamp_micros_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); + EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); + }); + read_and_validate("timestamp_micros_utc_col", [](const ParquetColumnSchema& schema, + const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); + EXPECT_TRUE(schema.type_descriptor.timestamp_is_adjusted_to_utc); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); + EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); + }); + read_and_validate("decimal_fixed_binary_9_2_col", [](const ParquetColumnSchema& schema, + const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); + EXPECT_TRUE(schema.type_descriptor.is_decimal); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL32); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_element(0), Decimal32(12345)); + EXPECT_EQ(schema.type->to_string(column, 0), "123.45"); + }); + read_and_validate("decimal_fixed_binary_18_6_col", [](const ParquetColumnSchema& schema, + const IColumn& column) { + EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); + EXPECT_TRUE(schema.type_descriptor.is_decimal); + EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL64); + const auto& values = assert_cast(column); + EXPECT_EQ(values.get_element(0), Decimal64(1234567)); + EXPECT_EQ(schema.type->to_string(column, 0), "1.234567"); + }); + read_and_validate( + "nullable_int_col", [](const ParquetColumnSchema& schema, const IColumn& column) { + EXPECT_TRUE(schema.type->is_nullable()); + const auto& nullable_column = assert_cast(column); + const auto& nested_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(nullable_column.size(), ROW_COUNT); + EXPECT_FALSE(nullable_column.is_null_at(0)); + EXPECT_TRUE(nullable_column.is_null_at(1)); + EXPECT_FALSE(nullable_column.is_null_at(2)); + EXPECT_TRUE(nullable_column.is_null_at(3)); + EXPECT_EQ(nested_column.get_element(0), 1); + EXPECT_EQ(nested_column.get_element(2), 3); + }); +} + +TEST_F(ParquetSerdeReaderTest, ReadInt96TimestampAsDateTimeV2) { + const auto file_path = (_test_dir / "int96_timestamp.parquet").string(); + auto field = arrow::field("col_datetime", arrow::timestamp(arrow::TimeUnit::MICRO), false); + auto array = build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), + {0, 1234567, 1609459200000000, 1609459201000000, -1}); + auto table = arrow::Table::Make(arrow::schema({field}), {array}); + + ::parquet::ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_force_write_int96_timestamps(); + _fields.clear(); + _file_reader.reset(); + _row_group.reset(); + write_table(file_path, table, arrow_builder.build()); + open_file(file_path); + + ASSERT_EQ(_fields.size(), 1); + EXPECT_EQ(_fields[0]->type_descriptor.physical_type, ::parquet::Type::INT96); + EXPECT_EQ(_fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); + ASSERT_TRUE(supports_record_reader(_fields[0]->type_descriptor)); + ASSERT_EQ(remove_nullable(_fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); + + auto reader = create_reader(0); + ASSERT_NE(reader, nullptr); + auto column = _fields[0]->type->create_column(); + int64_t rows_read = 0; + ASSERT_TRUE(reader->read(ROW_COUNT, column, &rows_read).ok()); + ASSERT_EQ(rows_read, ROW_COUNT); + EXPECT_EQ(_fields[0]->type->to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(_fields[0]->type->to_string(*column, 1), "1970-01-01 00:00:01.234567"); + EXPECT_EQ(_fields[0]->type->to_string(*column, 2), "2021-01-01 00:00:00.000000"); + EXPECT_EQ(_fields[0]->type->to_string(*column, 4), "1969-12-31 23:59:59.999999"); +} + +} // namespace +} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp new file mode 100644 index 00000000000000..dd2138279b11d1 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -0,0 +1,1036 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_statistics.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/field.h" +#include "exprs/expr_zonemap_filter.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "storage/index/bloom_filter/block_split_bloom_filter.h" +#include "storage/index/zone_map/zonemap_eval_context.h" +#include "storage/index/zone_map/zonemap_filter_result.h" + +namespace doris { +namespace { + +std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { + std::shared_ptr array; + EXPECT_TRUE(builder->Finish(&array).ok()); + return array; +} + +std::shared_ptr int32_array(const std::vector>& values) { + arrow::Int32Builder builder; + for (const auto& value : values) { + if (value.has_value()) { + EXPECT_TRUE(builder.Append(*value).ok()); + } else { + EXPECT_TRUE(builder.AppendNull().ok()); + } + } + return finish_array(&builder); +} + +std::shared_ptr uint32_array(const std::vector& values) { + arrow::UInt32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr float_array(const std::vector& values) { + arrow::FloatBuilder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr double_array(const std::vector& values) { + arrow::DoubleBuilder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +template +std::string encoded_value(const NativeType& value) { + return {reinterpret_cast(&value), sizeof(value)}; +} + +std::shared_ptr string_array(const std::vector& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr timestamp_array(const std::vector& values) { + arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::unique_ptr<::parquet::ParquetFileReader> make_reader( + const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary, + bool enable_statistics) { + auto out_result = arrow::io::BufferOutputStream::Create(); + EXPECT_TRUE(out_result.ok()); + auto out = *out_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.compression(::parquet::Compression::UNCOMPRESSED); + if (enable_dictionary) { + builder.enable_dictionary(); + } else { + builder.disable_dictionary(); + } + if (!enable_statistics) { + builder.disable_statistics(); + } + EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + row_group_size, builder.build()) + .ok()); + auto buffer_result = out->Finish(); + EXPECT_TRUE(buffer_result.ok()); + return ::parquet::ParquetFileReader::Open( + std::make_shared(*buffer_result)); +} + +std::vector> build_file_schema( + const ::parquet::ParquetFileReader& reader) { + std::vector> file_schema; + EXPECT_TRUE( + format::parquet::build_parquet_column_schema(*reader.metadata()->schema(), &file_schema) + .ok()); + return file_schema; +} + +template +class TestColumnIndex final : public ::parquet::TypedColumnIndex { +public: + using NativeType = typename ParquetDType::c_type; + + TestColumnIndex(NativeType min_value, NativeType max_value) + : TestColumnIndex(std::vector {min_value}, + std::vector {max_value}) {} + + TestColumnIndex(std::vector min_values, std::vector max_values) + : _null_pages(min_values.size(), false), + _null_counts(min_values.size(), 0), + _min_values(std::move(min_values)), + _max_values(std::move(max_values)) { + EXPECT_EQ(_min_values.size(), _max_values.size()); + for (size_t page_idx = 0; page_idx < _min_values.size(); ++page_idx) { + _non_null_page_indices.push_back(static_cast(page_idx)); + } + } + + const std::vector& null_pages() const override { return _null_pages; } + const std::vector& encoded_min_values() const override { return _encoded_values; } + const std::vector& encoded_max_values() const override { return _encoded_values; } + ::parquet::BoundaryOrder::type boundary_order() const override { + return ::parquet::BoundaryOrder::Unordered; + } + bool has_null_counts() const override { return true; } + const std::vector& null_counts() const override { return _null_counts; } + const std::vector& non_null_page_indices() const override { + return _non_null_page_indices; + } + const std::vector& min_values() const override { return _min_values; } + const std::vector& max_values() const override { return _max_values; } + +private: + const std::vector _null_pages; + const std::vector _encoded_values; + const std::vector _null_counts; + std::vector _non_null_page_indices; + const std::vector _min_values; + const std::vector _max_values; +}; + +class Int32ZoneMapExpr final : public VExpr { +public: + enum class Op { GE, GT, IS_NULL, IS_NOT_NULL }; + + Int32ZoneMapExpr(int column_id, Op op, int32_t value = 0) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _op(op), + _value(value) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError("Int32ZoneMapExpr is only used by parquet statistics tests"); + } + + bool can_evaluate_zonemap_filter() const override { return true; } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + auto zone_map = ctx.zone_map(_column_id); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (_op == Op::IS_NULL) { + return zone_map->has_null ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; + } + if (_op == Op::IS_NOT_NULL) { + return zone_map->has_not_null ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + const auto literal = Field::create_field(_value); + if (_op == Op::GE) { + return zone_map->max_value < literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + return zone_map->max_value <= literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + +private: + int _column_id; + Op _op; + int32_t _value; + const std::string _expr_name = "Int32ZoneMapExpr"; +}; + +class StringDictionaryInExpr final : public VExpr { +public: + StringDictionaryInExpr(int column_id, std::vector values) + : VExpr(std::make_shared(), false), + _slot(VSlotRef::create_shared(0, column_id, -1, std::make_shared(), + "c0")) { + _values.reserve(values.size()); + for (auto& value : values) { + _values.emplace_back(Field::create_field(std::move(value))); + } + } + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError( + "StringDictionaryInExpr is only used by parquet statistics tests"); + } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + return expr_zonemap::eval_in_dictionary(ctx, _slot, false, _values); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + _slot->collect_slot_column_ids(column_ids); + } + +private: + VExprSPtr _slot; + std::vector _values; + const std::string _expr_name = "StringDictionaryInExpr"; +}; + +class BloomInExpr final : public VExpr { +public: + BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) + : VExpr(std::make_shared(), false), + _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), + _values(std::move(values)) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); + } + + bool can_evaluate_bloom_filter() const override { return true; } + + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { + return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + _slot->collect_slot_column_ids(column_ids); + } + +private: + VExprSPtr _slot; + std::vector _values; + const std::string _expr_name = "BloomInExpr"; +}; + +format::FileScanRequest request_with_zonemap_conjunct(std::shared_ptr expr) { + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts.push_back(VExprContext::create_shared(std::move(expr))); + return request; +} + +format::FileScanRequest request_with_dictionary_conjunct(std::vector values) { + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, std::move(values)))); + return request; +} + +VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type, std::vector values) { + return {VExprContext::create_shared( + std::make_shared(0, std::move(data_type), std::move(values)))}; +} + +format::FileScanRequest request_with_bloom_conjunct(DataTypePtr data_type, + std::vector values) { + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.conjuncts = bloom_conjuncts(std::move(data_type), std::move(values)); + return request; +} + +void add_bloom_field(segment_v2::BlockSplitBloomFilter* bloom_filter, const Field& value, + PrimitiveType type) { + DORIS_CHECK(bloom_filter != nullptr); + switch (type) { + case TYPE_BOOLEAN: { + const bool typed_value = value.get(); + bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); + break; + } + case TYPE_INT: { + const int32_t typed_value = value.get(); + bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); + break; + } + case TYPE_STRING: { + const auto& typed_value = value.get(); + bloom_filter->add_bytes(typed_value.data(), typed_value.size()); + break; + } + default: + DORIS_CHECK(false); + } +} + +std::unique_ptr bloom_filter_for_fields( + const std::vector& values, PrimitiveType type) { + auto bloom_filter = std::make_unique(); + EXPECT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok()); + for (const auto& value : values) { + add_bloom_field(bloom_filter.get(), value, type); + } + return bloom_filter; +} + +BloomFilterEvalContext bloom_context(const DataTypePtr& data_type, + const segment_v2::BloomFilter* bloom_filter) { + BloomFilterEvalContext ctx; + ctx.slots.emplace(0, BloomFilterEvalContext::SlotBloomFilter {.data_type = data_type, + .bloom_filter = bloom_filter}); + return ctx; +} + +std::unique_ptr<::parquet::BlockSplitBloomFilter> parquet_bloom_filter() { + auto bloom_filter = std::make_unique<::parquet::BlockSplitBloomFilter>(); + bloom_filter->Init(::parquet::BlockSplitBloomFilter::kMinimumBloomFilterBytes); + return bloom_filter; +} + +format::parquet::ParquetColumnSchema uint32_parquet_bloom_schema() { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = ::parquet::Type::INT32; + column_schema.type_descriptor.integer_bit_width = 32; + column_schema.type_descriptor.is_unsigned_integer = true; + return column_schema; +} + +format::parquet::ParquetColumnSchema fixed_len_string_parquet_bloom_schema(int fixed_length) { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + column_schema.type_descriptor.fixed_length = fixed_length; + column_schema.type_descriptor.is_string_like = true; + return column_schema; +} + +format::parquet::ParquetColumnSchema float16_parquet_bloom_schema() { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + column_schema.type_descriptor.fixed_length = 2; + column_schema.type_descriptor.extra_type_info = format::parquet::ParquetExtraTypeInfo::FLOAT16; + return column_schema; +} + +TEST(ParquetStatisticsTransformTest, ConvertsMinMaxNullCountUnsignedStringAndTimestamp) { + auto table = arrow::Table::Make( + arrow::schema({ + arrow::field("i", arrow::int32(), true), + arrow::field("u", arrow::uint32(), false), + arrow::field("s", arrow::utf8(), false), + arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), + }), + {int32_array({1, std::nullopt, 5}), uint32_array({7, 9, 11}), + string_array({"alpha", "beta", "omega"}), timestamp_array({1000, 2000, 3000})}); + auto reader = make_reader(table, 3, false, true); + auto schema = build_file_schema(*reader); + auto row_group = reader->metadata()->RowGroup(0); + + const auto int_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], row_group->ColumnChunk(0)->statistics()); + EXPECT_TRUE(int_stats.has_min_max); + EXPECT_TRUE(int_stats.has_null_count); + EXPECT_TRUE(int_stats.has_null); + EXPECT_TRUE(int_stats.has_not_null); + EXPECT_EQ(int_stats.min_value.get(), 1); + EXPECT_EQ(int_stats.max_value.get(), 5); + + const auto uint_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[1], row_group->ColumnChunk(1)->statistics()); + EXPECT_TRUE(uint_stats.has_min_max); + EXPECT_EQ(uint_stats.min_value.get(), 7); + EXPECT_EQ(uint_stats.max_value.get(), 11); + + const auto string_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[2], row_group->ColumnChunk(2)->statistics()); + EXPECT_TRUE(string_stats.has_min_max); + EXPECT_EQ(string_stats.min_value.get(), "alpha"); + EXPECT_EQ(string_stats.max_value.get(), "omega"); + + auto utc = cctz::utc_time_zone(); + const auto timestamp_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[3], row_group->ColumnChunk(3)->statistics(), &utc); + EXPECT_TRUE(timestamp_stats.has_min_max); + EXPECT_EQ(timestamp_stats.min_value.get_type(), TYPE_DATETIMEV2); + EXPECT_EQ(timestamp_stats.max_value.get_type(), TYPE_DATETIMEV2); + EXPECT_LT(timestamp_stats.min_value, timestamp_stats.max_value); +} + +TEST(ParquetStatisticsTransformTest, DisablesUtcTimestampMinMaxAcrossDstRollback) { + constexpr int64_t MICROS_PER_SECOND = 1000000; + // America/New_York moved from UTC-04:00 to UTC-05:00 at 2021-11-07 06:00:00 UTC. + // Both UTC endpoints below map to 01:30 local time, while values inside the interval cover + // 01:00 through 01:59. Endpoint conversion therefore cannot represent the true local range. + auto table = arrow::Table::Make( + arrow::schema( + {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), + {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636263900 * MICROS_PER_SECOND, + 1636266600 * MICROS_PER_SECOND})}); + auto reader = make_reader(table, 3, false, true); + auto schema = build_file_schema(*reader); + auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); + + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + const auto local_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], statistics, &new_york); + EXPECT_TRUE(local_stats.has_not_null); + EXPECT_FALSE(local_stats.has_min_max); + + auto utc = cctz::utc_time_zone(); + const auto utc_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], statistics, &utc); + EXPECT_TRUE(utc_stats.has_min_max); + EXPECT_LT(utc_stats.min_value, utc_stats.max_value); +} + +TEST(ParquetStatisticsTransformTest, KeepsTimestampTzMinMaxAcrossDstRollback) { + constexpr int64_t MICROS_PER_SECOND = 1000000; + auto table = arrow::Table::Make( + arrow::schema( + {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), + {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636266600 * MICROS_PER_SECOND})}); + auto reader = make_reader(table, 2, false, true); + auto schema = build_file_schema(*reader); + auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); + + // This is the effective type produced by enable_mapping_timestamp_tz. The physical timestamp + // flags intentionally remain adjusted-to-UTC so decoding can preserve the source semantics. + schema[0]->type = std::make_shared(6); + schema[0]->type_descriptor.doris_type = schema[0]->type; + + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + const auto timestamp_tz_stats = + format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], statistics, &new_york); + EXPECT_TRUE(timestamp_tz_stats.has_min_max); + EXPECT_EQ(timestamp_tz_stats.min_value.get_type(), TYPE_TIMESTAMPTZ); + EXPECT_EQ(timestamp_tz_stats.max_value.get_type(), TYPE_TIMESTAMPTZ); + EXPECT_LT(timestamp_tz_stats.min_value, timestamp_tz_stats.max_value); +} + +TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { + auto no_stats_table = arrow::Table::Make( + arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({1, 2, 3})}); + auto no_stats_reader = make_reader(no_stats_table, 3, false, false); + auto no_stats_schema = build_file_schema(*no_stats_reader); + auto no_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *no_stats_schema[0], + no_stats_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); + EXPECT_FALSE(no_stats.has_min_max); + + auto all_null_table = + arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), + {int32_array({std::nullopt, std::nullopt})}); + auto all_null_reader = make_reader(all_null_table, 2, false, true); + auto all_null_schema = build_file_schema(*all_null_reader); + auto all_null_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *all_null_schema[0], + all_null_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); + EXPECT_TRUE(all_null_stats.has_null_count); + EXPECT_TRUE(all_null_stats.has_null); + EXPECT_FALSE(all_null_stats.has_not_null); + EXPECT_FALSE(all_null_stats.has_min_max); +} + +TEST(ParquetStatisticsTransformTest, MissingNullCountConservativelyReportsPossibleNulls) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), + {int32_array({1, std::nullopt, 3})}); + auto reader = make_reader(table, 3, false, true); + auto schema = build_file_schema(*reader); + auto file_statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); + auto statistics_without_null_count = ::parquet::MakeStatistics<::parquet::Int32Type>( + reader->metadata()->schema()->Column(0), file_statistics->EncodeMin(), + file_statistics->EncodeMax(), file_statistics->num_values(), 0, 0, true, false, false); + + const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], statistics_without_null_count); + EXPECT_FALSE(statistics.has_null_count); + EXPECT_TRUE(statistics.has_null); + EXPECT_TRUE(statistics.has_not_null); + EXPECT_TRUE(statistics.has_min_max); + EXPECT_EQ(statistics.min_value.get(), 1); + EXPECT_EQ(statistics.max_value.get(), 3); +} + +TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleMinMax) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), + arrow::field("d", arrow::float64(), false)}), + {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); + auto reader = make_reader(table, 2, false, true); + auto schema = build_file_schema(*reader); + + const float float_nan = std::numeric_limits::quiet_NaN(); + const float float_max = 2.0F; + auto float_stats = ::parquet::MakeStatistics<::parquet::FloatType>( + schema[0]->descriptor, encoded_value(float_nan), encoded_value(float_max), 2, 0, 0, + true, true, false); + const auto converted_float = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], float_stats); + EXPECT_FALSE(converted_float.has_min_max); + EXPECT_TRUE(converted_float.has_not_null); + + const double double_nan = std::numeric_limits::quiet_NaN(); + const double double_min = 1.0; + auto double_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( + schema[1]->descriptor, encoded_value(double_min), encoded_value(double_nan), 2, 0, 0, + true, true, false); + const auto converted_double = + format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], + double_stats); + EXPECT_FALSE(converted_double.has_min_max); + EXPECT_TRUE(converted_double.has_not_null); + + const double double_max = 2.0; + auto finite_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( + schema[1]->descriptor, encoded_value(double_min), encoded_value(double_max), 2, 0, 0, + true, true, false); + const auto converted_finite = + format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], + finite_stats); + EXPECT_TRUE(converted_finite.has_min_max); +} + +TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleColumnIndexMinMax) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), + arrow::field("d", arrow::float64(), false)}), + {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); + auto reader = make_reader(table, 2, false, true); + auto schema = build_file_schema(*reader); + + auto float_index = std::make_shared>( + 1.0F, std::numeric_limits::quiet_NaN()); + format::parquet::ParquetColumnStatistics float_page_stats; + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( + float_index, *schema[0], 0, &float_page_stats)); + EXPECT_FALSE(float_page_stats.has_min_max); + EXPECT_TRUE(float_page_stats.has_not_null); + + auto double_index = std::make_shared>( + std::numeric_limits::quiet_NaN(), 2.0); + format::parquet::ParquetColumnStatistics double_page_stats; + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( + double_index, *schema[1], 0, &double_page_stats)); + EXPECT_FALSE(double_page_stats.has_min_max); + EXPECT_TRUE(double_page_stats.has_not_null); + + auto finite_index = std::make_shared>(1.0, 2.0); + format::parquet::ParquetColumnStatistics finite_page_stats; + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( + finite_index, *schema[1], 0, &finite_page_stats)); + EXPECT_TRUE(finite_page_stats.has_min_max); + + auto mixed_index = std::make_shared>( + std::vector {std::numeric_limits::quiet_NaN(), 1.0}, + std::vector {std::numeric_limits::quiet_NaN(), 2.0}); + format::parquet::ParquetColumnStatistics nan_page_stats; + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( + mixed_index, *schema[1], 0, &nan_page_stats)); + EXPECT_FALSE(nan_page_stats.has_min_max); + + format::parquet::ParquetColumnStatistics following_page_stats; + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( + mixed_index, *schema[1], 1, &following_page_stats)); + EXPECT_TRUE(following_page_stats.has_min_max); + EXPECT_EQ(following_page_stats.min_value.get(), 1.0); + EXPECT_EQ(following_page_stats.max_value.get(), 2.0); +} + +TEST(ParquetStatisticsTransformTest, IgnoresInvertedFooterAndColumnIndexMinMax) { + auto table = arrow::Table::Make( + arrow::schema( + {arrow::field("i", arrow::int32(), false), + arrow::field("s", arrow::utf8(), false), + arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), + {int32_array({1, 2}), string_array({"a", "z"}), timestamp_array({1000000, 2000000})}); + auto reader = make_reader(table, 2, false, true); + auto schema = build_file_schema(*reader); + + const int32_t inverted_min = 10; + const int32_t inverted_max = 1; + auto int_stats = ::parquet::MakeStatistics<::parquet::Int32Type>( + schema[0]->descriptor, encoded_value(inverted_min), encoded_value(inverted_max), 2, 0, + 0, true, true, false); + const auto converted_int = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], int_stats); + EXPECT_TRUE(converted_int.has_not_null); + EXPECT_FALSE(converted_int.has_min_max); + + const std::string inverted_string_min = "z"; + const std::string inverted_string_max = "a"; + auto string_stats = ::parquet::MakeStatistics<::parquet::ByteArrayType>( + schema[1]->descriptor, inverted_string_min, inverted_string_max, 2, 0, 0, true, true, + false); + const auto converted_string = + format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], + string_stats); + EXPECT_TRUE(converted_string.has_not_null); + EXPECT_FALSE(converted_string.has_min_max); + + auto int_index = + std::make_shared>(inverted_min, inverted_max); + format::parquet::ParquetColumnStatistics page_stats; + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( + int_index, *schema[0], 0, &page_stats)); + EXPECT_TRUE(page_stats.has_not_null); + EXPECT_FALSE(page_stats.has_min_max); + + // These endpoints are inverted within one second. Whole-second validation alone would miss + // the corruption, and TIMESTAMPTZ must reject the same raw inversion before its UTC shortcut. + constexpr int64_t timestamp_min = 1500000; + constexpr int64_t timestamp_max = 1000000; + auto timestamp_stats = ::parquet::MakeStatistics<::parquet::Int64Type>( + schema[2]->descriptor, encoded_value(timestamp_min), encoded_value(timestamp_max), 2, 0, + 0, true, true, false); + auto utc = cctz::utc_time_zone(); + const auto converted_timestamp = + format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[2], timestamp_stats, &utc); + EXPECT_FALSE(converted_timestamp.has_min_max); + + schema[2]->type = std::make_shared(6); + schema[2]->type_descriptor.doris_type = schema[2]->type; + const auto converted_timestamp_tz = + format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[2], timestamp_stats, &utc); + EXPECT_FALSE(converted_timestamp_tz.has_min_max); +} + +TEST(ParquetStatisticsTransformTest, PreservesNullCountWhenNaNInvalidatesMinMax) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float64(), false)}), + {double_array({1.0, 2.0})}); + auto reader = make_reader(table, 2, false, true); + auto schema = build_file_schema(*reader); + + const double nan = std::numeric_limits::quiet_NaN(); + const double max_value = 2.0; + auto footer_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( + schema[0]->descriptor, encoded_value(nan), encoded_value(max_value), 2, 0, 0, true, + true, false); + const auto converted_footer = + format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[0], + footer_stats); + auto footer_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(converted_footer); + ASSERT_NE(footer_zone_map, nullptr); + EXPECT_TRUE(footer_zone_map->pass_all); + EXPECT_FALSE(footer_zone_map->has_null); + EXPECT_TRUE(footer_zone_map->has_not_null); + EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*footer_zone_map, schema[0]->type)); + + ZoneMapEvalContext footer_ctx; + footer_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, + .zone_map = footer_zone_map}); + Int32ZoneMapExpr is_null_expr(0, Int32ZoneMapExpr::Op::IS_NULL); + EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(footer_ctx), ZoneMapFilterResult::kNoMatch); + + auto column_index = std::make_shared>(nan, max_value); + format::parquet::ParquetColumnStatistics page_stats; + ASSERT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( + column_index, *schema[0], 0, &page_stats)); + auto page_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(page_stats); + ASSERT_NE(page_zone_map, nullptr); + EXPECT_TRUE(page_zone_map->pass_all); + EXPECT_FALSE(page_zone_map->has_null); + EXPECT_TRUE(page_zone_map->has_not_null); + EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*page_zone_map, schema[0]->type)); + + ZoneMapEvalContext page_ctx; + page_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, + .zone_map = page_zone_map}); + EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(page_ctx), ZoneMapFilterResult::kNoMatch); +} + +TEST(ParquetStatisticsPruningTest, ExprZonemapPredicatesAndNullPredicatesPruneRowGroups) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), + {int32_array({std::nullopt, std::nullopt, 3, 4, 5, 6})}); + auto reader = make_reader(table, 2, false, true); + auto schema = build_file_schema(*reader); + + std::vector selected; + format::parquet::ParquetPruningStats pruning_stats; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + *reader->metadata(), reader.get(), schema, + request_with_zonemap_conjunct( + std::make_shared(0, Int32ZoneMapExpr::Op::GE, 5)), + nullptr, &selected, false, &pruning_stats) + .ok()); + EXPECT_EQ(selected, std::vector({2})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 2); + + selected.clear(); + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + *reader->metadata(), reader.get(), schema, + request_with_zonemap_conjunct(std::make_shared( + 0, Int32ZoneMapExpr::Op::IS_NOT_NULL)), + nullptr, &selected, false, &pruning_stats) + .ok()); + EXPECT_EQ(selected, std::vector({1, 2})); + + selected.clear(); + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + *reader->metadata(), reader.get(), schema, + request_with_zonemap_conjunct(std::make_shared( + 0, Int32ZoneMapExpr::Op::IS_NULL)), + nullptr, &selected, false, &pruning_stats) + .ok()); + EXPECT_EQ(selected, std::vector({0})); +} + +TEST(ParquetStatisticsPruningTest, DictionaryPruningHandlesExcludeIncludeAndUnsupportedPaths) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), + {string_array({"alpha", "beta", "gamma", "omega"})}); + auto reader = make_reader(table, 2, true, false); + auto schema = build_file_schema(*reader); + + std::vector selected; + format::parquet::ParquetPruningStats pruning_stats; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + *reader->metadata(), reader.get(), schema, + request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, + &pruning_stats) + .ok()); + EXPECT_TRUE(selected.empty()); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); + + selected.clear(); + pruning_stats = {}; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + *reader->metadata(), reader.get(), schema, + request_with_dictionary_conjunct({"gamma"}), nullptr, &selected, false, + &pruning_stats) + .ok()); + EXPECT_EQ(selected, std::vector({1})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 1); + + auto plain_reader = make_reader(table, 2, false, false); + auto plain_schema = build_file_schema(*plain_reader); + selected.clear(); + pruning_stats = {}; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + *plain_reader->metadata(), plain_reader.get(), plain_schema, + request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, + &pruning_stats) + .ok()); + EXPECT_EQ(selected, std::vector({0, 1})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 0); +} + +TEST(ParquetStatisticsPruningTest, VExprUsesDictionaryAndMissingBloomKeepsRows) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), + {string_array({"alpha", "beta", "gamma", "omega"})}); + auto reader = make_reader(table, 2, true, true); + auto schema = build_file_schema(*reader); + + std::vector selected; + format::parquet::ParquetPruningStats pruning_stats; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + *reader->metadata(), reader.get(), schema, + request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, + &pruning_stats) + .ok()); + EXPECT_TRUE(selected.empty()); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 0); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); + + auto no_stats_reader = make_reader(table, 2, false, false); + auto no_stats_schema = build_file_schema(*no_stats_reader); + selected.clear(); + pruning_stats = {}; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + *no_stats_reader->metadata(), no_stats_reader.get(), no_stats_schema, + request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, + &pruning_stats) + .ok()); + EXPECT_EQ(selected, std::vector({0, 1})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); +} + +TEST(ParquetStatisticsPruningTest, BloomFilterCacheIsScopedToRowGroupAndColumn) { + auto input = arrow::io::ReadableFile::Open( + "./be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet"); + ASSERT_TRUE(input.ok()); + auto reader = ::parquet::ParquetFileReader::Open(*input); + ASSERT_EQ(reader->metadata()->num_row_groups(), 2); + auto& bloom_filter_reader = reader->GetBloomFilterReader(); + for (int row_group_idx = 0; row_group_idx < 2; ++row_group_idx) { + auto row_group_reader = bloom_filter_reader.RowGroup(row_group_idx); + ASSERT_NE(row_group_reader, nullptr); + ASSERT_NE(row_group_reader->GetColumnBloomFilter(0), nullptr); + } + auto schema = build_file_schema(*reader); + + std::vector selected; + format::parquet::ParquetPruningStats pruning_stats; + auto request = request_with_bloom_conjunct(std::make_shared(), + {Field::create_field(12345)}); + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), + schema, request, nullptr, &selected, + false, &pruning_stats) + .ok()); + EXPECT_EQ(selected, std::vector({0, 1})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); + + selected.clear(); + pruning_stats = {}; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), + schema, request, nullptr, &selected, + true, &pruning_stats) + .ok()); + EXPECT_EQ(selected, std::vector({1})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 1); +} + +TEST(ParquetBloomFilterPruningTest, VExprEqPrunesAbsentIntValue) { + auto data_type = std::make_shared(); + auto bloom_filter = bloom_filter_for_fields( + {Field::create_field(1), Field::create_field(3)}, TYPE_INT); + auto ctx = bloom_context(data_type, bloom_filter.get()); + + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(data_type, {Field::create_field(2)}), ctx), + ZoneMapFilterResult::kNoMatch); + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(data_type, {Field::create_field(3)}), ctx), + ZoneMapFilterResult::kMayMatch); +} + +TEST(ParquetBloomFilterPruningTest, VExprInPrunesOnlyWhenAllValuesAreAbsent) { + auto data_type = std::make_shared(); + auto bloom_filter = bloom_filter_for_fields( + {Field::create_field(1), Field::create_field(3)}, TYPE_INT); + auto ctx = bloom_context(data_type, bloom_filter.get()); + + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(data_type, {Field::create_field(2), + Field::create_field(4)}), + ctx), + ZoneMapFilterResult::kNoMatch); + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(data_type, {Field::create_field(2), + Field::create_field(3)}), + ctx), + ZoneMapFilterResult::kMayMatch); +} + +TEST(ParquetBloomFilterPruningTest, VExprBoolAndStringUseSlotBloomFilter) { + auto bool_type = std::make_shared(); + auto bool_filter = + bloom_filter_for_fields({Field::create_field(true)}, TYPE_BOOLEAN); + auto bool_ctx = bloom_context(bool_type, bool_filter.get()); + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(bool_type, {Field::create_field(false)}), + bool_ctx), + ZoneMapFilterResult::kNoMatch); + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(bool_type, {Field::create_field(true)}), + bool_ctx), + ZoneMapFilterResult::kMayMatch); + + auto string_type = std::make_shared(); + auto string_filter = bloom_filter_for_fields( + {Field::create_field("alpha"), Field::create_field("omega")}, + TYPE_STRING); + auto string_ctx = bloom_context(string_type, string_filter.get()); + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(string_type, {Field::create_field("beta")}), + string_ctx), + ZoneMapFilterResult::kNoMatch); + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(string_type, {Field::create_field("alpha")}), + string_ctx), + ZoneMapFilterResult::kMayMatch); +} + +TEST(ParquetBloomFilterPruningTest, MissingOrUnsupportedBloomContextKeepsRowGroup) { + auto int_type = std::make_shared(); + BloomFilterEvalContext missing_ctx; + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(int_type, {Field::create_field(2)}), missing_ctx), + ZoneMapFilterResult::kMayMatch); + + auto smallint_type = std::make_shared(); + auto bloom_filter = bloom_filter_for_fields({Field::create_field(1)}, TYPE_INT); + auto unsupported_ctx = bloom_context(smallint_type, bloom_filter.get()); + EXPECT_EQ(VExprContext::evaluate_bloom_filter( + bloom_conjuncts(smallint_type, {Field::create_field(2)}), + unsupported_ctx), + ZoneMapFilterResult::kMayMatch); +} + +TEST(ParquetBloomFilterPruningTest, ParquetUint32BloomUsesPhysicalInt32Hash) { + const auto column_schema = uint32_parquet_bloom_schema(); + auto bloom_filter = parquet_bloom_filter(); + + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter->InsertHash(bloom_filter->Hash(physical_value)); + + // UINT32 is exposed to VExpr as Doris BIGINT, but Parquet stores and hashes it as a physical + // INT32 carrier. A present value above INT32_MAX must therefore be narrowed to the physical + // bit pattern before probing the file bloom filter. + EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field( + static_cast(present_value))}), + *bloom_filter)); + + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), + *bloom_filter)); + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( + column_schema, 0, + bloom_conjuncts( + column_schema.type, + {Field::create_field( + static_cast(std::numeric_limits::max()) + 1)}), + *bloom_filter)); +} + +TEST(ParquetBloomFilterPruningTest, ParquetFixedLenByteArrayBloomUsesFlbaHash) { + const auto column_schema = fixed_len_string_parquet_bloom_schema(4); + auto bloom_filter = parquet_bloom_filter(); + + const std::string present_value = "abcd"; + ::parquet::FLBA physical_value(reinterpret_cast(present_value.data())); + bloom_filter->InsertHash( + bloom_filter->Hash(&physical_value, column_schema.type_descriptor.fixed_length)); + + EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field(present_value)}), + *bloom_filter)); + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field("abc")}), + *bloom_filter)); +} + +TEST(ParquetBloomFilterPruningTest, ParquetFloat16BloomDoesNotUseFloatHash) { + const auto column_schema = float16_parquet_bloom_schema(); + auto bloom_filter = parquet_bloom_filter(); + + EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field(1.0F)}), + *bloom_filter)); +} + +} // namespace +} // namespace doris diff --git a/be/test/format_v2/parquet/parquet_type_test.cpp b/be/test/format_v2/parquet/parquet_type_test.cpp new file mode 100644 index 00000000000000..4bca77c1803b49 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_type_test.cpp @@ -0,0 +1,494 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/parquet_type.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/primitive_type.h" + +namespace doris::format::parquet { +namespace { + +::parquet::SchemaDescriptor make_descriptor(const ::parquet::schema::NodePtr& node) { + auto schema = + ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, {node}); + ::parquet::SchemaDescriptor descriptor; + descriptor.Init(schema); + return descriptor; +} + +ParquetTypeDescriptor resolve_node(const ::parquet::schema::NodePtr& node) { + auto descriptor = make_descriptor(node); + return resolve_parquet_type(descriptor.Column(0)); +} + +PrimitiveType primitive_type(const DataTypePtr& type) { + return remove_nullable(type)->get_primitive_type(); +} + +int scale_of(const DataTypePtr& type) { + return remove_nullable(type)->get_scale(); +} + +std::shared_ptr make_float16_array() { + arrow::HalfFloatBuilder builder; + EXPECT_TRUE(builder.Append(0x3E00).ok()); + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +ParquetTypeDescriptor resolve_arrow_float16_type() { + const auto schema = arrow::schema({arrow::field("f16", arrow::float16(), true)}); + const auto table = arrow::Table::Make(schema, {make_float16_array()}); + auto out_result = arrow::io::BufferOutputStream::Create(); + EXPECT_TRUE(out_result.ok()); + auto out = *out_result; + EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1).ok()); + auto buffer_result = out->Finish(); + EXPECT_TRUE(buffer_result.ok()); + + auto reader = ::parquet::ParquetFileReader::Open( + std::make_shared(*buffer_result)); + return resolve_parquet_type(reader->metadata()->schema()->Column(0)); +} + +} // namespace + +TEST(ParquetTypeTest, ResolveLogicalIntegerMappings) { + struct Case { + int bit_width; + bool is_signed; + PrimitiveType expected_type; + bool expected_unsigned; + }; + const std::vector cases = { + {8, true, TYPE_TINYINT, false}, {8, false, TYPE_SMALLINT, true}, + {16, true, TYPE_SMALLINT, false}, {16, false, TYPE_INT, true}, + {32, true, TYPE_INT, false}, {32, false, TYPE_BIGINT, true}, + {64, true, TYPE_BIGINT, false}, {64, false, TYPE_LARGEINT, true}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.bit_width); + const auto node = ::parquet::schema::PrimitiveNode::Make( + "c", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Int(test_case.bit_width, test_case.is_signed), + test_case.bit_width == 64 ? ::parquet::Type::INT64 : ::parquet::Type::INT32); + const auto type = resolve_node(node); + ASSERT_NE(type.doris_type, nullptr); + EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); + EXPECT_EQ(type.integer_bit_width, test_case.bit_width); + EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); + EXPECT_TRUE(type.supports_record_reader); + } +} + +TEST(ParquetTypeTest, ResolveLogicalTimeAndTimestampMappings) { + const auto time_millis = resolve_node(::parquet::schema::PrimitiveNode::Make( + "time_ms", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32)); + ASSERT_NE(time_millis.doris_type, nullptr); + EXPECT_EQ(primitive_type(time_millis.doris_type), TYPE_TIMEV2); + EXPECT_EQ(time_millis.time_unit, ParquetTimeUnit::MILLIS); + EXPECT_EQ(time_millis.extra_type_info, ParquetExtraTypeInfo::UNIT_MS); + + const auto time_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( + "time_us", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MICROS), + ::parquet::Type::INT64)); + ASSERT_NE(time_micros.doris_type, nullptr); + EXPECT_EQ(primitive_type(time_micros.doris_type), TYPE_TIMEV2); + EXPECT_EQ(time_micros.time_unit, ParquetTimeUnit::MICROS); + EXPECT_EQ(time_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); + + const auto adjusted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( + "time_adjusted", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32)); + EXPECT_EQ(adjusted_time.doris_type, nullptr); + EXPECT_FALSE(adjusted_time.supports_record_reader); + EXPECT_FALSE(adjusted_time.unsupported_reason.empty()); + + const auto timestamp_nanos = resolve_node(::parquet::schema::PrimitiveNode::Make( + "ts_ns", ::parquet::Repetition::OPTIONAL, + ::parquet::LogicalType::Timestamp(true, ::parquet::LogicalType::TimeUnit::NANOS), + ::parquet::Type::INT64)); + ASSERT_NE(timestamp_nanos.doris_type, nullptr); + EXPECT_TRUE(timestamp_nanos.doris_type->is_nullable()); + EXPECT_EQ(primitive_type(timestamp_nanos.doris_type), TYPE_DATETIMEV2); + EXPECT_TRUE(timestamp_nanos.is_timestamp); + EXPECT_TRUE(timestamp_nanos.timestamp_is_adjusted_to_utc); + EXPECT_EQ(timestamp_nanos.time_unit, ParquetTimeUnit::NANOS); + EXPECT_EQ(timestamp_nanos.extra_type_info, ParquetExtraTypeInfo::UNIT_NS); +} + +TEST(ParquetTypeTest, ResolveLogicalTimestampMatrix) { + struct Case { + ::parquet::LogicalType::TimeUnit::unit parquet_unit; + bool adjusted_to_utc; + ParquetTimeUnit expected_unit; + ParquetExtraTypeInfo expected_extra; + int expected_scale; + }; + const std::vector cases = { + {::parquet::LogicalType::TimeUnit::MILLIS, true, ParquetTimeUnit::MILLIS, + ParquetExtraTypeInfo::UNIT_MS, 3}, + {::parquet::LogicalType::TimeUnit::MILLIS, false, ParquetTimeUnit::MILLIS, + ParquetExtraTypeInfo::UNIT_MS, 3}, + {::parquet::LogicalType::TimeUnit::MICROS, true, ParquetTimeUnit::MICROS, + ParquetExtraTypeInfo::UNIT_MICROS, 6}, + {::parquet::LogicalType::TimeUnit::MICROS, false, ParquetTimeUnit::MICROS, + ParquetExtraTypeInfo::UNIT_MICROS, 6}, + {::parquet::LogicalType::TimeUnit::NANOS, true, ParquetTimeUnit::NANOS, + ParquetExtraTypeInfo::UNIT_NS, 6}, + {::parquet::LogicalType::TimeUnit::NANOS, false, ParquetTimeUnit::NANOS, + ParquetExtraTypeInfo::UNIT_NS, 6}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.expected_scale); + const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( + "ts", ::parquet::Repetition::OPTIONAL, + ::parquet::LogicalType::Timestamp(test_case.adjusted_to_utc, + test_case.parquet_unit), + ::parquet::Type::INT64)); + ASSERT_NE(type.doris_type, nullptr); + EXPECT_TRUE(type.doris_type->is_nullable()); + EXPECT_EQ(primitive_type(type.doris_type), TYPE_DATETIMEV2); + EXPECT_EQ(scale_of(type.doris_type), test_case.expected_scale); + EXPECT_TRUE(type.is_timestamp); + EXPECT_EQ(type.timestamp_is_adjusted_to_utc, test_case.adjusted_to_utc); + EXPECT_EQ(type.time_unit, test_case.expected_unit); + EXPECT_EQ(type.extra_type_info, test_case.expected_extra); + } +} + +TEST(ParquetTypeTest, ConvertedTimeIsRejectedButConvertedTimestampIsSupported) { + const auto converted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( + "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, + ::parquet::ConvertedType::TIME_MILLIS)); + EXPECT_EQ(converted_time.doris_type, nullptr); + EXPECT_FALSE(converted_time.supports_record_reader); + EXPECT_FALSE(converted_time.unsupported_reason.empty()); + + const auto converted_timestamp = resolve_node(::parquet::schema::PrimitiveNode::Make( + "ts_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64, + ::parquet::ConvertedType::TIMESTAMP_MILLIS)); + ASSERT_NE(converted_timestamp.doris_type, nullptr); + EXPECT_EQ(primitive_type(converted_timestamp.doris_type), TYPE_DATETIMEV2); + EXPECT_TRUE(converted_timestamp.is_timestamp); + EXPECT_TRUE(converted_timestamp.timestamp_is_adjusted_to_utc); + EXPECT_EQ(converted_timestamp.time_unit, ParquetTimeUnit::MILLIS); + + const auto converted_timestamp_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( + "ts_us", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT64, + ::parquet::ConvertedType::TIMESTAMP_MICROS)); + ASSERT_NE(converted_timestamp_micros.doris_type, nullptr); + EXPECT_TRUE(converted_timestamp_micros.doris_type->is_nullable()); + EXPECT_EQ(primitive_type(converted_timestamp_micros.doris_type), TYPE_DATETIMEV2); + EXPECT_EQ(scale_of(converted_timestamp_micros.doris_type), 6); + EXPECT_TRUE(converted_timestamp_micros.is_timestamp); + EXPECT_TRUE(converted_timestamp_micros.timestamp_is_adjusted_to_utc); + EXPECT_EQ(converted_timestamp_micros.time_unit, ParquetTimeUnit::MICROS); + EXPECT_EQ(converted_timestamp_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); +} + +TEST(ParquetTypeTest, ResolveConvertedIntegerMappingsAndDecodedKinds) { + struct Case { + ::parquet::ConvertedType::type converted_type; + ::parquet::Type::type physical_type; + PrimitiveType expected_type; + int bit_width; + bool expected_unsigned; + DecodedValueKind expected_value_kind; + }; + const std::vector cases = { + {::parquet::ConvertedType::INT_8, ::parquet::Type::INT32, TYPE_TINYINT, 8, false, + DecodedValueKind::INT32}, + {::parquet::ConvertedType::UINT_8, ::parquet::Type::INT32, TYPE_SMALLINT, 8, true, + DecodedValueKind::INT32}, + {::parquet::ConvertedType::INT_16, ::parquet::Type::INT32, TYPE_SMALLINT, 16, false, + DecodedValueKind::INT32}, + {::parquet::ConvertedType::UINT_16, ::parquet::Type::INT32, TYPE_INT, 16, true, + DecodedValueKind::INT32}, + {::parquet::ConvertedType::INT_32, ::parquet::Type::INT32, TYPE_INT, 32, false, + DecodedValueKind::INT32}, + {::parquet::ConvertedType::UINT_32, ::parquet::Type::INT32, TYPE_BIGINT, 32, true, + DecodedValueKind::UINT32}, + {::parquet::ConvertedType::INT_64, ::parquet::Type::INT64, TYPE_BIGINT, 64, false, + DecodedValueKind::INT64}, + {::parquet::ConvertedType::UINT_64, ::parquet::Type::INT64, TYPE_LARGEINT, 64, true, + DecodedValueKind::UINT64}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.converted_type); + const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( + "c", ::parquet::Repetition::REQUIRED, test_case.physical_type, + test_case.converted_type)); + ASSERT_NE(type.doris_type, nullptr); + EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); + EXPECT_EQ(type.integer_bit_width, test_case.bit_width); + EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); + EXPECT_EQ(decoded_value_kind(type), test_case.expected_value_kind); + } +} + +TEST(ParquetTypeTest, ResolveConvertedDecimalCarriers) { + struct Case { + ::parquet::Type::type physical_type; + int type_length; + int precision; + int scale; + PrimitiveType expected_type; + ParquetExtraTypeInfo expected_extra; + }; + const std::vector cases = { + {::parquet::Type::INT32, -1, 9, 2, TYPE_DECIMAL32, ParquetExtraTypeInfo::DECIMAL_INT32}, + {::parquet::Type::INT64, -1, 18, 6, TYPE_DECIMAL64, + ParquetExtraTypeInfo::DECIMAL_INT64}, + {::parquet::Type::BYTE_ARRAY, -1, 20, 5, TYPE_DECIMAL128I, + ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, + {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16, 38, 6, TYPE_DECIMAL128I, + ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, + {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20, 39, 6, TYPE_DECIMAL256, + ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.physical_type); + const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( + "d", ::parquet::Repetition::REQUIRED, test_case.physical_type, + ::parquet::ConvertedType::DECIMAL, test_case.type_length, test_case.precision, + test_case.scale)); + ASSERT_NE(type.doris_type, nullptr); + EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); + EXPECT_TRUE(type.is_decimal); + EXPECT_FALSE(type.is_string_like); + EXPECT_EQ(type.decimal_precision, test_case.precision); + EXPECT_EQ(type.decimal_scale, test_case.scale); + EXPECT_EQ(type.extra_type_info, test_case.expected_extra); + } +} + +TEST(ParquetTypeTest, ResolveLogicalStringDateAndDecimalMappings) { + const std::vector> string_like_logical_types = { + ::parquet::LogicalType::String(), ::parquet::LogicalType::Enum(), + ::parquet::LogicalType::JSON(), ::parquet::LogicalType::BSON()}; + for (const auto& logical_type : string_like_logical_types) { + const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( + "s", ::parquet::Repetition::OPTIONAL, logical_type, ::parquet::Type::BYTE_ARRAY)); + ASSERT_NE(type.doris_type, nullptr); + EXPECT_TRUE(type.doris_type->is_nullable()); + EXPECT_EQ(primitive_type(type.doris_type), TYPE_STRING); + EXPECT_TRUE(type.is_string_like); + } + + const auto uuid = resolve_node(::parquet::schema::PrimitiveNode::Make( + "uuid", ::parquet::Repetition::OPTIONAL, ::parquet::LogicalType::UUID(), + ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); + ASSERT_NE(uuid.doris_type, nullptr); + EXPECT_TRUE(uuid.doris_type->is_nullable()); + EXPECT_EQ(primitive_type(uuid.doris_type), TYPE_STRING); + EXPECT_TRUE(uuid.is_string_like); + + const auto date = resolve_node(::parquet::schema::PrimitiveNode::Make( + "d", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Date(), + ::parquet::Type::INT32)); + ASSERT_NE(date.doris_type, nullptr); + EXPECT_EQ(primitive_type(date.doris_type), TYPE_DATEV2); + + const auto decimal64 = resolve_node(::parquet::schema::PrimitiveNode::Make( + "d64", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(18, 6), + ::parquet::Type::INT64)); + ASSERT_NE(decimal64.doris_type, nullptr); + EXPECT_EQ(primitive_type(decimal64.doris_type), TYPE_DECIMAL64); + EXPECT_TRUE(decimal64.is_decimal); + EXPECT_EQ(decimal64.decimal_precision, 18); + EXPECT_EQ(decimal64.decimal_scale, 6); + EXPECT_EQ(decimal64.extra_type_info, ParquetExtraTypeInfo::DECIMAL_INT64); + + const auto decimal128 = resolve_node(::parquet::schema::PrimitiveNode::Make( + "d128", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(38, 6), + ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); + ASSERT_NE(decimal128.doris_type, nullptr); + EXPECT_EQ(primitive_type(decimal128.doris_type), TYPE_DECIMAL128I); + EXPECT_TRUE(decimal128.is_decimal); + EXPECT_EQ(decimal128.decimal_precision, 38); + EXPECT_EQ(decimal128.decimal_scale, 6); + EXPECT_EQ(decimal128.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); + + const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( + "d256", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(39, 6), + ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20)); + ASSERT_NE(decimal256.doris_type, nullptr); + EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); + EXPECT_TRUE(decimal256.is_decimal); + EXPECT_EQ(decimal256.decimal_precision, 39); + EXPECT_EQ(decimal256.decimal_scale, 6); + EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); + EXPECT_FALSE(decimal256.is_string_like); +} + +TEST(ParquetTypeTest, LogicalConvertedAndPhysicalFallbackLevelsAreDistinct) { + const auto logical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( + "c", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Int(8, true), + ::parquet::Type::INT32)); + ASSERT_NE(logical_type.doris_type, nullptr); + EXPECT_EQ(primitive_type(logical_type.doris_type), TYPE_TINYINT); + EXPECT_EQ(logical_type.integer_bit_width, 8); + + const auto converted_type = resolve_node(::parquet::schema::PrimitiveNode::Make( + "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, + ::parquet::ConvertedType::INT_8)); + ASSERT_NE(converted_type.doris_type, nullptr); + EXPECT_EQ(primitive_type(converted_type.doris_type), TYPE_TINYINT); + EXPECT_EQ(converted_type.integer_bit_width, 8); + + const auto physical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( + "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32)); + ASSERT_NE(physical_type.doris_type, nullptr); + EXPECT_EQ(primitive_type(physical_type.doris_type), TYPE_INT); + EXPECT_EQ(physical_type.integer_bit_width, -1); +} + +TEST(ParquetTypeTest, ResolveDecimalStringLikeFloat16AndPhysicalFallback) { + const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( + "d", ::parquet::Repetition::REQUIRED, ::parquet::Type::FIXED_LEN_BYTE_ARRAY, + ::parquet::ConvertedType::DECIMAL, 20, 39, 6)); + ASSERT_NE(decimal256.doris_type, nullptr); + EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); + EXPECT_TRUE(decimal256.is_decimal); + EXPECT_FALSE(decimal256.is_string_like); + EXPECT_EQ(decimal256.decimal_precision, 39); + EXPECT_EQ(decimal256.decimal_scale, 6); + EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); + + const auto plain_binary = resolve_node(::parquet::schema::PrimitiveNode::Make( + "s", ::parquet::Repetition::REQUIRED, ::parquet::Type::BYTE_ARRAY)); + ASSERT_NE(plain_binary.doris_type, nullptr); + EXPECT_EQ(primitive_type(plain_binary.doris_type), TYPE_STRING); + EXPECT_TRUE(plain_binary.is_string_like); + + const auto float16 = resolve_arrow_float16_type(); + ASSERT_NE(float16.doris_type, nullptr); + EXPECT_TRUE(float16.doris_type->is_nullable()); + EXPECT_EQ(float16.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); + EXPECT_EQ(float16.fixed_length, 2); + EXPECT_EQ(primitive_type(float16.doris_type), TYPE_FLOAT); + EXPECT_EQ(float16.extra_type_info, ParquetExtraTypeInfo::FLOAT16); + EXPECT_FALSE(float16.is_string_like); + EXPECT_EQ(decoded_value_kind(float16), DecodedValueKind::FIXED_BINARY); +} + +TEST(ParquetTypeTest, ResolveNullDescriptorAndPhysicalFallback) { + const auto null_type = resolve_parquet_type(nullptr); + EXPECT_EQ(null_type.doris_type, nullptr); + EXPECT_EQ(null_type.physical_type, ::parquet::Type::UNDEFINED); + EXPECT_TRUE(null_type.supports_record_reader); + + const auto int96 = resolve_node(::parquet::schema::PrimitiveNode::Make( + "ts", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT96)); + ASSERT_NE(int96.doris_type, nullptr); + EXPECT_EQ(primitive_type(int96.doris_type), TYPE_DATETIMEV2); + EXPECT_EQ(int96.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); + EXPECT_EQ(decoded_value_kind(int96), DecodedValueKind::INT96); +} + +TEST(ParquetTypeTest, ResolveEveryPhysicalFallback) { + struct Case { + ::parquet::schema::NodePtr node; + PrimitiveType expected_type; + DecodedValueKind expected_kind; + bool expected_string_like = false; + }; + const std::vector cases = { + {::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::REQUIRED, + ::parquet::Type::BOOLEAN), + TYPE_BOOLEAN, DecodedValueKind::BOOL}, + {::parquet::schema::PrimitiveNode::Make("i32", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32), + TYPE_INT, DecodedValueKind::INT32}, + {::parquet::schema::PrimitiveNode::Make("i64", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT64), + TYPE_BIGINT, DecodedValueKind::INT64}, + {::parquet::schema::PrimitiveNode::Make("f", ::parquet::Repetition::REQUIRED, + ::parquet::Type::FLOAT), + TYPE_FLOAT, DecodedValueKind::FLOAT}, + {::parquet::schema::PrimitiveNode::Make("d", ::parquet::Repetition::REQUIRED, + ::parquet::Type::DOUBLE), + TYPE_DOUBLE, DecodedValueKind::DOUBLE}, + {::parquet::schema::PrimitiveNode::Make("s", ::parquet::Repetition::REQUIRED, + ::parquet::Type::BYTE_ARRAY), + TYPE_STRING, DecodedValueKind::BINARY, true}, + {::parquet::schema::PrimitiveNode::Make("fs", ::parquet::Repetition::REQUIRED, + ::parquet::Type::FIXED_LEN_BYTE_ARRAY, + ::parquet::ConvertedType::NONE, 4), + TYPE_STRING, DecodedValueKind::FIXED_BINARY, true}, + {::parquet::schema::PrimitiveNode::Make("ts", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT96), + TYPE_DATETIMEV2, DecodedValueKind::INT96}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.expected_type); + const auto type = resolve_node(test_case.node); + ASSERT_NE(type.doris_type, nullptr); + EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); + EXPECT_EQ(decoded_value_kind(type), test_case.expected_kind); + EXPECT_EQ(type.is_string_like, test_case.expected_string_like); + EXPECT_TRUE(type.supports_record_reader); + } +} + +TEST(ParquetTypeTest, InvalidLogicalAnnotationsFallBackOrRejectAsSpecified) { + EXPECT_THROW(::parquet::LogicalType::Int(24, true), ::parquet::ParquetException); + + const auto nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( + "time_ns", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::NANOS), + ::parquet::Type::INT64)); + ASSERT_NE(nanos_time.doris_type, nullptr); + EXPECT_EQ(primitive_type(nanos_time.doris_type), TYPE_BIGINT); + EXPECT_TRUE(nanos_time.unsupported_reason.empty()); + + const auto adjusted_nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( + "time_ns_utc", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::NANOS), + ::parquet::Type::INT64)); + EXPECT_EQ(adjusted_nanos_time.doris_type, nullptr); + EXPECT_FALSE(adjusted_nanos_time.supports_record_reader); + EXPECT_FALSE(adjusted_nanos_time.unsupported_reason.empty()); + + EXPECT_THROW(::parquet::schema::PrimitiveNode::Make("f16_bad", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Float16(), + ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 4), + ::parquet::ParquetException); +} + +} // namespace doris::format::parquet diff --git a/be/test/format_v2/table/hive_reader_test.cpp b/be/test/format_v2/table/hive_reader_test.cpp new file mode 100644 index 00000000000000..d4f9be81129c6c --- /dev/null +++ b/be/test/format_v2/table/hive_reader_test.cpp @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/hive_reader.h" + +#include + +#include +#include +#include +#include +#include + +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "format_v2/column_data.h" +#include "format_v2/column_mapper.h" +#include "gen_cpp/PlanNodes_types.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::hive { +namespace { + +ColumnDefinition table_column(const std::string& name, DataTypePtr type) { + ColumnDefinition column; + column.identifier = Field::create_field(name); + column.name = name; + column.type = std::move(type); + return column; +} + +ColumnDefinition file_column(const std::string& name, DataTypePtr type, int32_t local_id) { + ColumnDefinition column; + column.local_id = local_id; + column.name = name; + column.type = std::move(type); + return column; +} + +Status init_hive_reader(FileFormat format, TFileScanRangeParams* params, RuntimeState* state, + RuntimeProfile* profile, HiveReader* reader, + std::vector projected_columns) { + return reader->init({ + .projected_columns = std::move(projected_columns), + .conjuncts = {}, + .format = format, + .scan_params = params, + .io_ctx = nullptr, + .runtime_state = state, + .scanner_profile = profile, + }); +} + +Status init_hive_reader(FileFormat format, TFileScanRangeParams* params, RuntimeState* state, + RuntimeProfile* profile, HiveReader* reader) { + return init_hive_reader(format, params, state, profile, reader, + {table_column("id", std::make_shared()), + table_column("name", std::make_shared())}); +} + +bool has_name_mapping(const ColumnDefinition& column, const std::string& name) { + return std::ranges::find(column.name_mapping, name) != column.name_mapping.end(); +} + +class HiveV2ReaderTest : public testing::Test { +public: + HiveV2ReaderTest() : profile("hive_v2_reader_test") { state.set_query_options(query_options); } + +protected: + TQueryOptions query_options; + TQueryGlobals query_globals; + RuntimeState state; + RuntimeProfile profile; +}; + +// Scenario: Hive tables using OpenCSVSerde are planned as table_format=hive with CSV file format. +// HiveReader must allow that file format so TableReader can create the v2 CsvReader. +TEST_F(HiveV2ReaderTest, InitSupportsCsvFileFormat) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_CSV_PLAIN); + HiveReader reader; + + ASSERT_TRUE(init_hive_reader(FileFormat::CSV, ¶ms, &state, &profile, &reader).ok()); + EXPECT_EQ(reader.mapping_mode(), TableColumnMappingMode::BY_NAME); +} + +// Scenario: Hive text files also synthesize a file-local schema from FE slots, so they should use +// name mapping at the table-reader layer while TextReader consumes column_idxs for field ordinals. +TEST_F(HiveV2ReaderTest, InitSupportsTextFileFormat) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_TEXT); + HiveReader reader; + + ASSERT_TRUE(init_hive_reader(FileFormat::TEXT, ¶ms, &state, &profile, &reader).ok()); + EXPECT_EQ(reader.mapping_mode(), TableColumnMappingMode::BY_NAME); +} + +// Scenario: Hive JSON files also synthesize a file-local schema from FE slots, so they should use +// name mapping at the table-reader layer while JsonReader consumes JSON attributes. +TEST_F(HiveV2ReaderTest, InitSupportsJsonFileFormat) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_JSON); + HiveReader reader; + + ASSERT_TRUE(init_hive_reader(FileFormat::JSON, ¶ms, &state, &profile, &reader).ok()); + EXPECT_EQ(reader.mapping_mode(), TableColumnMappingMode::BY_NAME); +} + +TEST_F(HiveV2ReaderTest, MappingModeUsesInitializedFormat) { + query_options.hive_parquet_use_column_names = false; + query_options.hive_orc_use_column_names = true; + state.set_query_options(query_options); + + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + HiveReader reader; + + ASSERT_TRUE(init_hive_reader(FileFormat::PARQUET, ¶ms, &state, &profile, &reader).ok()); + EXPECT_EQ(reader.mapping_mode(), TableColumnMappingMode::BY_INDEX); + + SplitReadOptions parquet_split; + parquet_split.current_range.__set_path("split.parquet"); + parquet_split.current_split_format = FileFormat::PARQUET; + ASSERT_TRUE(reader.prepare_split(parquet_split).ok()); + EXPECT_EQ(reader.mapping_mode(), TableColumnMappingMode::BY_INDEX); + + SplitReadOptions orc_split; + orc_split.current_range.__set_path("split.orc"); + orc_split.current_split_format = FileFormat::ORC; + EXPECT_FALSE(reader.prepare_split(orc_split).ok()); +} + +TEST_F(HiveV2ReaderTest, OrcConsumesColumnIdxsAsPositionalSchemaMapping) { + query_options.hive_orc_use_column_names = false; + state.set_query_options(query_options); + + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_ORC); + params.__set_column_idxs({3}); + ProjectedColumnBuildContext context { + .scan_params = ¶ms, + .runtime_state = &state, + }; + HiveReader reader; + + TFileScanSlotInfo slot; + slot.__set_is_file_slot(true); + auto column = table_column("value", std::make_shared()); + + ASSERT_TRUE(reader.annotate_projected_column(slot, &context, &column).ok()); + ASSERT_TRUE(column.has_identifier_field_id()); + EXPECT_EQ(column.get_identifier_position(), 3); + EXPECT_EQ(context.next_file_column_idx, 1); +} + +// Scenario: positional mapping is only for Hive Parquet/ORC sessions that disable name mapping. +// CSV keeps the synthesized file-column names and leaves column_idxs for the CsvReader itself. +TEST_F(HiveV2ReaderTest, CsvDoesNotConsumeColumnIdxsAsPositionalSchemaMapping) { + query_options.hive_parquet_use_column_names = false; + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_CSV_PLAIN); + params.__set_column_idxs({3}); + ProjectedColumnBuildContext context { + .scan_params = ¶ms, + .runtime_state = &state, + }; + HiveReader reader; + + TFileScanSlotInfo slot; + slot.__set_is_file_slot(true); + auto column = table_column("value", std::make_shared()); + + ASSERT_TRUE(reader.annotate_projected_column(slot, &context, &column).ok()); + ASSERT_TRUE(column.has_identifier_name()); + EXPECT_EQ(column.get_identifier_name(), "value"); + EXPECT_EQ(context.next_file_column_idx, 0); +} + +TEST_F(HiveV2ReaderTest, OrcHive1ColumnNamesUsePositionAliasesWhenNameMappingEnabled) { + query_options.hive_orc_use_column_names = true; + state.set_query_options(query_options); + + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_ORC); + params.__set_column_idxs({0, 1, 2}); + std::vector projected_columns { + table_column("a", std::make_shared()), + table_column("b", std::make_shared()), + table_column("c", std::make_shared())}; + HiveReader reader; + ASSERT_TRUE( + init_hive_reader(FileFormat::ORC, ¶ms, &state, &profile, &reader, projected_columns) + .ok()); + + std::vector file_schema { + file_column("_col0", std::make_shared(), 0), + file_column("_col1", std::make_shared(), 1), + file_column("_col2", std::make_shared(), 2)}; + ASSERT_TRUE(reader.TEST_annotate_file_schema(&file_schema).ok()); + + EXPECT_TRUE(has_name_mapping(file_schema[0], "a")); + EXPECT_TRUE(has_name_mapping(file_schema[1], "b")); + EXPECT_TRUE(has_name_mapping(file_schema[2], "c")); + + TableColumnMapper mapper({.mode = reader.mapping_mode()}); + ASSERT_TRUE(mapper.create_mapping(projected_columns, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 3); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + ASSERT_TRUE(mapper.mappings()[1].file_local_id.has_value()); + ASSERT_TRUE(mapper.mappings()[2].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); + EXPECT_EQ(*mapper.mappings()[1].file_local_id, 1); + EXPECT_EQ(*mapper.mappings()[2].file_local_id, 2); +} + +} // namespace +} // namespace doris::format::hive diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp new file mode 100644 index 00000000000000..77cd75bc948ac8 --- /dev/null +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/hudi_reader.h" + +#include + +#include +#include +#include +#include +#include + +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/field.h" +#include "format_v2/column_data.h" +#include "gen_cpp/ExternalTableSchema_types.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format { +namespace { + +schema::external::TFieldPtr external_schema_field(std::string name, int32_t id, + std::vector aliases = {}) { + auto field = std::make_shared(); + field->__set_name(std::move(name)); + field->__set_id(id); + if (!aliases.empty()) { + field->__set_name_mapping(std::move(aliases)); + } + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; +} + +schema::external::TSchema external_schema(int64_t schema_id, + std::vector fields) { + schema::external::TStructField root_field; + root_field.__set_fields(std::move(fields)); + schema::external::TSchema schema; + schema.__set_schema_id(schema_id); + schema.__set_root_field(std::move(root_field)); + return schema; +} + +ColumnDefinition make_file_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition field; + field.identifier = Field::create_field(id); + field.local_id = id; + field.name = name; + field.type = type; + return field; +} + +TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("hudi"); + THudiFileDesc hudi_params; + if (schema_id.has_value()) { + hudi_params.__set_schema_id(*schema_id); + } + table_format_params.__set_hudi_params(hudi_params); + return table_format_params; +} + +// Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical +// file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id +// mapping, including renamed nested children. +TEST(HudiReaderTest, AnnotatesFileSchemaFromSplitHistorySchema) { + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(200); + + auto profile_field = external_schema_field("profile", 20); + schema::external::TStructField profile_struct; + profile_struct.__set_fields({external_schema_field("old_age", 21, {"age"})}); + profile_field.field_ptr->nestedField.__set_struct_field(std::move(profile_struct)); + profile_field.field_ptr->__isset.nestedField = true; + + scan_params.__set_history_schema_info({ + external_schema(100, {external_schema_field("old_name", 10, {"name"}), profile_field}), + external_schema( + 200, {external_schema_field("name", 10), external_schema_field("profile", 20)}), + }); + + hudi::HudiReader reader; + reader.TEST_set_scan_params(&scan_params); + + SplitReadOptions split_options; + split_options.current_range.__set_table_format_params(hudi_table_format_desc(100)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_FIELD_ID); + + auto string_type = std::make_shared(); + auto int_type = std::make_shared(); + auto profile_type = std::make_shared(DataTypes {int_type}, Strings {"old_age"}); + auto profile_column = make_file_column(1, "profile", profile_type); + profile_column.children = {make_file_column(0, "old_age", int_type)}; + std::vector file_schema { + make_file_column(0, "old_name", string_type), + profile_column, + }; + + ASSERT_TRUE(reader.TEST_annotate_file_schema(&file_schema).ok()); + ASSERT_EQ(file_schema.size(), 2); + EXPECT_EQ(file_schema[0].get_identifier_field_id(), 10); + EXPECT_EQ(file_schema[0].name_mapping, std::vector({"name"})); + EXPECT_EQ(file_schema[1].get_identifier_field_id(), 20); + ASSERT_EQ(file_schema[1].children.size(), 1); + EXPECT_EQ(file_schema[1].children[0].get_identifier_field_id(), 21); + EXPECT_EQ(file_schema[1].children[0].name_mapping, std::vector({"age"})); +} + +// Scenario: a Hudi split can only use field-id mapping when its schema id resolves to a historical +// schema sent by FE. Unknown or missing split schema ids must fall back to BY_NAME and leave the +// physical file schema untouched. +TEST(HudiReaderTest, FallsBackToByNameWhenSplitHistorySchemaIsMissing) { + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(200); + scan_params.__set_history_schema_info({ + external_schema(200, {external_schema_field("name", 10)}), + }); + + hudi::HudiReader reader; + reader.TEST_set_scan_params(&scan_params); + + SplitReadOptions split_options; + split_options.current_range.__set_table_format_params(hudi_table_format_desc(100)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_NAME); + + std::vector file_schema { + make_file_column(0, "old_name", std::make_shared()), + }; + ASSERT_TRUE(reader.TEST_annotate_file_schema(&file_schema).ok()); + EXPECT_EQ(file_schema[0].get_identifier_field_id(), 0); + EXPECT_TRUE(file_schema[0].name_mapping.empty()); +} + +// Scenario: HudiReader must reset the previous split schema id before each split. Otherwise a +// BY_FIELD_ID split could leak its schema id into the next split that carries no schema id. +TEST(HudiReaderTest, ResetsSplitSchemaIdBeforePreparingNextSplit) { + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(200); + scan_params.__set_history_schema_info({ + external_schema(100, {external_schema_field("old_name", 10, {"name"})}), + external_schema(200, {external_schema_field("name", 10)}), + }); + + hudi::HudiReader reader; + reader.TEST_set_scan_params(&scan_params); + + SplitReadOptions split_with_schema_id; + split_with_schema_id.current_range.__set_table_format_params(hudi_table_format_desc(100)); + ASSERT_TRUE(reader.prepare_split(split_with_schema_id).ok()); + EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_FIELD_ID); + + SplitReadOptions split_without_schema_id; + split_without_schema_id.current_range.__set_table_format_params( + hudi_table_format_desc(std::nullopt)); + ASSERT_TRUE(reader.prepare_split(split_without_schema_id).ok()); + EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_NAME); +} + +TEST(HudiHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { + hudi::HudiHybridReader reader; + reader.TEST_install_batch_size_children(); + reader.set_batch_size(123); + const auto child_batch_sizes = reader.TEST_child_batch_sizes(); + EXPECT_EQ(child_batch_sizes.first, 123); + EXPECT_EQ(child_batch_sizes.second, 123); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp new file mode 100644 index 00000000000000..a367abf96f6a9f --- /dev/null +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -0,0 +1,2936 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_const.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" +#include "exec/common/endian.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr.h" +#include "exprs/vliteral.h" +#include "exprs/vruntimefilter_wrapper.h" +#include "exprs/vslot_ref.h" +#include "format/format_common.h" +#include "format_v2/deletion_vector_reader.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/Exprs_types.h" +#include "gen_cpp/ExternalTableSchema_types.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/io_common.h" +#include "roaring/roaring64map.hh" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/segment/condition_cache.h" +#include "util/debug_points.h" + +namespace doris::format { +namespace { + +LocalColumnIndex field_projection(int32_t column_id) { + return LocalColumnIndex {.index = column_id}; +} + +std::vector projection_ids(const std::vector& projections) { + std::vector ids; + ids.reserve(projections.size()); + for (const auto& projection : projections) { + ids.push_back(projection.index); + } + return ids; +} +VExprSPtr table_int32_slot_ref(int slot_id, int column_id, const std::string& column_name) { + const auto nullable_int_type = make_nullable(std::make_shared()); + return VSlotRef::create_shared(slot_id, column_id, slot_id, nullable_int_type, column_name); +} + +VExprSPtr table_int32_literal(int32_t value) { + return VLiteral::create_shared(std::make_shared(), + Field::create_field(value)); +} + +VExprSPtr table_int64_literal(int64_t value) { + return VLiteral::create_shared(std::make_shared(), + Field::create_field(value)); +} + +TExprNode table_function_node(const std::string& function_name, const DataTypePtr& return_type, + const std::vector& arg_types, + TExprNodeType::type node_type, + TExprOpcode::type opcode = TExprOpcode::INVALID_OPCODE, + bool short_circuit_evaluation = false) { + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + std::vector thrift_arg_types; + thrift_arg_types.reserve(arg_types.size()); + for (const auto& arg_type : arg_types) { + thrift_arg_types.push_back(arg_type->to_thrift()); + } + fn.__set_arg_types(thrift_arg_types); + fn.__set_ret_type(return_type->to_thrift()); + fn.__set_has_var_args(false); + + TExprNode node; + node.__set_node_type(node_type); + node.__set_opcode(opcode); + node.__set_type(return_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(static_cast(arg_types.size())); + node.__set_is_nullable(return_type->is_nullable()); + if (short_circuit_evaluation) { + node.__set_short_circuit_evaluation(true); + } + return node; +} + +VExprSPtr table_function_expr(const std::string& function_name, const DataTypePtr& return_type, + const std::vector& arg_types, + TExprNodeType::type node_type = TExprNodeType::FUNCTION_CALL, + TExprOpcode::type opcode = TExprOpcode::INVALID_OPCODE) { + const auto node = table_function_node(function_name, return_type, arg_types, node_type, opcode); + return VectorizedFnCall::create_shared(node); +} + +VExprSPtr table_int32_greater_than_expr(int slot_id, int column_id, int32_t value) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + auto expr = table_function_expr("gt", make_nullable(std::make_shared()), + {nullable_int_type, int_type}, TExprNodeType::BINARY_PRED, + TExprOpcode::GT); + expr->add_child(table_int32_slot_ref(slot_id, column_id, "id")); + expr->add_child(table_int32_literal(value)); + return expr; +} + +VExprSPtr table_nullable_int64_binary_predicate(const std::string& function_name, + TExprOpcode::type opcode, int slot_id, + int column_id, const std::string& column_name, + int64_t value) { + const auto int64_type = std::make_shared(); + const auto nullable_int64_type = make_nullable(int64_type); + auto expr = table_function_expr(function_name, make_nullable(std::make_shared()), + {nullable_int64_type, int64_type}, TExprNodeType::BINARY_PRED, + opcode); + expr->add_child( + VSlotRef::create_shared(slot_id, column_id, slot_id, nullable_int64_type, column_name)); + expr->add_child(table_int64_literal(value)); + return expr; +} + +class IcebergTableReaderDeleteFileTestHelper final + : public doris::format::iceberg::IcebergTableReader { +public: + Status parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, + bool* has_delete_file) { + return _parse_deletion_vector_file(t_desc, desc, has_delete_file); + } +}; + +class IcebergTableReaderScanRequestTestHelper final + : public doris::format::iceberg::IcebergTableReader { +public: + Status init_for_scan_request_test(std::vector projected_columns) { + _query_options = std::make_unique(); + _query_globals = std::make_unique(); + _state = std::make_unique(*_query_options, *_query_globals); + RETURN_IF_ERROR(init({ + .projected_columns = std::move(projected_columns), + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = _state.get(), + .scanner_profile = nullptr, + })); + + SplitReadOptions split_options; + split_options.current_range.__set_path("scan-request-test.parquet"); + TTableFormatFileDesc table_format_params; + TIcebergFileDesc iceberg_params; + iceberg_params.__set_first_row_id(1000); + table_format_params.__set_iceberg_params(iceberg_params); + split_options.current_range.__set_table_format_params(table_format_params); + RETURN_IF_ERROR(prepare_split(split_options)); + + _delete_rows_storage = {1}; + _delete_rows = &_delete_rows_storage; + return Status::OK(); + } + + Status customize_request(FileScanRequest* request) { + return customize_file_scan_request(request); + } + +private: + std::unique_ptr _query_options; + std::unique_ptr _query_globals; + std::unique_ptr _state; + DeleteRows _delete_rows_storage; +}; + +class IcebergTableReaderMappingModeTestHelper final + : public doris::format::iceberg::IcebergTableReader { +public: + TableColumnMappingMode mapping_mode_for_schema(std::vector file_schema) { + _data_reader.file_schema = std::move(file_schema); + return mapping_mode(); + } +}; + +std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { + std::shared_ptr array; + EXPECT_TRUE(builder->Finish(&array).ok()); + return array; +} + +std::shared_ptr build_int32_array(const std::vector& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_int64_array(const std::vector& values) { + arrow::Int64Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_nullable_int64_array( + const std::vector>& values) { + arrow::Int64Builder builder; + for (const auto& value : values) { + if (value.has_value()) { + EXPECT_TRUE(builder.Append(*value).ok()); + } else { + EXPECT_TRUE(builder.AppendNull().ok()); + } + } + return finish_array(&builder); +} + +std::shared_ptr build_string_array(const std::vector& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +schema::external::TFieldPtr external_schema_field( + std::string name, int32_t id, std::vector aliases = {}, + std::optional initial_default = std::nullopt, + std::optional type = std::nullopt, bool initial_default_is_base64 = false) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + if (!aliases.empty()) { + field->__set_name_mapping(aliases); + } + if (initial_default.has_value()) { + field->__set_initial_default_value(*initial_default); + if (initial_default_is_base64) { + field->__set_initial_default_value_is_base64(true); + } + } + if (type.has_value()) { + field->__set_type(*type); + } + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; +} + +TColumnType external_primitive_type(TPrimitiveType::type type, int32_t len = -1, + int32_t scale = -1) { + TColumnType result; + result.__set_type(type); + if (len >= 0) { + result.__set_len(len); + } + if (scale >= 0) { + result.__set_scale(scale); + } + return result; +} + +schema::external::TSchema external_schema(int64_t schema_id, + std::vector fields) { + schema::external::TStructField root_field; + root_field.__set_fields(fields); + schema::external::TSchema schema; + schema.__set_schema_id(schema_id); + schema.__set_root_field(root_field); + return schema; +} + +void write_iceberg_equality_delete_parquet_file(const std::string& file_path, int32_t field_id, + int32_t value, + const std::string& field_name = "id") { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + auto schema = arrow::schema({ + arrow::field(field_name, arrow::int32(), false)->WithMetadata(metadata), + }); + auto table = arrow::Table::Make(schema, {build_int32_array({value})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_iceberg_timestamp_equality_delete_parquet_file(const std::string& file_path, + int32_t field_id, int64_t value, + const std::string& field_name, + bool adjusted_to_utc = false) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + const auto timestamp_type = adjusted_to_utc ? arrow::timestamp(arrow::TimeUnit::MICRO, "UTC") + : arrow::timestamp(arrow::TimeUnit::MICRO); + arrow::TimestampBuilder value_builder(timestamp_type, arrow::default_memory_pool()); + ASSERT_TRUE(value_builder.Append(value).ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto schema = arrow::schema({ + arrow::field(field_name, timestamp_type, false)->WithMetadata(metadata), + }); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_iceberg_binary_equality_delete_parquet_file(const std::string& file_path, + int32_t field_id, const std::string& value, + const std::string& field_name) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + arrow::BinaryBuilder value_builder; + ASSERT_TRUE(value_builder.Append(value).ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto schema = arrow::schema({ + arrow::field(field_name, arrow::binary(), false)->WithMetadata(metadata), + }); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_iceberg_null_equality_delete_parquet_file(const std::string& file_path, int32_t field_id, + const std::string& field_name) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + auto schema = arrow::schema({ + arrow::field(field_name, arrow::int32(), true)->WithMetadata(metadata), + }); + arrow::Int32Builder value_builder; + ASSERT_TRUE(value_builder.AppendNull().ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_single_int_parquet_file(const std::string& file_path, const std::string& field_name, + const std::vector& values, + std::optional field_id) { + auto field = arrow::field(field_name, arrow::int32(), false); + if (field_id.has_value()) { + field = field->WithMetadata( + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(*field_id)})); + } + auto table = arrow::Table::Make(arrow::schema({field}), {build_int32_array(values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(values.size()), + builder.build())); +} + +void write_two_int_parquet_file(const std::string& file_path, const std::string& first_name, + const std::vector& first_values, + std::optional first_field_id, + const std::string& second_name, + const std::vector& second_values, + std::optional second_field_id) { + ASSERT_EQ(first_values.size(), second_values.size()); + auto first_field = arrow::field(first_name, arrow::int32(), false); + if (first_field_id.has_value()) { + first_field = first_field->WithMetadata( + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(*first_field_id)})); + } + auto second_field = arrow::field(second_name, arrow::int32(), false); + if (second_field_id.has_value()) { + second_field = second_field->WithMetadata(arrow::key_value_metadata( + {"PARQUET:field_id"}, {std::to_string(*second_field_id)})); + } + auto schema = arrow::schema({first_field, second_field}); + auto table = arrow::Table::Make( + schema, {build_int32_array(first_values), build_int32_array(second_values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(first_values.size()), + builder.build())); +} + +void write_timestamp_int_parquet_file(const std::string& file_path, + const std::vector& timestamps, + const std::vector& ids) { + ASSERT_EQ(timestamps.size(), ids.size()); + const auto timestamp_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"0"}); + const auto id_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}); + const auto timestamp_type = arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + arrow::TimestampBuilder timestamp_builder(timestamp_type, arrow::default_memory_pool()); + ASSERT_TRUE(timestamp_builder.AppendValues(timestamps).ok()); + auto timestamp_result = timestamp_builder.Finish(); + ASSERT_TRUE(timestamp_result.ok()) << timestamp_result.status(); + auto schema = arrow::schema( + {arrow::field("event_time", timestamp_type, false)->WithMetadata(timestamp_metadata), + arrow::field("id", arrow::int32(), false)->WithMetadata(id_metadata)}); + auto table = arrow::Table::Make(schema, {*timestamp_result, build_int32_array(ids)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(timestamps.size()), + builder.build())); +} + +void write_iceberg_equality_delete_bigint_parquet_file(const std::string& file_path, + int32_t field_id, int64_t value) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + auto schema = arrow::schema({ + arrow::field("id", arrow::int64(), false)->WithMetadata(metadata), + }); + auto table = arrow::Table::Make(schema, {build_int64_array({value})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_int_pair_parquet_file(const std::string& file_path, const std::vector& ids, + const std::vector& scores, + const std::vector& values, + int64_t row_group_size = -1) { + const auto id_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"0"}); + const auto score_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}); + const auto value_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"2"}); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false)->WithMetadata(id_metadata), + arrow::field("score", arrow::int32(), false)->WithMetadata(score_metadata), + arrow::field("value", arrow::utf8(), false)->WithMetadata(value_metadata), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(scores), + build_string_array(values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + const auto write_row_group_size = + row_group_size > 0 ? row_group_size : static_cast(ids.size()); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + write_row_group_size, builder.build())); +} + +void write_iceberg_row_lineage_parquet_file( + const std::string& file_path, const std::vector& ids, + const std::vector>& row_ids, + const std::vector>& last_updated_sequence_numbers = {}) { + ASSERT_EQ(ids.size(), row_ids.size()); + if (!last_updated_sequence_numbers.empty()) { + ASSERT_EQ(ids.size(), last_updated_sequence_numbers.size()); + } + const auto id_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"0"}); + const auto row_id_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"2147483540"}); + const auto last_updated_sequence_number_metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {"2147483539"}); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false)->WithMetadata(id_metadata), + arrow::field("_row_id", arrow::int64(), true)->WithMetadata(row_id_metadata), + }); + std::vector> arrays = { + build_int32_array(ids), + build_nullable_int64_array(row_ids), + }; + if (!last_updated_sequence_numbers.empty()) { + schema = + schema->AddField(schema->num_fields(), + arrow::field("_last_updated_sequence_number", arrow::int64(), true) + ->WithMetadata(last_updated_sequence_number_metadata)) + .ValueOrDie(); + arrays.push_back(build_nullable_int64_array(last_updated_sequence_numbers)); + } + auto table = arrow::Table::Make(schema, arrays); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(ids.size()), + builder.build())); +} + +void write_position_delete_parquet_file(const std::string& file_path, + const std::vector& data_file_paths, + const std::vector& positions) { + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), false), + arrow::field("pos", arrow::int64(), false), + }); + auto table = arrow::Table::Make( + schema, {build_string_array(data_file_paths), build_int64_array(positions)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(positions.size()), + builder.build())); +} + +int64_t write_iceberg_deletion_vector_file(const std::string& file_path, + const std::vector& deleted_positions) { + roaring::Roaring64Map rows; + for (const auto position : deleted_positions) { + rows.add(position); + } + + const size_t bitmap_size = rows.getSizeInBytes(); + std::vector blob(4 + 4 + bitmap_size + 4); + rows.write(blob.data() + 8); + + const uint32_t total_length = static_cast(4 + bitmap_size); + BigEndian::Store32(blob.data(), total_length); + constexpr char DV_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; + memcpy(blob.data() + 4, DV_MAGIC, 4); + BigEndian::Store32(blob.data() + 8 + bitmap_size, 0); + + std::ofstream output(file_path, std::ios::binary); + EXPECT_TRUE(output.is_open()); + output.write(blob.data(), static_cast(blob.size())); + EXPECT_TRUE(output.good()); + return static_cast(blob.size()); +} + +class ScopedDebugPoint { +public: + explicit ScopedDebugPoint(std::string name) + : _name(std::move(name)), _enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add(_name); + } + + ~ScopedDebugPoint() { + DebugPoints::instance()->remove(_name); + config::enable_debug_points = _enable_debug_points; + } + +private: + std::string _name; + bool _enable_debug_points; +}; + +Block build_table_block(const std::vector& columns) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +void expect_nullable_int64_column_values(const IColumn& column, + const std::vector& expected_values) { + const auto full_column = column.convert_to_full_column_if_const(); + const auto& nullable_column = assert_cast(*full_column); + const auto& values = + assert_cast(nullable_column.get_nested_column()).get_data(); + ASSERT_EQ(nullable_column.size(), expected_values.size()); + for (size_t row = 0; row < expected_values.size(); ++row) { + EXPECT_EQ(nullable_column.get_null_map_data()[row], 0); + EXPECT_EQ(values[row], expected_values[row]); + } +} + +void expect_nullable_int64_column_optional_values( + const IColumn& column, const std::vector>& expected_values) { + const auto full_column = column.convert_to_full_column_if_const(); + const auto& nullable_column = assert_cast(*full_column); + const auto& values = + assert_cast(nullable_column.get_nested_column()).get_data(); + ASSERT_EQ(nullable_column.size(), expected_values.size()); + for (size_t row = 0; row < expected_values.size(); ++row) { + if (expected_values[row].has_value()) { + EXPECT_EQ(nullable_column.get_null_map_data()[row], 0); + EXPECT_EQ(values[row], *expected_values[row]); + } else { + EXPECT_EQ(nullable_column.get_null_map_data()[row], 1); + } + } +} + +const IColumn& expect_not_null_nullable_nested_column(const IColumn& column) { + if (!column.is_nullable()) { + return column; + } + const auto& nullable_column = assert_cast(column); + for (const auto is_null : nullable_column.get_null_map_data()) { + EXPECT_EQ(is_null, 0); + } + return nullable_column.get_nested_column(); +} + +const IColumn& expect_not_null_table_column(const Block& block, size_t position) { + return expect_not_null_nullable_nested_column(*block.get_by_position(position).column); +} + +ColumnDefinition make_table_column(int32_t id, const std::string& name, const DataTypePtr& type); + +DataTypePtr make_iceberg_rowid_type() { + return make_nullable(std::make_shared( + DataTypes {std::make_shared(), std::make_shared(), + std::make_shared(), std::make_shared()}, + Strings {"file_path", "row_pos", "partition_spec_id", "partition_data_json"})); +} + +ColumnDefinition make_iceberg_row_lineage_row_id_column() { + return make_table_column(2147483540, "_row_id", + make_nullable(std::make_shared())); +} + +ColumnDefinition make_iceberg_last_updated_sequence_number_column() { + return make_table_column(2147483539, "_last_updated_sequence_number", + make_nullable(std::make_shared())); +} + +void expect_iceberg_rowid_column_values(const IColumn& column, const std::string& file_path, + const std::vector& row_positions, + int32_t partition_spec_id, + const std::string& partition_data_json) { + const auto full_column = column.convert_to_full_column_if_const(); + const auto& nullable_column = assert_cast(*full_column); + const auto& struct_column = + assert_cast(nullable_column.get_nested_column()); + const auto& file_path_column = assert_cast( + expect_not_null_nullable_nested_column(struct_column.get_column(0))); + const auto& row_pos_column = assert_cast( + expect_not_null_nullable_nested_column(struct_column.get_column(1))); + const auto& spec_id_column = assert_cast( + expect_not_null_nullable_nested_column(struct_column.get_column(2))); + const auto& partition_data_column = assert_cast( + expect_not_null_nullable_nested_column(struct_column.get_column(3))); + + ASSERT_EQ(nullable_column.size(), row_positions.size()); + for (size_t row = 0; row < row_positions.size(); ++row) { + EXPECT_EQ(nullable_column.get_null_map_data()[row], 0); + EXPECT_EQ(file_path_column.get_data_at(row).to_string(), file_path); + EXPECT_EQ(row_pos_column.get_element(row), row_positions[row]); + EXPECT_EQ(spec_id_column.get_element(row), partition_spec_id); + EXPECT_EQ(partition_data_column.get_data_at(row).to_string(), partition_data_json); + } +} + +void expect_int32_column_values(const IColumn& column, + const std::vector& expected_values) { + const auto full_column = column.convert_to_full_column_if_const(); + const auto& nested_column = expect_not_null_nullable_nested_column(*full_column); + const auto& values = assert_cast(nested_column).get_data(); + ASSERT_EQ(values.size(), expected_values.size()); + for (size_t row = 0; row < expected_values.size(); ++row) { + EXPECT_EQ(values[row], expected_values[row]); + } +} + +SplitReadOptions build_split_options(const std::string& file_path) { + SplitReadOptions options; + EXPECT_EQ(options.cache, nullptr); + options.current_range.__set_path(file_path); + options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + return options; +} + +void set_table_level_row_count(SplitReadOptions* split_options, int64_t row_count) { + split_options->current_range.__isset.table_format_params = true; + split_options->current_range.table_format_params.__isset.table_level_row_count = true; + split_options->current_range.table_format_params.table_level_row_count = row_count; +} + +void set_iceberg_row_lineage_params(SplitReadOptions* split_options, int64_t first_row_id, + int64_t last_updated_sequence_number) { + TTableFormatFileDesc table_format_params; + TIcebergFileDesc iceberg_params; + iceberg_params.__set_first_row_id(first_row_id); + iceberg_params.__set_last_updated_sequence_number(last_updated_sequence_number); + table_format_params.__set_iceberg_params(iceberg_params); + split_options->current_range.__set_table_format_params(table_format_params); +} + +void set_iceberg_rowid_params(SplitReadOptions* split_options, + const std::string& original_file_path, int32_t partition_spec_id, + const std::string& partition_data_json) { + TTableFormatFileDesc table_format_params; + TIcebergFileDesc iceberg_params; + iceberg_params.__set_original_file_path(original_file_path); + iceberg_params.__set_partition_spec_id(partition_spec_id); + iceberg_params.__set_partition_data_json(partition_data_json); + table_format_params.__set_iceberg_params(iceberg_params); + split_options->current_range.__set_table_format_params(table_format_params); +} + +TIcebergDeleteFileDesc make_iceberg_deletion_vector(const std::string& path, int64_t offset, + int64_t size) { + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(3); + delete_file.__set_path(path); + delete_file.__set_content_offset(offset); + delete_file.__set_content_size_in_bytes(size); + return delete_file; +} + +TIcebergDeleteFileDesc make_iceberg_position_delete_file(const std::string& path) { + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + return delete_file; +} + +TIcebergDeleteFileDesc make_iceberg_equality_delete_file(const std::string& path, + const std::vector& field_ids) { + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(2); + delete_file.__set_path(path); + delete_file.__set_field_ids(field_ids); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + return delete_file; +} + +TFileScanRangeParams make_local_parquet_scan_params() { + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + return scan_params; +} + +std::shared_ptr make_io_context(io::FileReaderStats* file_reader_stats, + io::FileCacheStatistics* file_cache_stats) { + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = file_reader_stats; + io_ctx->file_cache_stats = file_cache_stats; + return io_ctx; +} + +TTableFormatFileDesc make_iceberg_table_format_desc( + const std::string& data_file_path, + const std::vector& delete_files) { + TTableFormatFileDesc table_format_params; + TIcebergFileDesc iceberg_params; + iceberg_params.__set_format_version(2); + iceberg_params.__set_original_file_path(data_file_path); + iceberg_params.__set_delete_files(delete_files); + table_format_params.__set_iceberg_params(iceberg_params); + return table_format_params; +} + +std::vector read_iceberg_ids(doris::format::iceberg::IcebergTableReader* reader, + const std::vector& projected_columns) { + std::vector ids; + bool eos = false; + while (!eos) { + Block block = build_table_block(projected_columns); + auto status = reader->get_block(&block, &eos); + if (!status.ok()) { + ADD_FAILURE() << status; + return ids; + } + if (block.rows() == 0) { + continue; + } + const auto& id_column = + assert_cast(expect_not_null_table_column(block, 0)); + for (size_t row = 0; row < block.rows(); ++row) { + ids.push_back(id_column.get_element(row)); + } + } + return ids; +} + +void init_iceberg_reader(doris::format::iceberg::IcebergTableReader* reader, + const std::vector& projected_columns, + TFileScanRangeParams* scan_params, + const std::shared_ptr& io_ctx, RuntimeState* state, + RuntimeProfile* profile) { + ASSERT_TRUE(reader->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = scan_params, + .io_ctx = io_ctx, + .runtime_state = state, + .scanner_profile = profile, + }) + .ok()); +} + +DataTypePtr make_table_test_type(const DataTypePtr& type, bool nullable_root = true) { + DORIS_CHECK(type != nullptr); + const auto nested_type = remove_nullable(type); + DataTypePtr result; + if (const auto* struct_type = typeid_cast(nested_type.get())) { + DataTypes child_types; + child_types.reserve(struct_type->get_elements().size()); + for (const auto& child_type : struct_type->get_elements()) { + child_types.push_back(make_table_test_type(child_type)); + } + result = std::make_shared(child_types, struct_type->get_element_names()); + } else if (const auto* array_type = typeid_cast(nested_type.get())) { + result = std::make_shared( + make_table_test_type(array_type->get_nested_type())); + } else if (const auto* map_type = typeid_cast(nested_type.get())) { + result = std::make_shared(make_table_test_type(map_type->get_key_type()), + make_table_test_type(map_type->get_value_type())); + } else { + result = nested_type; + } + return nullable_root ? make_nullable(result) : result; +} + +ColumnDefinition make_table_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition column; + if (id >= 0) { + column.identifier = Field::create_field(id); + } + column.name = name; + // TableReader tests model external table scan descriptors. Those table columns are nullable + // even when the Parquet file field itself is required, so keep the test schema aligned with + // the real scan contract at the construction boundary. + column.type = make_table_test_type(type); + return column; +} + +ColumnDefinition make_file_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition field; + field.identifier = Field::create_field(id); + field.local_id = id; + field.name = name; + field.type = make_table_test_type(type); + return field; +} + +void set_name_identifiers(std::vector* columns); + +void set_name_identifier(ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + column->identifier = Field::create_field(column->name); + set_name_identifiers(&column->children); +} + +void set_name_identifiers(std::vector* columns) { + DORIS_CHECK(columns != nullptr); + for (auto& column : *columns) { + set_name_identifier(&column); + } +} + +VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { + auto ctx = VExprContext::create_shared(expr); + auto status = ctx->prepare(state, RowDescriptor()); + EXPECT_TRUE(status.ok()) << status; + status = ctx->open(state); + EXPECT_TRUE(status.ok()) << status; + return ctx; +} + +void apply_final_conjuncts(Block* block, const VExprContextSPtrs& conjuncts) { + const auto status = VExprContext::filter_block(conjuncts, block, block->columns()); + ASSERT_TRUE(status.ok()) << status; +} + +TEST(IcebergV2ReaderTest, IcebergVirtualColumnsUseRowLineageMetadata) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_virtual_columns_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); + projected_columns.push_back(make_iceberg_last_updated_sequence_number_column()); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(2, 2, 1))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + set_iceberg_row_lineage_params(&split_options, 1000, 77); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& id_column = assert_cast(expect_not_null_table_column(block, 2)); + + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(id_column.get_element(0), 2); + EXPECT_EQ(id_column.get_element(1), 3); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001, 1002}); + expect_nullable_int64_column_values(*block.get_by_position(1).column, {77, 77}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergRowLineageUsesPhysicalRowIdAndFillsNulls) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_physical_row_id_fill_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_iceberg_row_lineage_parquet_file(file_path, {1, 2, 3}, {7000, std::nullopt, 7002}, + {80, std::nullopt, 82}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column( + 2147483540, "_row_id", make_nullable(std::make_shared()))); + projected_columns.push_back( + make_table_column(2147483539, "_last_updated_sequence_number", + make_nullable(std::make_shared()))); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + set_iceberg_row_lineage_params(&split_options, 1000, 77); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + ASSERT_EQ(block.rows(), 3); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {7000, 1001, 7002}); + expect_nullable_int64_column_values(*block.get_by_position(1).column, {80, 77, 82}); + expect_int32_column_values(*block.get_by_position(2).column, {1, 2, 3}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPhysicalRowIdKeepsNullsWithoutFirstRowId) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_physical_row_id_no_first_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_iceberg_row_lineage_parquet_file(file_path, {1, 2, 3}, {7000, std::nullopt, 7002}, + {80, std::nullopt, 82}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column( + 2147483540, "_row_id", make_nullable(std::make_shared()))); + projected_columns.push_back( + make_table_column(2147483539, "_last_updated_sequence_number", + make_nullable(std::make_shared()))); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + ASSERT_EQ(block.rows(), 3); + expect_nullable_int64_column_optional_values( + *block.get_by_position(0).column, + std::vector> {7000, std::nullopt, 7002}); + expect_nullable_int64_column_optional_values( + *block.get_by_position(1).column, + std::vector> {80, std::nullopt, 82}); + expect_int32_column_values(*block.get_by_position(2).column, {1, 2, 3}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergMissingRowIdStaysNullWithoutFirstRowId) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_missing_row_id_no_first_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); + projected_columns.push_back(make_iceberg_last_updated_sequence_number_column()); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + ASSERT_EQ(block.rows(), 3); + expect_nullable_int64_column_optional_values( + *block.get_by_position(0).column, + std::vector> {std::nullopt, std::nullopt, std::nullopt}); + expect_nullable_int64_column_optional_values( + *block.get_by_position(1).column, + std::vector> {std::nullopt, std::nullopt, std::nullopt}); + expect_int32_column_values(*block.get_by_position(2).column, {1, 2, 3}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergRowIdPredicateFiltersAfterRowLineageMaterialization) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_row_id_finalize_filter_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_iceberg_row_lineage_parquet_file(file_path, {1, 2, 3}, {7000, std::nullopt, 7002}, + {80, std::nullopt, 82}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column( + 2147483540, "_row_id", make_nullable(std::make_shared()))); + projected_columns.push_back( + make_table_column(2147483539, "_last_updated_sequence_number", + make_nullable(std::make_shared()))); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + VExprContextSPtrs conjuncts = {prepared_conjunct( + &state, + table_nullable_int64_binary_predicate("eq", TExprOpcode::EQ, 0, 0, "_row_id", 1001))}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = conjuncts, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + set_iceberg_row_lineage_params(&split_options, 1000, 77); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 3); + + apply_final_conjuncts(&block, conjuncts); + ASSERT_EQ(block.rows(), 1); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); + expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); + expect_int32_column_values(*block.get_by_position(2).column, {2}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergLastUpdatedSequencePredicateFiltersAfterMaterialization) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_sequence_finalize_filter_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_iceberg_row_lineage_parquet_file(file_path, {1, 2, 3}, {7000, std::nullopt, 7002}, + {80, std::nullopt, 82}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column( + 2147483540, "_row_id", make_nullable(std::make_shared()))); + projected_columns.push_back( + make_table_column(2147483539, "_last_updated_sequence_number", + make_nullable(std::make_shared()))); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + VExprContextSPtrs conjuncts = {prepared_conjunct( + &state, table_nullable_int64_binary_predicate("eq", TExprOpcode::EQ, 1, 1, + "_last_updated_sequence_number", 77))}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = conjuncts, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + set_iceberg_row_lineage_params(&split_options, 1000, 77); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 3); + + apply_final_conjuncts(&block, conjuncts); + ASSERT_EQ(block.rows(), 1); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); + expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); + expect_int32_column_values(*block.get_by_position(2).column, {2}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergRowidVirtualColumnUsesDataFilePosition) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_rowid_virtual_column_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back( + make_table_column(-1, BeConsts::ICEBERG_ROWID_COL, make_iceberg_rowid_type())); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(1, 1, 1))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + const auto original_file_path = "s3://bucket/table/data/original.parquet"; + const auto partition_data_json = R"({"part":"p1"})"; + set_iceberg_rowid_params(&split_options, original_file_path, 17, partition_data_json); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + ASSERT_EQ(block.rows(), 2); + expect_iceberg_rowid_column_values(*block.get_by_position(0).column, original_file_path, {1, 2}, + 17, partition_data_json); + expect_int32_column_values(*block.get_by_position(1).column, {2, 3}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergVirtualColumnsKeepRowLineageAfterConjunctFiltering) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_virtual_columns_conjunct_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); + projected_columns.push_back(make_iceberg_last_updated_sequence_number_column()); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(2, 2, 1))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + set_iceberg_row_lineage_params(&split_options, 3000, 88); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& id_column = assert_cast(expect_not_null_table_column(block, 2)); + + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(id_column.get_element(0), 2); + EXPECT_EQ(id_column.get_element(1), 3); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {3001, 3002}); + expect_nullable_int64_column_values(*block.get_by_position(1).column, {88, 88}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergVirtualColumnsKeepRowLineageAfterRowGroupPredicatePruning) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_virtual_columns_row_group_predicate_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + // Keep one row per row group so the VExpr ZoneMap path can prune the first two row groups and + // leave only the third file-local row. + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}, 1); + + std::vector projected_columns; + projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); + projected_columns.push_back(make_iceberg_last_updated_sequence_number_column()); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(2, 2, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + set_iceberg_row_lineage_params(&split_options, 4000, 99); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& id_column = assert_cast(expect_not_null_table_column(block, 2)); + + ASSERT_EQ(block.rows(), 1); + EXPECT_EQ(id_column.get_element(0), 3); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {4002}); + expect_nullable_int64_column_values(*block.get_by_position(1).column, {99}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergDeletionVectorUsesTableReaderDeleteFileInterface) { + TTableFormatFileDesc table_format_desc; + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_format_version(2); + iceberg_desc.__set_original_file_path("data-a.parquet"); + iceberg_desc.__set_delete_files({make_iceberg_deletion_vector("dv.bin", 8, 128)}); + table_format_desc.__set_iceberg_params(iceberg_desc); + + IcebergTableReaderDeleteFileTestHelper reader; + DeleteFileDesc desc; + bool has_delete_file = false; + ASSERT_TRUE(reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file).ok()); + + EXPECT_TRUE(has_delete_file); + EXPECT_EQ(desc.path, "dv.bin"); + EXPECT_EQ(desc.start_offset, 8); + EXPECT_EQ(desc.size, 128); + EXPECT_EQ(desc.file_size, -1); + EXPECT_EQ(desc.format, DeleteFileDesc::Format::ICEBERG); +} + +TEST(IcebergV2ReaderTest, IcebergDeletionVectorCacheKeyIncludesDataFileAndRange) { + // Scenario: Iceberg Puffin deletion vectors are scoped to the data file. The same Puffin blob + // referenced by different data files, offsets or lengths must not share a DeleteFileDesc key. + const auto shared_dv = + make_iceberg_deletion_vector("s3://bucket/table/shared-dv.puffin", 8, 128); + IcebergTableReaderDeleteFileTestHelper reader; + + DeleteFileDesc first_desc; + bool has_delete_file = false; + ASSERT_TRUE(reader.parse_deletion_vector_file( + make_iceberg_table_format_desc("s3://bucket/table/data-a.parquet", + {shared_dv}), + &first_desc, &has_delete_file) + .ok()); + EXPECT_TRUE(has_delete_file); + + DeleteFileDesc different_data_file_desc; + ASSERT_TRUE(reader.parse_deletion_vector_file( + make_iceberg_table_format_desc("s3://bucket/table/data-b.parquet", + {shared_dv}), + &different_data_file_desc, &has_delete_file) + .ok()); + + DeleteFileDesc different_offset_desc; + ASSERT_TRUE(reader.parse_deletion_vector_file( + make_iceberg_table_format_desc( + "s3://bucket/table/data-a.parquet", + {make_iceberg_deletion_vector( + "s3://bucket/table/shared-dv.puffin", 16, 128)}), + &different_offset_desc, &has_delete_file) + .ok()); + + DeleteFileDesc different_length_desc; + ASSERT_TRUE(reader.parse_deletion_vector_file( + make_iceberg_table_format_desc( + "s3://bucket/table/data-a.parquet", + {make_iceberg_deletion_vector( + "s3://bucket/table/shared-dv.puffin", 8, 256)}), + &different_length_desc, &has_delete_file) + .ok()); + + EXPECT_NE(first_desc.key, different_data_file_desc.key); + EXPECT_NE(first_desc.key, different_offset_desc.key); + EXPECT_NE(first_desc.key, different_length_desc.key); +} + +TEST(IcebergV2ReaderTest, IcebergDeletionVectorRejectsMultipleDeleteFiles) { + TTableFormatFileDesc table_format_desc; + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_format_version(2); + iceberg_desc.__set_delete_files({make_iceberg_deletion_vector("dv-a.bin", 8, 128), + make_iceberg_deletion_vector("dv-b.bin", 16, 256)}); + table_format_desc.__set_iceberg_params(iceberg_desc); + + IcebergTableReaderDeleteFileTestHelper reader; + DeleteFileDesc desc; + bool has_delete_file = false; + auto status = reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file); + + EXPECT_FALSE(status.ok()); +} + +TEST(IcebergV2ReaderTest, IcebergDeletionVectorRejectsMissingRange) { + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(3); + delete_file.__set_path("dv.bin"); + + TTableFormatFileDesc table_format_desc; + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_format_version(2); + iceberg_desc.__set_delete_files({delete_file}); + table_format_desc.__set_iceberg_params(iceberg_desc); + + IcebergTableReaderDeleteFileTestHelper reader; + DeleteFileDesc desc; + bool has_delete_file = false; + auto status = reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file); + + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("missing content offset or length"), std::string::npos); + EXPECT_FALSE(has_delete_file); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesDeletionVectorFile) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto dv_path = (test_dir / "delete-vector.bin").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {0, 4}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_deletion_vector(dv_path, 0, dv_size)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2, 3, 4})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderReportsInjectedDeletionVectorReadError) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_v2_deletion_vector_injected_read_error_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto dv_path = (test_dir / "delete-vector.bin").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {0}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_deletion_vector(dv_path, 0, dv_size)})); + + ScopedDebugPoint debug_point("TableReader.parse_deletion_vector.io_error"); + auto status = reader.prepare_split(split_options); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("injected format v2 deletion vector read failure"), + std::string::npos); + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderStopsDuringDeletionVectorRead) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_v2_deletion_vector_should_stop_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto dv_path = (test_dir / "delete-vector.bin").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {0}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_deletion_vector(dv_path, 0, dv_size)})); + + ScopedDebugPoint debug_point("TableReader.parse_deletion_vector.should_stop"); + auto status = reader.prepare_split(split_options); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("stop read"), std::string::npos); + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderRejectsCorruptDeletionVectorPayload) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_v2_corrupt_dv_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto dv_path = (test_dir / "delete-vector.bin").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + std::vector corrupted(12, 0); + BigEndian::Store32(corrupted.data(), 4); + memcpy(corrupted.data() + 4, "BAD!", 4); + { + std::ofstream output(dv_path, std::ios::binary); + ASSERT_TRUE(output.is_open()); + output.write(corrupted.data(), static_cast(corrupted.size())); + ASSERT_TRUE(output.good()); + } + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, + {make_iceberg_deletion_vector(dv_path, 0, static_cast(corrupted.size()))})); + auto status = reader.prepare_split(split_options); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("magic number mismatch"), std::string::npos); + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderDoesNotPushDownAggregateWithDeletes) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_aggregate_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto dv_path = (test_dir / "delete-vector.bin").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {0}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_deletion_vector(dv_path, 0, dv_size)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(id_column.get_element(0), 2); + EXPECT_EQ(id_column.get_element(1), 3); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +// Covers TopN lazy materialization on Iceberg schema-evolution tables. The first-phase scan adds a +// synthesized GLOBAL_ROWID column to the file schema. That virtual column must not make Iceberg +// fall back from field-id mapping to name mapping, otherwise renamed columns are read as defaults +// from old files. +TEST(IcebergV2ReaderTest, IcebergMappingModeIgnoresGlobalRowIdVirtualColumn) { + IcebergTableReaderMappingModeTestHelper reader; + std::vector file_schema { + make_file_column(1, "id", std::make_shared()), + make_file_column(2, "name", std::make_shared()), + global_rowid_column_definition(), + }; + + EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema)), + TableColumnMappingMode::BY_FIELD_ID); +} + +// Covers the fallback side of the previous case. Only synthesized columns are ignored; a real data +// column without an Iceberg field id still disables field-id mapping. +TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { + IcebergTableReaderMappingModeTestHelper reader; + std::vector file_schema { + make_file_column(1, "id", std::make_shared()), + make_file_column(2, "name", std::make_shared()), + global_rowid_column_definition(), + }; + file_schema[1].identifier = Field {}; + + EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema)), + TableColumnMappingMode::BY_NAME); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderDoesNotPushDownAggregateWithPositionDelete) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_aggregate_position_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 3); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableLevelCountUsesAssignedRowCountWithPositionDelete) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_table_level_count_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + TQueryOptions query_options; + query_options.__set_batch_size(10); + RuntimeState state {query_options, TQueryGlobals()}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + set_table_level_row_count(&split_options, 5); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.rows(), 5); + + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 0); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteFallsBackToSplitPath) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_path_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + TTableFormatFileDesc table_format_params; + TIcebergFileDesc iceberg_params; + iceberg_params.__set_format_version(2); + iceberg_params.__set_delete_files({make_iceberg_position_delete_file(delete_file_path)}); + table_format_params.__set_iceberg_params(iceberg_params); + split_options.current_range.__set_table_format_params(table_format_params); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderDoesNotPushDownAggregateWithEqualityDelete) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_aggregate_equality_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 3); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteCastsDataColumnToDeleteKeyType) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_cast_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_iceberg_equality_delete_bigint_parquet_file(delete_file_path, 0, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesNullForMissingDataColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_column_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_null_equality_delete_parquet_file(delete_file_path, 1, "added_column"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + // Iceberg treats a field missing from an older data file as NULL. The NULL equality-delete + // key therefore matches every row in this file. + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesInitialDefaultForMissingDataColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 1, 7, "added_column"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info( + {external_schema(100, {external_schema_field("id", 0), + external_schema_field("added_column", 1, {}, "7")})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + // The old file has no physical added_column, but Iceberg defines every row as the initial + // default 7. The equality delete key 7 therefore removes the entire file. + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesTimestampInitialDefaultForMissingColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_timestamp_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_timestamp_equality_delete_parquet_file(delete_file_path, 1, + 1'704'067'200'123'456L, "added_timestamp"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("id", 0), + external_schema_field("added_timestamp", 1, {}, "2024-01-01 00:00:00.123456", + external_primitive_type(TPrimitiveType::DATETIMEV2, -1, 6))})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultForMissingColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_varbinary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesStringMappedBinaryInitialDefault) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_binary_string_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::STRING), true)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteDecodesBinaryDefaultMappedToString) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_string_binary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::STRING), true)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeletePreservesTimestampTzAcrossDstFold) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_timestamp_tz_dst_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + // These instants are the two occurrences of 01:30 during the 2024 New York DST fall-back. + constexpr int64_t FIRST_0130_MICROS = 1'730'611'800'000'000L; + constexpr int64_t SECOND_0130_MICROS = 1'730'615'400'000'000L; + write_timestamp_int_parquet_file(file_path, {FIRST_0130_MICROS, SECOND_0130_MICROS}, {1, 2}); + write_iceberg_timestamp_equality_delete_parquet_file(delete_file_path, 0, FIRST_0130_MICROS, + "event_time", true); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_enable_mapping_timestamp_tz(true); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("event_time", 0, {}, std::nullopt, + external_primitive_type(TPrimitiveType::TIMESTAMPTZ, -1, 6)), + external_schema_field("id", 1)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("America/New_York"); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldIds) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + std::vector projected_columns; + auto id_column = make_table_column(0, "current_id", std::make_shared()); + id_column.name_mapping.push_back("legacy_id"); + projected_columns.push_back(std::move(id_column)); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_stale_field_id_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + // The real key has no id and is mapped by its historical name. A different physical column + // carries the stale id 0, forcing the entire split into BY_NAME mode. + write_two_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt, "stale_key", + {100, 200, 300}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + std::vector projected_columns; + auto id_column = make_table_column(0, "current_id", std::make_shared()); + id_column.name_mapping.push_back("legacy_id"); + projected_columns.push_back(std::move(id_column)); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesUnprojectedTableSchemaNameMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_hidden_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_two_int_parquet_file(file_path, "LEGACY_ID", {1, 2, 3}, std::nullopt, "data", + {10, 20, 30}, std::nullopt); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + // Model SELECT data: the equality key is deliberately absent from the query projection. + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "data", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info( + {external_schema(100, {external_schema_field("current_id", 0, {"legacy_id"}), + external_schema_field("data", 1)})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({10, 30})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteFileIsReusedAcrossSplits) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_split_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // Removing the source after the first split proves that the second split consumes the parsed + // delete block from SplitReadOptions.cache instead of reopening the delete file. + ASSERT_TRUE(std::filesystem::remove(delete_file_path)); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteCacheIsScopedByFileSystem) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_filesystem_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_fs_name("filesystem-a"); + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // The same descriptor path resolves to different content in another filesystem namespace. + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 3); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_fs_name("filesystem-b"); + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteOnlyMatchesOriginalDataFilePath) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_path_match_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto other_file_path = (test_dir / "other.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {other_file_path, file_path}, {0, 1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergRowLineageRemainsFileLocalAfterDeleteFiltering) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_row_lineage_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + TTableFormatFileDesc table_format_params = make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)}); + table_format_params.iceberg_params.__set_first_row_id(1000); + split_options.current_range.__set_table_format_params(table_format_params); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {1000, 1002}); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 3); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + write_position_delete_parquet_file(delete_file_path, {file_path, file_path}, {1, 3}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteHonorsPositionBounds) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_bounds_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + write_position_delete_parquet_file(delete_file_path, {file_path, file_path, file_path}, + {0, 2, 4}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto delete_file = make_iceberg_position_delete_file(delete_file_path); + delete_file.__set_position_lower_bound(1); + delete_file.__set_position_upper_bound(3); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {delete_file})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2, 4, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteFileIsReusedAcrossSplits) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_split_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_position_delete_parquet_file(delete_file_path, {first_file_path, second_file_path}, + {1, 0}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // The cached delete file contains entries for every referenced data file, so another split can + // select its own positions without rescanning the shared delete file. + ASSERT_TRUE(std::filesystem::remove(delete_file_path)); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteCacheIsScopedByFileSystem) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_filesystem_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_position_delete_parquet_file(delete_file_path, {first_file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_fs_name("filesystem-a"); + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + write_position_delete_parquet_file(delete_file_path, {second_file_path}, {0}); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_fs_name("filesystem-b"); + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderIgnoresPositionDeleteFilesWhenDeletionVectorPresent) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_delete_files_merge_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto position_delete_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {0}); + write_position_delete_parquet_file(position_delete_path, {file_path, file_path}, {3, 3}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_deletion_vector(dv_path, 0, dv_size), + make_iceberg_position_delete_file(position_delete_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2, 3, 4, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, RowPositionDeletePredicateColumnIsNotRepeatedAsOutputColumn) { + const auto row_position_column_id = ROW_POSITION_COLUMN_ID; + std::vector projected_columns; + projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + IcebergTableReaderScanRequestTestHelper reader; + ASSERT_TRUE(reader.init_for_scan_request_test(projected_columns).ok()); + + FileScanRequest request; + request.non_predicate_columns.push_back(field_projection(0)); + request.local_positions.emplace(LocalColumnId(0), LocalIndex(0)); + + ASSERT_TRUE(reader.customize_request(&request).ok()); + + EXPECT_EQ(projection_ids(request.predicate_columns), + std::vector({row_position_column_id})); + EXPECT_EQ(projection_ids(request.non_predicate_columns), std::vector({0})); + ASSERT_TRUE(request.local_positions.contains(LocalColumnId(row_position_column_id))); + EXPECT_EQ(request.local_positions.at(LocalColumnId(row_position_column_id)).value(), 1); + ASSERT_TRUE(request.conjuncts.empty()); + ASSERT_EQ(request.delete_conjuncts.size(), 1); + EXPECT_NE(request.delete_conjuncts[0], nullptr); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp new file mode 100644 index 00000000000000..f97835f649bee4 --- /dev/null +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -0,0 +1,729 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/paimon_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "exec/common/endian.h" +#include "format/format_common.h" +#include "format/table/paimon_reader.h" +#include "format_v2/column_data.h" +#include "format_v2/deletion_vector_reader.h" +#include "format_v2/jni/paimon_jni_reader.h" +#include "gen_cpp/ExternalTableSchema_types.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/io_common.h" +#include "roaring/roaring.hh" +#include "runtime/exec_env.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/options.h" + +namespace doris::format { +namespace { + +DataTypePtr table_type(const DataTypePtr& type) { + return type->is_nullable() ? type : make_nullable(type); +} + +ColumnDefinition make_table_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition column; + column.identifier = Field::create_field(id); + column.name = name; + column.type = table_type(type); + return column; +} + +ColumnDefinition make_file_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition column; + column.identifier = Field::create_field(id); + column.local_id = id; + column.name = name; + column.type = type; + return column; +} + +schema::external::TFieldPtr external_schema_field(std::string name, int32_t id, + std::vector aliases = {}) { + auto field = std::make_shared(); + field->__set_name(std::move(name)); + field->__set_id(id); + if (!aliases.empty()) { + field->__set_name_mapping(std::move(aliases)); + } + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; +} + +schema::external::TFieldPtr external_array_field(std::string name, int32_t id, + schema::external::TFieldPtr item_field, + std::vector aliases = {}) { + auto field = external_schema_field(std::move(name), id, std::move(aliases)); + schema::external::TArrayField array_field; + array_field.__set_item_field(std::move(item_field)); + field.field_ptr->nestedField.__set_array_field(std::move(array_field)); + field.field_ptr->__isset.nestedField = true; + return field; +} + +schema::external::TFieldPtr external_map_field(std::string name, int32_t id, + schema::external::TFieldPtr key_field, + schema::external::TFieldPtr value_field, + std::vector aliases = {}) { + auto field = external_schema_field(std::move(name), id, std::move(aliases)); + schema::external::TMapField map_field; + map_field.__set_key_field(std::move(key_field)); + map_field.__set_value_field(std::move(value_field)); + field.field_ptr->nestedField.__set_map_field(std::move(map_field)); + field.field_ptr->__isset.nestedField = true; + return field; +} + +schema::external::TSchema external_schema(int64_t schema_id, + std::vector fields) { + schema::external::TStructField root_field; + root_field.__set_fields(std::move(fields)); + schema::external::TSchema schema; + schema.__set_schema_id(schema_id); + schema.__set_root_field(std::move(root_field)); + return schema; +} + +Block build_table_block(const std::vector& columns) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +const IColumn& expect_not_null_nullable_nested_column(const IColumn& column) { + if (!column.is_nullable()) { + return column; + } + const auto& nullable_column = assert_cast(column); + for (const auto is_null : nullable_column.get_null_map_data()) { + EXPECT_EQ(is_null, 0); + } + return nullable_column.get_nested_column(); +} + +const IColumn& expect_not_null_table_column(const Block& block, size_t position) { + return expect_not_null_nullable_nested_column(*block.get_by_position(position).column); +} + +std::shared_ptr build_int32_array(const std::vector& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +std::shared_ptr build_string_array(const std::vector& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +void write_int_pair_parquet_file(const std::string& file_path, const std::vector& ids, + const std::vector& scores, + const std::vector& values) { + ASSERT_EQ(ids.size(), scores.size()); + ASSERT_EQ(ids.size(), values.size()); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("score", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(scores), + build_string_array(values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(ids.size()), + builder.build())); +} + +std::vector build_paimon_deletion_vector_buffer( + const std::vector& deleted_positions) { + roaring::Roaring rows; + for (const auto position : deleted_positions) { + rows.add(position); + } + + const size_t bitmap_size = rows.getSizeInBytes(); + const uint32_t total_length = static_cast(4 + bitmap_size); + std::vector blob(4 + total_length); + BigEndian::Store32(blob.data(), total_length); + constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'}; + memcpy(blob.data() + 4, PAIMON_BITMAP_MAGIC, 4); + rows.write(blob.data() + 8); + return blob; +} + +int64_t write_paimon_deletion_vector_file(const std::string& file_path, + const std::vector& deleted_positions) { + const auto blob = build_paimon_deletion_vector_buffer(deleted_positions); + std::ofstream output(file_path, std::ios::binary); + EXPECT_TRUE(output.is_open()); + output.write(blob.data(), static_cast(blob.size())); + EXPECT_TRUE(output.good()); + // Paimon DeletionFile.length is magic + bitmap length, excluding the leading length field. + return static_cast(blob.size() - 4); +} + +TFileScanRangeParams make_local_parquet_scan_params() { + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + return scan_params; +} + +std::shared_ptr make_io_context(io::FileReaderStats* file_reader_stats, + io::FileCacheStatistics* file_cache_stats) { + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = file_reader_stats; + io_ctx->file_cache_stats = file_cache_stats; + return io_ctx; +} + +SplitReadOptions build_split_options(const std::string& file_path) { + SplitReadOptions options; + options.current_range.__set_path(file_path); + options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + return options; +} + +TTableFormatFileDesc make_paimon_table_format_desc(const std::string& deletion_file_path, + int64_t offset, int64_t length) { + TTableFormatFileDesc table_format_params; + TPaimonFileDesc paimon_params; + paimon_params.__set_file_format("parquet"); + TPaimonDeletionFileDesc deletion_file; + deletion_file.__set_path(deletion_file_path); + deletion_file.__set_offset(offset); + deletion_file.__set_length(length); + paimon_params.__set_deletion_file(deletion_file); + table_format_params.__set_paimon_params(paimon_params); + return table_format_params; +} + +TTableFormatFileDesc make_paimon_schema_table_format_desc(int64_t schema_id) { + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("paimon"); + TPaimonFileDesc paimon_params; + paimon_params.__set_file_format("parquet"); + paimon_params.__set_schema_id(schema_id); + table_format_params.__set_paimon_params(paimon_params); + return table_format_params; +} + +TFileRangeDesc make_paimon_native_range(TFileFormatType::type format_type) { + TFileRangeDesc range; + range.__set_path(format_type == TFileFormatType::FORMAT_ORC ? "s3://bucket/native.orc" + : "s3://bucket/native.parquet"); + range.__set_format_type(format_type); + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("paimon"); + TPaimonFileDesc paimon_params; + paimon_params.__set_file_format(format_type == TFileFormatType::FORMAT_ORC ? "orc" : "parquet"); + paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_NATIVE); + table_format_params.__set_paimon_params(paimon_params); + range.__set_table_format_params(table_format_params); + return range; +} + +TFileRangeDesc make_paimon_jni_range() { + TFileRangeDesc range; + range.__set_path("/data-placeholder.parquet"); + range.__set_format_type(TFileFormatType::FORMAT_JNI); + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("paimon"); + TPaimonFileDesc paimon_params; + paimon_params.__set_file_format("parquet"); + paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_JNI); + paimon_params.__set_paimon_split("serialized-paimon-split"); + table_format_params.__set_paimon_params(paimon_params); + range.__set_table_format_params(table_format_params); + return range; +} + +TFileRangeDesc make_paimon_range_without_reader_type(TFileFormatType::type format_type) { + TFileRangeDesc range = make_paimon_native_range(format_type); + range.table_format_params.paimon_params.__isset.reader_type = false; + return range; +} + +TFileScanRangeParams make_paimon_jni_scan_params() { + TFileScanRangeParams scan_params; + scan_params.__set_serialized_table("serialized-paimon-table"); + scan_params.__set_paimon_predicate("serialized-paimon-predicate"); + return scan_params; +} + +std::map build_paimon_jni_scanner_params( + TFileScanRangeParams* scan_params, RuntimeState* state) { + paimon::PaimonJniReader reader; + reader.TEST_set_scan_params(scan_params); + reader.TEST_set_runtime_state(state); + reader.TEST_set_current_range(make_paimon_jni_range()); + std::map params; + EXPECT_TRUE(reader.TEST_build_scanner_params(¶ms).ok()); + return params; +} + +class ScopedExecEnvStorePaths { +public: + explicit ScopedExecEnvStorePaths(std::vector store_paths) { + _current = &const_cast&>(ExecEnv::GetInstance()->store_paths()); + _previous = *_current; + *_current = std::move(store_paths); + } + + ~ScopedExecEnvStorePaths() { *_current = std::move(_previous); } + +private: + std::vector* _current = nullptr; + std::vector _previous; +}; + +// Scenario: PaimonReader shares Hudi's history-schema annotation path. A split whose schema id +// resolves to a historical schema should use field-id mapping and annotate array/map children so +// TableColumnMapper can match evolved physical Parquet columns by id instead of by the old names. +TEST(PaimonReaderTest, AnnotatesArrayAndMapFileSchemaFromSplitHistorySchema) { + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(200); + scan_params.__set_history_schema_info({ + external_schema( + 100, + {external_array_field("old_tags", 30, + external_schema_field("old_item", 31, {"tag"}), {"tags"}), + external_map_field( + "old_props", 40, external_schema_field("old_key", 41, {"key"}), + external_schema_field("old_value", 42, {"score"}), {"props"})}), + external_schema( + 200, {external_schema_field("tags", 30), external_schema_field("props", 40)}), + }); + + paimon::PaimonReader reader; + reader.TEST_set_scan_params(&scan_params); + + SplitReadOptions split_options; + split_options.current_range.__set_table_format_params( + make_paimon_schema_table_format_desc(100)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_FIELD_ID); + + const auto string_type = std::make_shared(); + const auto int_type = std::make_shared(); + + auto tags = make_file_column(0, "old_tags", std::make_shared(string_type)); + tags.children = {make_file_column(0, "old_item", string_type)}; + + auto props = + make_file_column(1, "old_props", std::make_shared(string_type, int_type)); + props.children = {make_file_column(0, "old_key", string_type), + make_file_column(1, "old_value", int_type)}; + + std::vector file_schema {tags, props}; + ASSERT_TRUE(reader.TEST_annotate_file_schema(&file_schema).ok()); + + ASSERT_EQ(file_schema.size(), 2); + EXPECT_EQ(file_schema[0].get_identifier_field_id(), 30); + EXPECT_EQ(file_schema[0].name_mapping, std::vector({"tags"})); + ASSERT_EQ(file_schema[0].children.size(), 1); + EXPECT_EQ(file_schema[0].children[0].get_identifier_field_id(), 31); + EXPECT_EQ(file_schema[0].children[0].name_mapping, std::vector({"tag"})); + + EXPECT_EQ(file_schema[1].get_identifier_field_id(), 40); + EXPECT_EQ(file_schema[1].name_mapping, std::vector({"props"})); + ASSERT_EQ(file_schema[1].children.size(), 2); + EXPECT_EQ(file_schema[1].children[0].get_identifier_field_id(), 41); + EXPECT_EQ(file_schema[1].children[0].name_mapping, std::vector({"key"})); + EXPECT_EQ(file_schema[1].children[1].get_identifier_field_id(), 42); + EXPECT_EQ(file_schema[1].children[1].name_mapping, std::vector({"score"})); +} + +// Scenario: when FE does not send a matching historical schema for the split schema id, Paimon must +// stay on BY_NAME mapping and must not rewrite the file schema identifiers. +TEST(PaimonReaderTest, FallsBackToByNameWhenSplitHistorySchemaIsMissing) { + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(200); + scan_params.__set_history_schema_info({ + external_schema(200, {external_schema_field("name", 10)}), + }); + + paimon::PaimonReader reader; + reader.TEST_set_scan_params(&scan_params); + + SplitReadOptions split_options; + split_options.current_range.__set_table_format_params( + make_paimon_schema_table_format_desc(100)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_NAME); + + std::vector file_schema { + make_file_column(0, "old_name", std::make_shared()), + }; + ASSERT_TRUE(reader.TEST_annotate_file_schema(&file_schema).ok()); + EXPECT_EQ(file_schema[0].get_identifier_field_id(), 0); + EXPECT_TRUE(file_schema[0].name_mapping.empty()); +} + +TEST(PaimonReaderTest, DeletionVectorCacheKeyIncludesOffsetAndLength) { + // Scenario: format_v2 converts Paimon split metadata into a generic DeleteFileDesc. The + // generated key must preserve offset and length so shared DV files do not collide. + TTableFormatFileDesc table_format_params; + table_format_params.__isset.paimon_params = true; + TPaimonDeletionFileDesc deletion_file; + deletion_file.__set_path("s3://bucket/table/deletion.dv"); + deletion_file.__set_offset(128); + deletion_file.__set_length(64); + table_format_params.paimon_params.__set_deletion_file(deletion_file); + + paimon::PaimonReader reader; + DeleteFileDesc first_desc; + bool has_delete_file = false; + ASSERT_TRUE(reader.TEST_parse_deletion_vector_file(table_format_params, &first_desc, + &has_delete_file) + .ok()); + EXPECT_TRUE(has_delete_file); + + table_format_params.paimon_params.deletion_file.__set_offset(256); + DeleteFileDesc different_offset_desc; + ASSERT_TRUE(reader.TEST_parse_deletion_vector_file(table_format_params, &different_offset_desc, + &has_delete_file) + .ok()); + + table_format_params.paimon_params.deletion_file.__set_offset(128); + table_format_params.paimon_params.deletion_file.__set_length(96); + DeleteFileDesc different_length_desc; + ASSERT_TRUE(reader.TEST_parse_deletion_vector_file(table_format_params, &different_length_desc, + &has_delete_file) + .ok()); + + EXPECT_NE(first_desc.key, different_offset_desc.key); + EXPECT_NE(first_desc.key, different_length_desc.key); +} + +TEST(PaimonReaderTest, DecodeDeletionVectorBufferUsesSharedFormatHelper) { + // Scenario: format_v2 TableReader reads a raw Paimon BitmapDeletionVector range and delegates + // the binary parsing to the same helper used by the format reader path. + const auto buffer = build_paimon_deletion_vector_buffer({0, 3, 5}); + DeletionVector deletion_vector; + + ASSERT_TRUE(decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector) + .ok()); + EXPECT_EQ(deletion_vector.cardinality(), 3); + EXPECT_TRUE(deletion_vector.contains(uint64_t {0})); + EXPECT_TRUE(deletion_vector.contains(uint64_t {3})); + EXPECT_TRUE(deletion_vector.contains(uint64_t {5})); +} + +TEST(PaimonReaderTest, DecodeDeletionVectorBufferRejectsShortBuffer) { + // Scenario: a truncated Paimon DV must fail before reading the magic or roaring payload. + const std::vector buffer = {'\0', '\0', '\0', '\4'}; + DeletionVector deletion_vector; + + EXPECT_FALSE( + decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector) + .ok()); +} + +TEST(PaimonReaderTest, DecodeDeletionVectorBufferRejectsLengthMismatch) { + // Scenario: a cached or remote Paimon DV range with a mismatched leading length must not be + // accepted as a valid bitmap. + auto buffer = build_paimon_deletion_vector_buffer({1, 2}); + BigEndian::Store32(buffer.data(), static_cast(buffer.size())); + DeletionVector deletion_vector; + + EXPECT_FALSE( + decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector) + .ok()); +} + +TEST(PaimonReaderTest, DecodeDeletionVectorBufferRejectsMagicMismatch) { + // Scenario: format_v2 must reject non-Paimon payloads even when the range length is valid. + auto buffer = build_paimon_deletion_vector_buffer({1, 2}); + buffer[4] = '\0'; + DeletionVector deletion_vector; + + EXPECT_FALSE( + decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector) + .ok()); +} + +TEST(PaimonReaderTest, DecodeDeletionVectorBufferRejectsCorruptRoaringBitmap) { + // Scenario: a valid Paimon DV header with a corrupt roaring body should return a data quality + // error instead of producing partial delete rows. + auto buffer = build_paimon_deletion_vector_buffer({1, 2}); + buffer.resize(8); + BigEndian::Store32(buffer.data(), 4); + DeletionVector deletion_vector; + + EXPECT_FALSE( + decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector) + .ok()); +} + +// Scenario: PaimonReader must clear the previous split schema id before reading a new split. A +// schema-evolved split must not force the following split without schema id to keep BY_FIELD_ID. +TEST(PaimonReaderTest, ResetsSplitSchemaIdBeforePreparingNextSplit) { + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(200); + scan_params.__set_history_schema_info({ + external_schema(100, {external_schema_field("old_name", 10, {"name"})}), + external_schema(200, {external_schema_field("name", 10)}), + }); + + paimon::PaimonReader reader; + reader.TEST_set_scan_params(&scan_params); + + SplitReadOptions split_with_schema_id; + split_with_schema_id.current_range.__set_table_format_params( + make_paimon_schema_table_format_desc(100)); + ASSERT_TRUE(reader.prepare_split(split_with_schema_id).ok()); + EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_FIELD_ID); + + SplitReadOptions split_without_schema_id; + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("paimon"); + table_format_params.__set_paimon_params(TPaimonFileDesc {}); + split_without_schema_id.current_range.__set_table_format_params(table_format_params); + ASSERT_TRUE(reader.prepare_split(split_without_schema_id).ok()); + EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_NAME); +} + +// Scenario: Paimon reader should parse its bitmap deletion vector and let TableReader apply the +// generated row-position delete predicate before returning table rows. +TEST(PaimonReaderTest, AppliesBitmapDeletionVectorFile) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_paimon_deletion_vector_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto dv_path = (test_dir / "delete-vector.bin").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + const auto dv_length = write_paimon_deletion_vector_file(dv_path, {0, 4}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + paimon::PaimonReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_paimon_table_format_desc(dv_path, 0, dv_length)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + std::vector ids; + bool eos = false; + while (!eos) { + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (block.rows() == 0) { + continue; + } + const auto& id_column = + assert_cast(expect_not_null_table_column(block, 0)); + for (size_t row = 0; row < block.rows(); ++row) { + ids.push_back(id_column.get_element(row)); + } + } + EXPECT_EQ(ids, std::vector({2, 3, 4})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(PaimonHybridReaderTest, ClassifiesJniSplitByReaderType) { + EXPECT_FALSE(paimon::PaimonHybridReader::TEST_is_jni_split( + make_paimon_native_range(TFileFormatType::FORMAT_PARQUET))); + EXPECT_FALSE(paimon::PaimonHybridReader::TEST_is_jni_split( + make_paimon_range_without_reader_type(TFileFormatType::FORMAT_JNI))); + EXPECT_TRUE(paimon::PaimonHybridReader::TEST_is_jni_split(make_paimon_jni_range())); +} + +TEST(PaimonHybridReaderTest, ConvertsNativeSplitFileFormat) { + FileFormat file_format; + ASSERT_TRUE(paimon::PaimonHybridReader::TEST_to_file_format( + make_paimon_native_range(TFileFormatType::FORMAT_PARQUET), &file_format) + .ok()); + EXPECT_EQ(file_format, FileFormat::PARQUET); + + ASSERT_TRUE(paimon::PaimonHybridReader::TEST_to_file_format( + make_paimon_native_range(TFileFormatType::FORMAT_ORC), &file_format) + .ok()); + EXPECT_EQ(file_format, FileFormat::ORC); + + auto status = + paimon::PaimonHybridReader::TEST_to_file_format(make_paimon_jni_range(), &file_format); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("Unsupported native Paimon file format")); +} + +TEST(PaimonHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { + paimon::PaimonHybridReader reader; + reader.TEST_install_batch_size_children(); + reader.set_batch_size(321); + const auto child_batch_sizes = reader.TEST_child_batch_sizes(); + EXPECT_EQ(child_batch_sizes.first, 321); + EXPECT_EQ(child_batch_sizes.second, 321); +} + +TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + SplitReadOptions native_split; + native_split.current_range = make_paimon_native_range(TFileFormatType::FORMAT_PARQUET); + native_split.current_split_format = FileFormat::PARQUET; + ASSERT_TRUE(reader.prepare_split(native_split).ok()); + + SplitReadOptions jni_split; + jni_split.current_range = make_paimon_jni_range(); + jni_split.current_split_format = FileFormat::JNI; + auto status = reader.prepare_split(jni_split); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("missing serialized_table")); + + ASSERT_TRUE(reader.close().ok()); +} + +TEST(PaimonJniReaderTest, BuildScannerParamsKeepsExplicitIOManagerTempDir) { + auto scan_params = make_paimon_jni_scan_params(); + scan_params.__set_paimon_options({ + {"doris.enable_jni_io_manager", "true"}, + {"doris.jni_io_manager.tmp_dir", "/tmp/explicit-paimon-spill"}, + {"doris.jni_io_manager.impl_class", "org.example.CustomIOManager"}, + }); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_exec_env(ExecEnv::GetInstance()); + + auto params = build_paimon_jni_scanner_params(&scan_params, &state); + EXPECT_EQ(params["paimon.doris.enable_jni_io_manager"], "true"); + EXPECT_EQ(params["paimon.doris.jni_io_manager.tmp_dir"], "/tmp/explicit-paimon-spill"); + EXPECT_EQ(params["paimon.doris.jni_io_manager.impl_class"], "org.example.CustomIOManager"); +} + +TEST(PaimonJniReaderTest, BuildScannerParamsInjectsStorageRootTmpDirForEnabledIOManager) { + ScopedExecEnvStorePaths store_paths({ + StorePath("/data1/doris", -1), + StorePath("/data2/doris", -1), + }); + auto scan_params = make_paimon_jni_scan_params(); + scan_params.__set_paimon_options({ + {"doris.enable_jni_io_manager", "true"}, + }); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_exec_env(ExecEnv::GetInstance()); + + auto params = build_paimon_jni_scanner_params(&scan_params, &state); + EXPECT_EQ(params["paimon.doris.enable_jni_io_manager"], "true"); + EXPECT_EQ(params["paimon.doris.jni_io_manager.tmp_dir"], + "/data1/doris/paimon_jni_scanner_io_tmp:/data2/doris/" + "paimon_jni_scanner_io_tmp"); +} + +TEST(PaimonJniReaderTest, BuildScannerParamsUsesStorageRootTmpDirWhenIOManagerTempDirMissing) { + std::vector paths; + paths.emplace_back("/data1/doris", -1); + paths.emplace_back("/data2/doris", -1); + EXPECT_EQ(paimon::PaimonJniReader::TEST_build_default_io_manager_tmp_dirs(paths), + "/data1/doris/paimon_jni_scanner_io_tmp:/data2/doris/" + "paimon_jni_scanner_io_tmp"); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/table/remote_doris_reader_test.cpp b/be/test/format_v2/table/remote_doris_reader_test.cpp new file mode 100644 index 00000000000000..b17f82f505c2c9 --- /dev/null +++ b/be/test/format_v2/table/remote_doris_reader_test.cpp @@ -0,0 +1,470 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/remote_doris_reader.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/object_pool.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/file_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/file_factory.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "testutil/desc_tbl_builder.h" + +namespace doris::format::remote_doris { +namespace { + +class BatchRemoteDorisStream final : public RemoteDorisStream { +public: + BatchRemoteDorisStream(std::vector> batches, + std::shared_ptr close_count) + : _batches(std::move(batches)), _close_count(std::move(close_count)) {} + + Status next(std::shared_ptr* batch) override { + DORIS_CHECK(batch != nullptr); + if (_next_batch >= _batches.size()) { + *batch = nullptr; + return Status::OK(); + } + *batch = _batches[_next_batch++]; + return Status::OK(); + } + + Status close() override { + ++(*_close_count); + return Status::OK(); + } + +private: + std::vector> _batches; + std::shared_ptr _close_count; + size_t _next_batch = 0; +}; + +TFileRangeDesc remote_doris_range() { + TRemoteDorisFileDesc remote_desc; + remote_desc.__set_location_uri("grpc://127.0.0.1:9050"); + remote_desc.__set_ticket("ticket-bytes"); + + TTableFormatFileDesc table_desc; + table_desc.__set_table_format_type("remote_doris"); + table_desc.__set_remote_doris_params(std::move(remote_desc)); + + TFileRangeDesc range; + range.__set_format_type(TFileFormatType::FORMAT_ARROW); + range.__set_path("/dummyPath"); + range.__set_table_format_params(std::move(table_desc)); + return range; +} + +std::vector remote_slots(ObjectPool* pool, DescriptorTbl** desc_tbl) { + DescriptorTblBuilder builder(pool); + builder.declare_tuple() << std::make_tuple(std::make_shared(), std::string("id")) + << std::make_tuple(std::make_shared(), + std::string("name")); + *desc_tbl = builder.build(); + return (*desc_tbl)->get_tuple_descriptor(0)->slots(); +} + +TSlotDescriptor remote_complex_slot_descriptor(int id, const DataTypePtr& type, + const std::string& name) { + TSlotDescriptor slot_desc; + slot_desc.__set_id(id); + slot_desc.__set_parent(0); + slot_desc.__set_slotType(type->to_thrift()); + slot_desc.__set_byteOffset(0); + slot_desc.__set_nullIndicatorByte(id / 8); + slot_desc.__set_nullIndicatorBit(id % 8); + slot_desc.__set_slotIdx(id); + slot_desc.__set_columnPos(id); + slot_desc.__set_isMaterialized(true); + slot_desc.__set_is_key(false); + slot_desc.__set_colName(name); + slot_desc.__set_col_unique_id(id); + return slot_desc; +} + +std::vector remote_complex_slots(ObjectPool* pool, DescriptorTbl** desc_tbl) { + const auto string_type = make_nullable(std::make_shared()); + const auto int_type = make_nullable(std::make_shared()); + const auto array_type = make_nullable(std::make_shared(string_type)); + const auto map_type = make_nullable(std::make_shared(string_type, int_type)); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {int_type, make_nullable(std::make_shared()), string_type}, + Strings {"f1", "f2", "f3"})); + + TDescriptorTable thrift_desc_tbl; + TTupleDescriptor tuple_desc; + tuple_desc.__set_id(0); + tuple_desc.__set_byteSize(0); + tuple_desc.__set_numNullBytes(1); + thrift_desc_tbl.tupleDescriptors.push_back(std::move(tuple_desc)); + thrift_desc_tbl.slotDescriptors.push_back( + remote_complex_slot_descriptor(0, array_type, "c_array_s")); + thrift_desc_tbl.slotDescriptors.push_back(remote_complex_slot_descriptor(1, map_type, "c_map")); + thrift_desc_tbl.slotDescriptors.push_back( + remote_complex_slot_descriptor(2, struct_type, "c_struct")); + auto status = DescriptorTbl::create(pool, thrift_desc_tbl, desc_tbl); + EXPECT_TRUE(status.ok()) << status; + return (*desc_tbl)->get_tuple_descriptor(0)->slots(); +} + +std::shared_ptr make_batch(const std::vector& names) { + arrow::Int32Builder id_builder; + EXPECT_TRUE(id_builder.Append(10).ok()); + EXPECT_TRUE(id_builder.Append(20).ok()); + std::shared_ptr id_array; + EXPECT_TRUE(id_builder.Finish(&id_array).ok()); + + arrow::StringBuilder name_builder; + EXPECT_TRUE(name_builder.Append("alice").ok()); + EXPECT_TRUE(name_builder.Append("bob").ok()); + std::shared_ptr name_array; + EXPECT_TRUE(name_builder.Finish(&name_array).ok()); + + std::vector> fields; + std::vector> arrays; + for (const auto& name : names) { + if (name == "id") { + fields.push_back(arrow::field("id", arrow::int32())); + arrays.push_back(id_array); + } else if (name == "name") { + fields.push_back(arrow::field("name", arrow::utf8())); + arrays.push_back(name_array); + } else { + fields.push_back(arrow::field(name, arrow::int32())); + arrays.push_back(id_array); + } + } + return arrow::RecordBatch::Make(arrow::schema(std::move(fields)), 2, std::move(arrays)); +} + +std::unique_ptr create_reader( + RuntimeProfile* profile, const TFileRangeDesc& range, + const std::vector& slots, + std::vector> batches, std::shared_ptr close_count, + std::shared_ptr io_ctx = nullptr) { + auto system_properties = std::make_shared(); + auto file_description = std::make_unique(); + file_description->path = "/dummyPath"; + auto factory = [batches = std::move(batches), close_count]( + const TFileRangeDesc&, + std::unique_ptr* stream) mutable { + *stream = std::make_unique(std::move(batches), close_count); + return Status::OK(); + }; + return std::make_unique(system_properties, file_description, + std::move(io_ctx), profile, range, slots, + std::move(factory)); +} + +Block make_request_block(const std::vector& schema, + const std::vector& local_ids) { + Block block; + for (const auto local_id : local_ids) { + const auto it = std::find_if(schema.begin(), schema.end(), [&](const auto& column) { + return column.local_id == local_id; + }); + DORIS_CHECK(it != schema.end()); + block.insert({it->type->create_column(), it->type, it->name}); + } + return block; +} + +int32_t nullable_int_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data()[row]; +} + +std::string nullable_string_at(const IColumn& column, size_t row) { + const auto& nullable = assert_cast(column); + const auto& nested = assert_cast(nullable.get_nested_column()); + return nested.get_data_at(row).to_string(); +} + +class NullableIntGreaterThanExpr final : public VExpr { +public: + NullableIntGreaterThanExpr(size_t block_position, int32_t value) + : VExpr(std::make_shared(), false), + _block_position(block_position), + _value(value) {} + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_block_position).column); + const auto& data = assert_cast(nullable.get_nested_column()); + + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto source_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable.is_null_at(source_row) && data.get_element(source_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_block_position, _value); + return Status::OK(); + } + +private: + size_t _block_position; + int32_t _value; + const std::string _name = "NullableIntGreaterThanExpr"; +}; + +VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { + auto context = VExprContext::create_shared(expr); + auto status = context->prepare(state, RowDescriptor()); + EXPECT_TRUE(status.ok()) << status; + status = context->open(state); + EXPECT_TRUE(status.ok()) << status; + return context; +} + +} // namespace + +TEST(RemoteDorisV2ReaderTest, BuildsSchemaFromSlotsAndProjectsRequestedColumns) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_test"); + auto close_count = std::make_shared(0); + auto reader = create_reader(&profile, remote_doris_range(), slots, {make_batch({"id", "name"})}, + close_count); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(schema[0].name, "id"); + EXPECT_EQ(schema[0].local_id, 0); + EXPECT_EQ(schema[1].name, "name"); + EXPECT_EQ(schema[1].local_id, 1); + + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(1)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_FALSE(eof); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "alice"); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 1), "bob"); + + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + ASSERT_TRUE(reader->close().ok()); + EXPECT_EQ(*close_count, 1); +} + +TEST(RemoteDorisV2ReaderTest, BuildsComplexSchemaChildrenFromSlots) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_complex_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_complex_schema_test"); + auto close_count = std::make_shared(0); + auto reader = create_reader(&profile, remote_doris_range(), slots, {}, close_count); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + + ASSERT_EQ(schema[0].name, "c_array_s"); + ASSERT_EQ(schema[0].children.size(), 1); + EXPECT_EQ(schema[0].children[0].name, "element"); + EXPECT_EQ(schema[0].children[0].local_id, 0); + EXPECT_TRUE(schema[0].children[0].children.empty()); + + ASSERT_EQ(schema[1].name, "c_map"); + ASSERT_EQ(schema[1].children.size(), 2); + EXPECT_EQ(schema[1].children[0].name, "key"); + EXPECT_EQ(schema[1].children[0].local_id, 0); + EXPECT_EQ(schema[1].children[1].name, "value"); + EXPECT_EQ(schema[1].children[1].local_id, 1); + + ASSERT_EQ(schema[2].name, "c_struct"); + ASSERT_EQ(schema[2].children.size(), 3); + EXPECT_EQ(schema[2].children[0].name, "f1"); + EXPECT_EQ(schema[2].children[0].local_id, 0); + EXPECT_EQ(schema[2].children[1].name, "f2"); + EXPECT_EQ(schema[2].children[1].local_id, 1); + EXPECT_EQ(schema[2].children[2].name, "f3"); + EXPECT_EQ(schema[2].children[2].local_id, 2); +} + +TEST(RemoteDorisV2ReaderTest, HandlesDifferentArrowColumnOrder) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_reordered_test"); + auto close_count = std::make_shared(0); + auto reader = create_reader(&profile, remote_doris_range(), slots, {make_batch({"name", "id"})}, + close_count); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(1)).ok()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {1, 0}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 2); + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "alice"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(1).column, 1), 20); +} + +TEST(RemoteDorisV2ReaderTest, AppliesConjunctsAndTracksPredicateFilteredRows) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_filter_test"); + auto close_count = std::make_shared(0); + auto io_ctx = std::make_shared(); + auto reader = create_reader(&profile, remote_doris_range(), slots, {make_batch({"id", "name"})}, + close_count, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(1)).ok()); + request->conjuncts = { + prepared_conjunct(&state, std::make_shared(0, 10))}; + ASSERT_TRUE(reader->open(request).ok()); + + auto block = make_request_block(schema, {0, 1}); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_FALSE(eof); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 20); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "bob"); + EXPECT_EQ(io_ctx->predicate_filtered_rows, 1); +} + +TEST(RemoteDorisV2ReaderTest, RejectsUnknownReturnedColumnAndMissingRequestedColumn) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_error_test"); + + { + auto close_count = std::make_shared(0); + auto reader = create_reader(&profile, remote_doris_range(), slots, + {make_batch({"unknown"})}, close_count); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + auto block = make_request_block(schema, {0}); + size_t rows = 0; + bool eof = false; + EXPECT_FALSE(reader->get_block(&block, &rows, &eof).ok()); + } + + { + auto close_count = std::make_shared(0); + auto reader = create_reader(&profile, remote_doris_range(), slots, {make_batch({"id"})}, + close_count); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(1)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + auto block = make_request_block(schema, {1}); + size_t rows = 0; + bool eof = false; + EXPECT_FALSE(reader->get_block(&block, &rows, &eof).ok()); + } +} + +TEST(RemoteDorisV2ReaderTest, RejectsInvalidRemoteDorisRange) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_bad_range_test"); + auto range = remote_doris_range(); + range.table_format_params.__isset.remote_doris_params = false; + auto close_count = std::make_shared(0); + auto reader = create_reader(&profile, range, slots, {}, close_count); + EXPECT_FALSE(reader->init(&state).ok()); +} + +} // namespace doris::format::remote_doris diff --git a/be/test/format_v2/table_reader_request_test.cpp b/be/test/format_v2/table_reader_request_test.cpp new file mode 100644 index 00000000000000..3845e086cea1b1 --- /dev/null +++ b/be/test/format_v2/table_reader_request_test.cpp @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "format_v2/table_reader.h" + +namespace doris::format { +namespace { + +class TableReaderRequestTestHelper final : public TableReader { +public: + using TableReader::_append_file_scan_column; +}; + +// Scenario: FileScanRequestBuilder owns request-local block positions and merges repeated nested +// projections for the same root. ColumnMapper can focus on producing file-local projection trees. +TEST(FileScanRequestBuilderTest, MergesNestedProjectionAndKeepsStableBlockPosition) { + FileScanRequest request; + FileScanRequestBuilder builder(&request); + + auto name_projection = LocalColumnIndex::partial_local(5); + name_projection.children.push_back(LocalColumnIndex::local(2)); + ASSERT_TRUE(builder.add_non_predicate_column(std::move(name_projection)).ok()); + + auto id_projection = LocalColumnIndex::partial_local(5); + id_projection.children.push_back(LocalColumnIndex::local(0)); + ASSERT_TRUE(builder.add_non_predicate_column(std::move(id_projection)).ok()); + + ASSERT_EQ(request.local_positions.size(), 1); + EXPECT_EQ(request.local_positions.at(LocalColumnId(5)).value(), 0); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + const auto& projection = request.non_predicate_columns[0]; + EXPECT_EQ(projection.column_id(), LocalColumnId(5)); + ASSERT_FALSE(projection.project_all_children); + ASSERT_EQ(projection.children.size(), 2); + EXPECT_EQ(projection.children[0].local_id(), 0); + EXPECT_EQ(projection.children[1].local_id(), 2); +} + +// Scenario: predicate scan columns dominate non-predicate columns because file readers return +// predicate columns in the same file-local block and TableReader can reuse them for output. +TEST(FileScanRequestBuilderTest, PredicateColumnRemovesDuplicateNonPredicateColumn) { + FileScanRequest request; + FileScanRequestBuilder builder(&request); + + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(1)).ok()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(2)).ok()); + ASSERT_TRUE(builder.add_predicate_column(LocalColumnId(1)).ok()); + + ASSERT_EQ(request.local_positions.size(), 2); + EXPECT_EQ(request.local_positions.at(LocalColumnId(1)).value(), 0); + EXPECT_EQ(request.local_positions.at(LocalColumnId(2)).value(), 1); + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(2)); +} + +// Scenario: TableReader's format-specific customization path delegates to FileScanRequestBuilder +// and preserves the same predicate/non-predicate de-duplication rule. +TEST(TableReaderRequestTest, AppendPredicateColumnKeepsOtherNonPredicateColumns) { + TableReaderRequestTestHelper reader; + FileScanRequest request; + + reader._append_file_scan_column(&request, LocalColumnId(1), &request.non_predicate_columns); + reader._append_file_scan_column(&request, LocalColumnId(2), &request.non_predicate_columns); + reader._append_file_scan_column(&request, LocalColumnId(1), &request.predicate_columns); + + ASSERT_EQ(request.local_positions.size(), 2); + EXPECT_EQ(request.local_positions.at(LocalColumnId(1)).value(), 0); + EXPECT_EQ(request.local_positions.at(LocalColumnId(2)).value(), 1); + + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(2)); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp new file mode 100644 index 00000000000000..855eb895302d80 --- /dev/null +++ b/be/test/format_v2/table_reader_test.cpp @@ -0,0 +1,4517 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_const.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr.h" +#include "exprs/vliteral.h" +#include "exprs/vruntimefilter_wrapper.h" +#include "exprs/vslot_ref.h" +#include "gen_cpp/Exprs_types.h" +#include "gen_cpp/ExternalTableSchema_types.h" +#include "gen_cpp/PlanNodes_types.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/segment/condition_cache.h" + +namespace doris::format { +namespace { + +std::vector projection_ids(const std::vector& projections) { + std::vector ids; + ids.reserve(projections.size()); + for (const auto& projection : projections) { + ids.push_back(projection.index); + } + return ids; +} + +TEST(LocalColumnIndexTest, MergeUnionsPartialChildrenAndFullProjectionDominates) { + LocalColumnIndex target {.index = 10, .project_all_children = false}; + target.children.push_back({.index = 1}); + target.children.push_back({.index = 2, .project_all_children = false}); + target.children.back().children.push_back({.index = 20}); + + LocalColumnIndex source {.index = 10, .project_all_children = false}; + source.children.push_back({.index = 2, .project_all_children = false}); + source.children.back().children.push_back({.index = 21}); + source.children.push_back({.index = 3}); + + ASSERT_TRUE(merge_local_column_index(&target, source).ok()); + ASSERT_FALSE(target.project_all_children); + ASSERT_EQ(std::vector({1, 2, 3}), projection_ids(target.children)); + ASSERT_FALSE(target.children[1].project_all_children); + ASSERT_EQ(std::vector({20, 21}), projection_ids(target.children[1].children)); + ASSERT_TRUE(target.children[2].project_all_children); + + LocalColumnIndex full_source {.index = 10}; + ASSERT_TRUE(merge_local_column_index(&target, full_source).ok()); + ASSERT_TRUE(target.project_all_children); + ASSERT_TRUE(target.children.empty()); +} + +TEST(LocalColumnIndexTest, FindsProjectedChildren) { + LocalColumnIndex projection {.index = 10, .project_all_children = false}; + projection.children.push_back({.index = 1}); + projection.children.push_back({.index = 2}); + + EXPECT_TRUE(is_full_projection(nullptr)); + EXPECT_FALSE(is_full_projection(&projection)); + EXPECT_TRUE(is_partial_projection(&projection)); + ASSERT_NE(find_child_projection(&projection, 2), nullptr); + EXPECT_EQ(find_child_projection(&projection, 2)->local_id(), 2); + EXPECT_EQ(find_child_projection(&projection, 3), nullptr); + EXPECT_TRUE(is_child_projected(nullptr, 3)); + EXPECT_TRUE(is_child_projected(&projection, 1)); + EXPECT_FALSE(is_child_projected(&projection, 3)); +} + +TEST(LocalColumnIndexTest, ProjectColumnDefinitionMatchesChildrenByLocalId) { + auto int_type = std::make_shared(); + auto string_type = std::make_shared(); + ColumnDefinition field; + field.identifier = Field::create_field(5); + field.name = "root"; + field.type = + std::make_shared(DataTypes {int_type, string_type}, Strings {"a", "b"}); + ColumnDefinition a_child; + a_child.identifier = Field::create_field(10); + a_child.local_id = 0; + a_child.name = "a"; + a_child.type = int_type; + ColumnDefinition b_child; + b_child.identifier = Field::create_field(20); + b_child.local_id = 1; + b_child.name = "b"; + b_child.type = string_type; + field.children = { + a_child, + b_child, + }; + LocalColumnIndex projection {.index = 5, .project_all_children = false}; + projection.children.push_back({.index = 1}); + + ColumnDefinition projected_field; + ASSERT_TRUE(project_column_definition(field, projection, &projected_field).ok()); + ASSERT_EQ(projected_field.children.size(), 1); + EXPECT_EQ(projected_field.children[0].get_identifier_field_id(), 20); + EXPECT_EQ(projected_field.children[0].name, "b"); + + const auto* projected_type = + assert_cast(remove_nullable(projected_field.type).get()); + ASSERT_EQ(projected_type->get_elements().size(), 1); + EXPECT_EQ(projected_type->get_element_name(0), "b"); + EXPECT_TRUE(projected_type->get_element(0)->equals(*string_type)); +} + +TEST(LocalColumnIndexTest, ProjectColumnDefinitionKeepsFileChildOrder) { + auto int_type = std::make_shared(); + auto string_type = std::make_shared(); + ColumnDefinition a_child; + a_child.identifier = Field::create_field(10); + a_child.local_id = 0; + a_child.name = "a"; + a_child.type = int_type; + ColumnDefinition b_child; + b_child.identifier = Field::create_field(20); + b_child.local_id = 1; + b_child.name = "b"; + b_child.type = string_type; + + ColumnDefinition field; + field.identifier = Field::create_field(5); + field.name = "root"; + field.type = + std::make_shared(DataTypes {int_type, string_type}, Strings {"a", "b"}); + field.children = {a_child, b_child}; + + LocalColumnIndex projection {.index = 5, .project_all_children = false}; + projection.children.push_back({.index = 1}); + projection.children.push_back({.index = 0}); + + ColumnDefinition projected_field; + ASSERT_TRUE(project_column_definition(field, projection, &projected_field).ok()); + ASSERT_EQ(projected_field.children.size(), 2); + EXPECT_EQ(projected_field.children[0].name, "a"); + EXPECT_EQ(projected_field.children[1].name, "b"); + + const auto* projected_type = + assert_cast(remove_nullable(projected_field.type).get()); + ASSERT_EQ(projected_type->get_elements().size(), 2); + EXPECT_EQ(projected_type->get_element_name(0), "a"); + EXPECT_EQ(projected_type->get_element_name(1), "b"); +} + +VExprSPtr table_int32_slot_ref(int slot_id, int column_id, const std::string& column_name) { + const auto nullable_int_type = make_nullable(std::make_shared()); + return VSlotRef::create_shared(slot_id, column_id, slot_id, nullable_int_type, column_name); +} + +VExprSPtr table_int32_literal(int32_t value) { + return VLiteral::create_shared(std::make_shared(), + Field::create_field(value)); +} + +TExprNode table_function_node(const std::string& function_name, const DataTypePtr& return_type, + const std::vector& arg_types, + TExprNodeType::type node_type, + TExprOpcode::type opcode = TExprOpcode::INVALID_OPCODE, + bool short_circuit_evaluation = false) { + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + std::vector thrift_arg_types; + thrift_arg_types.reserve(arg_types.size()); + for (const auto& arg_type : arg_types) { + thrift_arg_types.push_back(arg_type->to_thrift()); + } + fn.__set_arg_types(thrift_arg_types); + fn.__set_ret_type(return_type->to_thrift()); + fn.__set_has_var_args(false); + + TExprNode node; + node.__set_node_type(node_type); + node.__set_opcode(opcode); + node.__set_type(return_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(static_cast(arg_types.size())); + node.__set_is_nullable(return_type->is_nullable()); + if (short_circuit_evaluation) { + node.__set_short_circuit_evaluation(true); + } + return node; +} + +VExprSPtr create_expr_from_node(const TExprNode& node) { + VExprSPtr expr; + auto status = VExpr::create_expr(node, expr); + DORIS_CHECK(status.ok()) << status.to_string(); + return expr; +} + +VExprSPtr table_function_expr(const std::string& function_name, const DataTypePtr& return_type, + const std::vector& arg_types, + TExprNodeType::type node_type = TExprNodeType::FUNCTION_CALL, + TExprOpcode::type opcode = TExprOpcode::INVALID_OPCODE) { + const auto node = table_function_node(function_name, return_type, arg_types, node_type, opcode); + return VectorizedFnCall::create_shared(node); +} + +VExprSPtr table_int32_greater_than_expr(int slot_id, int column_id, int32_t value) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + auto expr = table_function_expr("gt", make_nullable(std::make_shared()), + {nullable_int_type, int_type}, TExprNodeType::BINARY_PRED, + TExprOpcode::GT); + expr->add_child(table_int32_slot_ref(slot_id, column_id, "id")); + expr->add_child(table_int32_literal(value)); + return expr; +} + +VExprSPtr runtime_filter_wrapper_expr(VExprSPtr impl) { + TExprNode node; + node.__set_node_type(TExprNodeType::SLOT_REF); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_num_children(1); + return VRuntimeFilterWrapper::create_shared(node, std::move(impl), 0, false, + /*filter_id=*/1); +} + +class NonDeterministicPartitionPredicate final : public VExpr { +public: + explicit NonDeterministicPartitionPredicate(bool* executed) + : VExpr(std::make_shared(), false), _executed(executed) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(_executed != nullptr); + *_executed = true; + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 0); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_executed); + return Status::OK(); + } + +private: + bool* const _executed; + const std::string _expr_name = "NonDeterministicPartitionPredicate"; +}; + +class NullableArrayBigintDefaultExpr final : public VExpr { +public: + explicit NullableArrayBigintDefaultExpr(DataTypePtr data_type) + : _name("single_element_groups") { + _data_type = std::move(data_type); + } + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector* selector, size_t count, + ColumnPtr& result_column) const override { + DCHECK(selector == nullptr || selector->size() == count); + auto values = ColumnInt64::create(); + auto offsets = ColumnArray::ColumnOffsets::create(); + auto null_map = ColumnUInt8::create(); + for (size_t i = 0; i < count; ++i) { + values->insert_value(7); + offsets->insert_value(static_cast(i + 1)); + null_map->insert_value(0); + } + auto array_column = ColumnArray::create(std::move(values), std::move(offsets)); + result_column = ColumnNullable::create(std::move(array_column), std::move(null_map)); + return Status::OK(); + } + +private: + std::string _name; +}; + +class TableReaderMaterializeTestHelper final : public TableReader { +public: + using TableReader::_materialize_map_mapping_column; +}; + +VExprSPtr table_int32_sum_expr(int left_slot_id, int left_column_id, int right_slot_id, + int right_column_id) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + auto expr = + table_function_expr("add", nullable_int_type, {nullable_int_type, nullable_int_type}); + expr->add_child(table_int32_slot_ref(left_slot_id, left_column_id, "id")); + expr->add_child(table_int32_slot_ref(right_slot_id, right_column_id, "score")); + return expr; +} + +VExprSPtr table_int32_sum_greater_than_expr(int left_slot_id, int left_column_id, int right_slot_id, + int right_column_id, int32_t value) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + auto expr = table_function_expr("gt", make_nullable(std::make_shared()), + {nullable_int_type, int_type}, TExprNodeType::BINARY_PRED, + TExprOpcode::GT); + expr->add_child( + table_int32_sum_expr(left_slot_id, left_column_id, right_slot_id, right_column_id)); + expr->add_child(table_int32_literal(value)); + return expr; +} + +VExprSPtr table_condition_function_expr(const std::string& function_name, bool short_circuit) { + const auto int_type = std::make_shared(); + std::vector arg_types; + if (function_name == "if") { + arg_types = {std::make_shared(), int_type, int_type}; + } else { + arg_types = {int_type, int_type}; + } + auto expr = create_expr_from_node( + table_function_node(function_name, int_type, arg_types, TExprNodeType::FUNCTION_CALL, + TExprOpcode::INVALID_OPCODE, short_circuit)); + if (function_name == "if") { + expr->add_child(table_int32_greater_than_expr(0, 0, 0)); + expr->add_child(table_int32_literal(1)); + expr->add_child(table_int32_literal(0)); + } else { + expr->add_child(table_int32_slot_ref(0, 0, "id")); + expr->add_child(table_int32_literal(0)); + } + return expr; +} + +VExprSPtr table_case_expr(bool short_circuit) { + const auto int_type = std::make_shared(); + TCaseExpr case_node; + case_node.__set_has_case_expr(false); + case_node.__set_has_else_expr(true); + + TExprNode node; + node.__set_node_type(TExprNodeType::CASE_EXPR); + node.__set_type(int_type->to_thrift()); + node.__set_is_nullable(false); + node.__set_num_children(3); + node.__set_case_expr(case_node); + if (short_circuit) { + node.__set_short_circuit_evaluation(true); + } + + auto expr = create_expr_from_node(node); + expr->add_child(table_int32_greater_than_expr(0, 0, 0)); + expr->add_child(table_int32_literal(1)); + expr->add_child(table_int32_literal(0)); + return expr; +} + +TEST(CloneTableExprTreeTest, ClonesConditionalExpressions) { + const std::vector expressions { + table_condition_function_expr("if", false), + table_condition_function_expr("if", true), + table_condition_function_expr("ifnull", false), + table_condition_function_expr("ifnull", true), + table_condition_function_expr("coalesce", false), + table_condition_function_expr("coalesce", true), + table_case_expr(false), + table_case_expr(true), + }; + + for (const auto& expr : expressions) { + VExprSPtr cloned; + const auto status = clone_table_expr_tree(expr, &cloned); + ASSERT_TRUE(status.ok()) << expr->debug_string() << ": " << status.to_string(); + ASSERT_NE(cloned, nullptr); + const auto* original_expr = expr.get(); + const auto* cloned_expr = cloned.get(); + EXPECT_TRUE(typeid(*original_expr) == typeid(*cloned_expr)) + << expr->expr_name() << " cloned as " << typeid(*cloned_expr).name(); + EXPECT_EQ(expr->expr_name(), cloned->expr_name()); + EXPECT_EQ(expr->get_num_children(), cloned->get_num_children()); + EXPECT_NE(original_expr, cloned_expr); + } +} + +// Scenario: cloning a VectorizedFnCall whose return type is complex must not reconstruct the expr +// from TExprNode, because DataTypeFactory rejects nested types through the primitive-type path. +TEST(CloneTableExprTreeTest, ClonesVectorizedFnCallWithComplexReturnType) { + const auto int_type = std::make_shared(); + const auto string_type = std::make_shared(); + const auto struct_type = + std::make_shared(DataTypes {int_type, string_type}, Strings {"a", "b"}); + const auto array_type = std::make_shared(struct_type); + + auto expr = table_function_expr("element_at", struct_type, {array_type, int_type}); + expr->add_child(VSlotRef::create_shared(0, 0, -1, array_type, "array_of_struct")); + expr->add_child(table_int32_literal(1)); + + VExprSPtr cloned; + const auto status = clone_table_expr_tree(expr, &cloned); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(cloned, nullptr); + EXPECT_EQ(cloned->expr_name(), expr->expr_name()); + EXPECT_TRUE(cloned->data_type()->equals(*struct_type)); + EXPECT_EQ(cloned->get_num_children(), 2); + EXPECT_NE(cloned.get(), expr.get()); +} + +std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { + std::shared_ptr array; + EXPECT_TRUE(builder->Finish(&array).ok()); + return array; +} + +std::shared_ptr build_int32_array(const std::vector& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_string_array(const std::vector& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +void write_parquet_file(const std::string& file_path, int32_t id, const std::string& value) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array({id}), build_string_array({value})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_struct_parquet_file(const std::string& file_path, int32_t id) { + auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false)}); + arrow::StructBuilder builder( + struct_type, arrow::default_memory_pool(), + {std::make_shared(arrow::default_memory_pool())}); + auto* id_builder = assert_cast(builder.field_builder(0)); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(id_builder->Append(id).ok()); + + auto schema = arrow::schema({ + arrow::field("s", struct_type, false), + }); + auto table = arrow::Table::Make(schema, {finish_array(&builder)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + writer_builder.build())); +} + +void write_struct_parquet_file(const std::string& file_path, const std::vector& ids, + int64_t row_group_size = -1) { + auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false)}); + arrow::StructBuilder builder( + struct_type, arrow::default_memory_pool(), + {std::make_shared(arrow::default_memory_pool())}); + auto* id_builder = assert_cast(builder.field_builder(0)); + for (const auto id : ids) { + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(id_builder->Append(id).ok()); + } + + auto schema = arrow::schema({ + arrow::field("s", struct_type, false), + }); + auto table = arrow::Table::Make(schema, {finish_array(&builder)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + const auto write_row_group_size = + row_group_size > 0 ? row_group_size : static_cast(ids.size()); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + write_row_group_size, + writer_builder.build())); +} + +void write_struct_with_nullable_child_parquet_file(const std::string& file_path) { + auto struct_type = arrow::struct_({ + arrow::field("id", arrow::int32(), false), + arrow::field("note", arrow::utf8(), true), + }); + std::vector> field_builders; + auto id_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(id_builder))); + auto note_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(note_builder))); + arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto* struct_id_builder = assert_cast(builder.field_builder(0)); + auto* struct_note_builder = assert_cast(builder.field_builder(1)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_id_builder->Append(7).ok()); + EXPECT_TRUE(struct_note_builder->Append("seven").ok()); + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_id_builder->Append(8).ok()); + EXPECT_TRUE(struct_note_builder->AppendNull().ok()); + + auto schema = arrow::schema({ + arrow::field("s", struct_type, false), + }); + auto table = arrow::Table::Make(schema, {finish_array(&builder)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + writer_builder.build())); +} + +void write_list_struct_parquet_file(const std::string& file_path) { + auto struct_type = arrow::struct_( + {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::int32(), false)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto b_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); + auto struct_builder = std::make_shared( + struct_type, arrow::default_memory_pool(), std::move(field_builders)); + auto list_type = arrow::list(arrow::field("element", struct_type, true)); + arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, list_type); + auto* a_builder = assert_cast(struct_builder->field_builder(0)); + auto* b_builder = assert_cast(struct_builder->field_builder(1)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(10).ok()); + EXPECT_TRUE(b_builder->Append(11).ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(20).ok()); + EXPECT_TRUE(b_builder->Append(21).ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(30).ok()); + EXPECT_TRUE(b_builder->Append(31).ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(struct_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(40).ok()); + EXPECT_TRUE(b_builder->Append(41).ok()); + + auto schema = arrow::schema({ + arrow::field("xs", list_type, false), + }); + auto table = arrow::Table::Make(schema, {finish_array(&builder)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 3, + writer_builder.build())); +} + +void write_map_struct_parquet_file(const std::string& file_path) { + auto key_builder = std::make_shared(); + auto struct_type = arrow::struct_( + {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), false)}); + std::vector> field_builders; + auto a_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); + auto b_array_builder = std::make_unique(); + field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); + auto value_builder = std::make_shared( + struct_type, arrow::default_memory_pool(), std::move(field_builders)); + auto map_type = arrow::map(arrow::int32(), arrow::field("value", struct_type, false)); + arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, map_type); + auto* a_builder = assert_cast(value_builder->field_builder(0)); + auto* b_builder = assert_cast(value_builder->field_builder(1)); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(1).ok()); + EXPECT_TRUE(value_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(10).ok()); + EXPECT_TRUE(b_builder->Append("ma").ok()); + EXPECT_TRUE(key_builder->Append(2).ok()); + EXPECT_TRUE(value_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(20).ok()); + EXPECT_TRUE(b_builder->Append("mb").ok()); + + EXPECT_TRUE(builder.Append().ok()); + EXPECT_TRUE(key_builder->Append(3).ok()); + EXPECT_TRUE(value_builder->Append().ok()); + EXPECT_TRUE(a_builder->Append(30).ok()); + EXPECT_TRUE(b_builder->Append("mc").ok()); + + EXPECT_TRUE(builder.AppendEmptyValue().ok()); + + auto schema = arrow::schema({ + arrow::field("kv", map_type, false), + }); + auto table = arrow::Table::Make(schema, {finish_array(&builder)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 3, + writer_builder.build())); +} + +void write_int_pair_parquet_file(const std::string& file_path, const std::vector& ids, + const std::vector& scores, + const std::vector& values, + int64_t row_group_size = -1) { + const auto id_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"0"}); + const auto score_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}); + const auto value_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"2"}); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false)->WithMetadata(id_metadata), + arrow::field("score", arrow::int32(), false)->WithMetadata(score_metadata), + arrow::field("value", arrow::utf8(), false)->WithMetadata(value_metadata), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(scores), + build_string_array(values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + const auto write_row_group_size = + row_group_size > 0 ? row_group_size : static_cast(ids.size()); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + write_row_group_size, builder.build())); +} + +Block build_table_block(const std::vector& columns) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +const IColumn& expect_not_null_nullable_nested_column(const IColumn& column) { + if (!column.is_nullable()) { + return column; + } + const auto& nullable_column = assert_cast(column); + for (const auto is_null : nullable_column.get_null_map_data()) { + EXPECT_EQ(is_null, 0); + } + return nullable_column.get_nested_column(); +} + +void expect_nullable_column_all_null(const IColumn& column) { + const auto full_column = column.convert_to_full_column_if_const(); + const auto& nullable_column = assert_cast(*full_column); + for (const auto is_null : nullable_column.get_null_map_data()) { + EXPECT_EQ(is_null, 1); + } +} + +const IColumn& expect_not_null_table_column(const Block& block, size_t position) { + return expect_not_null_nullable_nested_column(*block.get_by_position(position).column); +} + +ColumnDefinition make_table_column(int32_t id, const std::string& name, const DataTypePtr& type); + +void expect_int32_column_values(const IColumn& column, + const std::vector& expected_values) { + const auto full_column = column.convert_to_full_column_if_const(); + const auto& nested_column = expect_not_null_nullable_nested_column(*full_column); + const auto& values = assert_cast(nested_column).get_data(); + ASSERT_EQ(values.size(), expected_values.size()); + for (size_t row = 0; row < expected_values.size(); ++row) { + EXPECT_EQ(values[row], expected_values[row]); + } +} + +SplitReadOptions build_split_options(const std::string& file_path) { + SplitReadOptions options; + options.current_range.__set_path(file_path); + options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + return options; +} + +void set_table_level_row_count(SplitReadOptions* split_options, int64_t row_count) { + split_options->current_range.__isset.table_format_params = true; + split_options->current_range.table_format_params.__isset.table_level_row_count = true; + split_options->current_range.table_format_params.table_level_row_count = row_count; +} + +int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { + return column_metadata.has_dictionary_page() + ? static_cast(column_metadata.dictionary_page_offset()) + : static_cast(column_metadata.data_page_offset()); +} + +SplitReadOptions build_split_options_for_row_group_mid(const std::string& file_path, + int row_group_idx) { + auto options = build_split_options(file_path); + auto reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); + auto metadata = reader->metadata(); + auto row_group_metadata = metadata->RowGroup(row_group_idx); + auto first_column = row_group_metadata->ColumnChunk(0); + auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); + const int64_t row_group_start_offset = parquet_column_start_offset(*first_column); + const int64_t row_group_end_offset = + parquet_column_start_offset(*last_column) + last_column->total_compressed_size(); + const int64_t row_group_mid_offset = + row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; + options.current_range.__set_start_offset(row_group_mid_offset); + options.current_range.__set_size(1); + return options; +} + +DataTypePtr make_table_test_type(const DataTypePtr& type, bool nullable_root = true) { + DORIS_CHECK(type != nullptr); + const auto nested_type = remove_nullable(type); + DataTypePtr result; + if (const auto* struct_type = typeid_cast(nested_type.get())) { + DataTypes child_types; + child_types.reserve(struct_type->get_elements().size()); + for (const auto& child_type : struct_type->get_elements()) { + child_types.push_back(make_table_test_type(child_type)); + } + result = std::make_shared(child_types, struct_type->get_element_names()); + } else if (const auto* array_type = typeid_cast(nested_type.get())) { + result = std::make_shared( + make_table_test_type(array_type->get_nested_type())); + } else if (const auto* map_type = typeid_cast(nested_type.get())) { + result = std::make_shared(make_table_test_type(map_type->get_key_type()), + make_table_test_type(map_type->get_value_type())); + } else { + result = nested_type; + } + return nullable_root ? make_nullable(result) : result; +} + +ColumnDefinition make_table_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition column; + if (id >= 0) { + column.identifier = Field::create_field(id); + } + column.name = name; + // TableReader tests model external table scan descriptors. Those table columns are nullable + // even when the Parquet file field itself is required, so keep the test schema aligned with + // the real scan contract at the construction boundary. + column.type = make_table_test_type(type); + return column; +} + +ColumnDefinition make_file_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition field; + field.identifier = Field::create_field(id); + field.local_id = id; + field.name = name; + field.type = make_table_test_type(type); + return field; +} + +schema::external::TFieldPtr external_schema_field(std::string name, int32_t id, + std::vector aliases = {}) { + auto field = std::make_shared(); + field->__set_name(std::move(name)); + field->__set_id(id); + if (!aliases.empty()) { + field->__set_name_mapping(std::move(aliases)); + } + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; +} + +schema::external::TFieldPtr external_array_field(std::string name, int32_t id, + schema::external::TFieldPtr item_field, + std::vector aliases = {}) { + auto field = external_schema_field(std::move(name), id, std::move(aliases)); + schema::external::TArrayField array_field; + array_field.__set_item_field(std::move(item_field)); + field.field_ptr->nestedField.__set_array_field(std::move(array_field)); + field.field_ptr->__isset.nestedField = true; + return field; +} + +schema::external::TFieldPtr external_map_field(std::string name, int32_t id, + schema::external::TFieldPtr key_field, + schema::external::TFieldPtr value_field, + std::vector aliases = {}) { + auto field = external_schema_field(std::move(name), id, std::move(aliases)); + schema::external::TMapField map_field; + map_field.__set_key_field(std::move(key_field)); + map_field.__set_value_field(std::move(value_field)); + field.field_ptr->nestedField.__set_map_field(std::move(map_field)); + field.field_ptr->__isset.nestedField = true; + return field; +} + +schema::external::TFieldPtr external_struct_field(std::string name, int32_t id, + std::vector fields, + std::vector aliases = {}) { + auto field = external_schema_field(std::move(name), id, std::move(aliases)); + schema::external::TStructField struct_field; + struct_field.__set_fields(std::move(fields)); + field.field_ptr->nestedField.__set_struct_field(std::move(struct_field)); + field.field_ptr->__isset.nestedField = true; + return field; +} + +schema::external::TSchema external_schema(int64_t schema_id, + std::vector fields) { + schema::external::TStructField root_field; + root_field.__set_fields(std::move(fields)); + schema::external::TSchema schema; + schema.__set_schema_id(schema_id); + schema.__set_root_field(std::move(root_field)); + return schema; +} + +ColumnDefinition make_nullable_column_definition(ColumnDefinition column) { + column.type = make_table_test_type(column.type); + for (auto& child : column.children) { + child = make_nullable_column_definition(std::move(child)); + } + return column; +} + +MutableColumnPtr make_not_null_nullable_column(MutableColumnPtr nested_column) { + auto null_map = ColumnUInt8::create(); + for (size_t i = 0; i < nested_column->size(); ++i) { + null_map->insert_value(0); + } + return ColumnNullable::create(std::move(nested_column), std::move(null_map)); +} + +class TableReaderCharVarcharTestHelper final : public TableReader { +public: + using TableReader::_should_truncate_char_or_varchar_column; + using TableReader::_truncate_char_or_varchar_column; +}; + +class TableReaderCastTestHelper final : public TableReader { +public: + using TableReader::_cast_column_to_type; +}; + +TEST(TableReaderTest, TruncateCharOrVarcharPredicateOnlyAppliesToParquetStringWidthMismatch) { + ColumnMapping mapping; + mapping.table_type = std::make_shared(3, TYPE_VARCHAR); + mapping.file_type = std::make_shared(10, TYPE_VARCHAR); + EXPECT_TRUE(TableReaderCharVarcharTestHelper::_should_truncate_char_or_varchar_column(mapping)); + + mapping.file_type = std::make_shared(2, TYPE_VARCHAR); + EXPECT_FALSE( + TableReaderCharVarcharTestHelper::_should_truncate_char_or_varchar_column(mapping)); + + mapping.file_type = std::make_shared(); + EXPECT_TRUE(TableReaderCharVarcharTestHelper::_should_truncate_char_or_varchar_column(mapping)); + + mapping.file_type = std::make_shared(); + EXPECT_TRUE(TableReaderCharVarcharTestHelper::_should_truncate_char_or_varchar_column(mapping)); + + mapping.table_type = std::make_shared(); + EXPECT_FALSE( + TableReaderCharVarcharTestHelper::_should_truncate_char_or_varchar_column(mapping)); +} + +TEST(TableReaderTest, TruncateCharOrVarcharColumnKeepsNullMap) { + auto nested = ColumnString::create(); + nested->insert_data("abcdef", 6); + nested->insert_data("xyz", 3); + auto null_map = ColumnUInt8::create(); + null_map->insert_value(0); + null_map->insert_value(1); + + auto type = make_nullable(std::make_shared(3, TYPE_VARCHAR)); + Block block; + block.insert({ColumnNullable::create(std::move(nested), std::move(null_map)), type, "v"}); + + TableReaderCharVarcharTestHelper::_truncate_char_or_varchar_column(&block, 0, 3); + + ASSERT_EQ(block.columns(), 1); + ASSERT_EQ(block.rows(), 2); + const auto* nullable_column = + assert_cast(block.get_by_position(0).column.get()); + EXPECT_EQ(nullable_column->get_nested_column().get_data_at(0).to_string(), "abc"); + EXPECT_FALSE(nullable_column->is_null_at(0)); + EXPECT_TRUE(nullable_column->is_null_at(1)); +} + +void set_name_identifiers(std::vector* columns); + +void set_name_identifier(ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + column->identifier = Field::create_field(column->name); + set_name_identifiers(&column->children); +} + +void set_name_identifiers(std::vector* columns) { + DORIS_CHECK(columns != nullptr); + for (auto& column : *columns) { + set_name_identifier(&column); + } +} + +VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { + auto ctx = VExprContext::create_shared(expr); + auto status = ctx->prepare(state, RowDescriptor()); + EXPECT_TRUE(status.ok()) << status; + status = ctx->open(state); + EXPECT_TRUE(status.ok()) << status; + return ctx; +} + +struct FakeFileReaderState { + int init_count = 0; + int open_count = 0; + int close_count = 0; + int64_t total_rows = 2; + int64_t aggregate_count = -1; + int64_t condition_cache_base_granule = 0; + size_t condition_cache_num_granules = 0; + bool eof_with_first_batch = true; + bool inject_delete_conjunct = false; + bool stop_during_aggregate = false; + bool stop_during_read = false; + bool not_found_during_init = false; + std::shared_ptr last_request; + std::shared_ptr condition_cache_ctx; + std::shared_ptr io_ctx; +}; + +class FakeFileReader final : public FileReader { +public: + FakeFileReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::vector schema, std::shared_ptr state) + : FileReader(system_properties, file_description, state->io_ctx, nullptr), + _schema(std::move(schema)), + _state(std::move(state)) {} + + Status init(RuntimeState* state) override { + (void)state; + ++_state->init_count; + if (_state->not_found_during_init) { + return Status::NotFound("fake table reader input is missing"); + } + _eof = false; + return Status::OK(); + } + + Status get_schema(std::vector* file_schema) const override { + DORIS_CHECK(file_schema != nullptr); + *file_schema = _schema; + for (auto& column : *file_schema) { + column = make_nullable_column_definition(std::move(column)); + } + return Status::OK(); + } + + Status open(std::shared_ptr request) override { + RETURN_IF_ERROR(FileReader::open(std::move(request))); + _state->last_request = _request; + ++_state->open_count; + _returned_batch = false; + return Status::OK(); + } + + Status get_block(Block* file_block, size_t* rows, bool* eof) override { + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(_request != nullptr); + if (_returned_batch) { + *rows = 0; + *eof = true; + return Status::OK(); + } + + for (const auto& [file_column_id, block_position] : _request->local_positions) { + if (file_column_id == LocalColumnId(0)) { + auto column = ColumnInt32::create(); + column->insert_value(1); + column->insert_value(2); + file_block->replace_by_position(block_position.value(), + make_not_null_nullable_column(std::move(column))); + } else if (file_column_id == LocalColumnId(1)) { + auto column = ColumnString::create(); + column->insert_data("one", 3); + column->insert_data("two", 3); + file_block->replace_by_position(block_position.value(), + make_not_null_nullable_column(std::move(column))); + } else if (file_column_id == LocalColumnId(2)) { + auto country_values = ColumnString::create(); + country_values->insert_data("USA", 3); + country_values->insert_data("UK", 2); + auto country_column = make_not_null_nullable_column(std::move(country_values)); + + auto city_column = ColumnString::create(); + city_column->insert_data("New York", 8); + city_column->insert_data("London", 6); + + MutableColumns struct_children; + struct_children.push_back(std::move(country_column)); + struct_children.push_back(make_not_null_nullable_column(std::move(city_column))); + auto struct_column = ColumnStruct::create(std::move(struct_children)); + + file_block->replace_by_position( + block_position.value(), + make_not_null_nullable_column(std::move(struct_column))); + } else { + return Status::InvalidArgument("Unexpected fake file column id {}", + file_column_id.value()); + } + } + + if (_state->stop_during_read) { + DORIS_CHECK(_state->io_ctx != nullptr); + _state->io_ctx->should_stop = true; + } + _returned_batch = true; + *rows = 2; + *eof = _state->eof_with_first_batch; + if (_state->condition_cache_ctx != nullptr && !_state->condition_cache_ctx->is_hit && + _state->condition_cache_ctx->filter_result != nullptr && + !_state->condition_cache_ctx->filter_result->empty()) { + // The real file reader marks a granule after local row-level predicates keep at least + // one row from that granule. The fake reader does it here so TableReader tests can + // focus on condition-cache lifecycle decisions without depending on Parquet internals. + (*_state->condition_cache_ctx->filter_result)[0] = true; + } + return Status::OK(); + } + + Status get_aggregate_result(const FileAggregateRequest& request, + FileAggregateResult* result) override { + DORIS_CHECK(result != nullptr); + if (_state->aggregate_count < 0) { + return FileReader::get_aggregate_result(request, result); + } + if (request.agg_type != TPushAggOp::type::COUNT) { + return Status::NotSupported("fake reader only supports COUNT aggregate pushdown"); + } + if (_state->stop_during_aggregate) { + DORIS_CHECK(_state->io_ctx != nullptr); + _state->io_ctx->should_stop = true; + return Status::EndOfFile("stop"); + } + result->count = _state->aggregate_count; + result->columns.clear(); + _record_scan_rows(_state->aggregate_count); + _eof = true; + return Status::OK(); + } + + void set_condition_cache_context(std::shared_ptr ctx) override { + _state->condition_cache_ctx = std::move(ctx); + if (_state->condition_cache_ctx != nullptr && !_state->condition_cache_ctx->is_hit) { + _state->condition_cache_ctx->base_granule = _state->condition_cache_base_granule; + if (_state->condition_cache_num_granules > 0) { + _state->condition_cache_ctx->num_granules = _state->condition_cache_num_granules; + } + } + } + + int64_t get_total_rows() const override { return _state->total_rows; } + + Status close() override { + ++_state->close_count; + _request.reset(); + _eof = true; + return Status::OK(); + } + +private: + std::vector _schema; + std::shared_ptr _state; + bool _returned_batch = false; +}; + +class FakeTableReader final : public TableReader { +public: + FakeTableReader(std::vector file_schema, + std::shared_ptr state) + : _file_schema(std::move(file_schema)), _state(std::move(state)) {} + +protected: + Status create_file_reader(std::unique_ptr* reader) override { + DORIS_CHECK(reader != nullptr); + auto system_properties = std::make_shared(); + system_properties->system_type = TFileType::FILE_LOCAL; + auto file_description = std::make_unique(); + file_description->path = "fake-table-reader-input"; + *reader = std::make_unique(system_properties, file_description, + _file_schema, _state); + return Status::OK(); + } + + Status customize_file_scan_request(FileScanRequest* file_request) override { + RETURN_IF_ERROR(TableReader::customize_file_scan_request(file_request)); + if (_state->inject_delete_conjunct) { + // Table-format delete handling is represented in v2 by TableReader injecting + // delete_conjuncts into the file scan request. The fake reader does not execute it; + // this only tests that condition cache is disabled once such table-level delete state + // is present in the request. + file_request->delete_conjuncts.push_back( + VExprContext::create_shared(table_int32_literal(1))); + } + return Status::OK(); + } + +private: + std::vector _file_schema; + std::shared_ptr _state; +}; + +class ScopedConditionCacheForTest { +public: + ScopedConditionCacheForTest() + : _previous(ExecEnv::GetInstance()->get_condition_cache()), + _cache(segment_v2::ConditionCache::create_global_cache(1024 * 1024, 4)) { + ExecEnv::GetInstance()->_condition_cache = _cache.get(); + } + + ~ScopedConditionCacheForTest() { ExecEnv::GetInstance()->_condition_cache = _previous; } + + segment_v2::ConditionCache* get() { return _cache.get(); } + +private: + segment_v2::ConditionCache* _previous = nullptr; + std::unique_ptr _cache; +}; + +TEST(TableReaderTest, PrepareSplitPrunesPartitionRuntimeFilter) { + std::vector projected_columns; + auto partition_column = make_table_column(0, "part", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("scanner"); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + SplitReadOptions pruned_split; + pruned_split.current_range.__set_path("unused-pruned-file"); + pruned_split.partition_values.emplace("part", Field::create_field(7)); + pruned_split.partition_prune_conjuncts.push_back(VExprContext::create_shared( + runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 10)))); + ASSERT_TRUE(reader.prepare_split(pruned_split).ok()); + EXPECT_TRUE(reader.current_split_pruned()); + ASSERT_NE(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum"), nullptr); + EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 1); + + SplitReadOptions retained_split; + retained_split.current_range.__set_path("unused-retained-file"); + retained_split.partition_values.emplace("part", Field::create_field(11)); + retained_split.partition_prune_conjuncts.push_back(VExprContext::create_shared( + runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 10)))); + ASSERT_TRUE(reader.prepare_split(retained_split).ok()); + EXPECT_FALSE(reader.current_split_pruned()); +} + +TEST(TableReaderTest, PrepareSplitDoesNotEvaluateNonDeterministicPartitionPredicate) { + std::vector projected_columns; + auto partition_column = make_table_column(0, "part", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("scanner"); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + bool predicate_executed = false; + auto predicate = std::make_shared(&predicate_executed); + predicate->add_child(table_int32_slot_ref(0, 0, "part")); + SplitReadOptions split; + split.current_range.__set_path("unused-nondeterministic-file"); + split.partition_values.emplace("part", Field::create_field(7)); + split.partition_prune_conjuncts.push_back( + VExprContext::create_shared(runtime_filter_wrapper_expr(std::move(predicate)))); + split.partition_prune_conjuncts.push_back(VExprContext::create_shared( + runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 10)))); + + ASSERT_TRUE(reader.prepare_split(split).ok()); + EXPECT_FALSE(predicate_executed); + EXPECT_FALSE(reader.current_split_pruned()); + ASSERT_NE(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum"), nullptr); + EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 0); +} + +TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { + std::vector projected_columns; + auto partition_column = make_table_column(0, "part", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_predicate = + std::make_shared(&predicate_executed); + unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "part")); + auto fake_state = std::make_shared(); + FakeTableReader reader({}, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = + { + prepared_conjunct(&state, unsafe_predicate), + prepared_conjunct(&state, + table_int32_greater_than_expr( + 0, 0, 10)), + }, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + split.partition_values.emplace("part", Field::create_field(7)); + ASSERT_TRUE(reader.prepare_split(split).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(predicate_executed); + EXPECT_FALSE(eos); + EXPECT_EQ(fake_state->open_count, 1); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { + std::vector projected_columns; + auto partition_column = make_table_column(0, "part", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_slotless_predicate = + std::make_shared(&predicate_executed); + auto fake_state = std::make_shared(); + FakeTableReader reader({}, fake_state); + ASSERT_TRUE( + reader + .init({ + .projected_columns = projected_columns, + .conjuncts = + { + prepared_conjunct(&state, unsafe_slotless_predicate), + prepared_conjunct(&state, table_int32_greater_than_expr( + 0, 0, 10)), + }, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + split.partition_values.emplace("part", Field::create_field(7)); + ASSERT_TRUE(reader.prepare_split(split).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(predicate_executed); + EXPECT_FALSE(eos); + // The later partition predicate is false for part=7. Opening the file proves constant pruning + // stopped at the earlier unsafe expression even though that expression had no slot and thus no + // entry in `_table_filters`. + EXPECT_EQ(fake_state->open_count, 1); + ASSERT_NE(fake_state->last_request, nullptr); + // A slotless unsafe conjunct is an ordering barrier even though it has no TableFilter entry. + // The later predicate must stay on the scanner's row-level path instead of running inside the + // file reader before the unsafe conjunct. + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, CanUseInjectedFileReaderForStandaloneUnitTest) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + file_schema.push_back(make_file_column(1, "value", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "value", std::make_shared())); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + + ASSERT_EQ(fake_state->init_count, 1); + ASSERT_EQ(fake_state->open_count, 1); + ASSERT_EQ(fake_state->close_count, 1); + ASSERT_NE(fake_state->last_request, nullptr); + ASSERT_EQ(fake_state->last_request->local_positions.at(LocalColumnId(1)).value(), 0); + ASSERT_EQ(fake_state->last_request->local_positions.at(LocalColumnId(0)).value(), 1); + EXPECT_EQ(projection_ids(fake_state->last_request->non_predicate_columns), + std::vector({1, 0})); + EXPECT_TRUE(fake_state->last_request->predicate_columns.empty()); + + const auto& value_column = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(value_column.get_data_at(0).to_string(), "one"); + EXPECT_EQ(value_column.get_data_at(1).to_string(), "two"); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 2); + + block = build_table_block(projected_columns); + eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); +} + +TEST(TableReaderTest, PrepareSplitReplacesInitialConjunctSnapshot) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {VExprContext::create_shared( + table_int32_greater_than_expr(0, 0, 0))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + split_options.conjuncts = VExprContextSPtrs {VExprContext::create_shared( + runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 1)))}; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->last_request, nullptr); + ASSERT_EQ(fake_state->last_request->conjuncts.size(), 1); + EXPECT_TRUE(fake_state->last_request->conjuncts.front()->root()->is_rf_wrapper()); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, RefreshedConjunctDisablesTableLevelCount) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + split_options.conjuncts = VExprContextSPtrs {VExprContext::create_shared( + runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 1)))}; + set_table_level_row_count(&split_options, 5); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + // The metadata count advertises five rows, while the fake reader contains two. Opening the + // reader and returning its rows proves the fresh runtime filter did not take the synthetic + // table-level COUNT path that would bypass all row predicates. + EXPECT_EQ(fake_state->open_count, 1); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, PendingRuntimeFilterDisablesTableLevelCount) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = state.batch_size() + 5; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + // A pending runtime filter makes metadata COUNT ineligible before its first synthetic batch. + // This prevents the filter from arriving between scheduler reads after unfiltered rows have + // already escaped. + split_options.all_runtime_filters_applied = false; + set_table_level_row_count(&split_options, state.batch_size() + 5); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(fake_state->open_count, 1); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, PendingRuntimeFilterDisablesMinMaxPushdown) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_pending_rf_minmax_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {3, 1, 5, 2}, {30, 10, 50, 20}, + {"three", "one", "five", "two"}, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(1, "score", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::MINMAX, + }) + .ok()); + auto split_options = build_split_options(file_path); + split_options.all_runtime_filters_applied = false; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + bool eos = false; + size_t total_rows = 0; + bool checked_first_batch = false; + while (!eos) { + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + total_rows += block.rows(); + if (!checked_first_batch && block.rows() > 0) { + const auto& ids = + assert_cast(expect_not_null_table_column(block, 0)); + ASSERT_EQ(ids.size(), 2); + EXPECT_EQ(ids.get_element(0), 3); + EXPECT_EQ(ids.get_element(1), 1); + checked_first_batch = true; + } + } + // MIN/MAX pushdown would return the two synthetic extrema [1, 5]. Reading the original first + // row group [3, 1] and all four rows proves a pending RF kept the physical reader active. + EXPECT_TRUE(checked_first_batch); + EXPECT_EQ(total_rows, 4); + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, SlotlessConjunctDisablesAggregatePushdown) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + // The slotless predicate cannot become a TableFilter or a file-reader conjunct, but its + // presence still prevents the fake aggregate count (3) from replacing the two physical rows. + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_EQ(block.rows(), 2); + EXPECT_TRUE(predicate_executed); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, AbortSplitClearsReaderAfterIgnorableNotFound) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->not_found_during_init = true; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("missing-fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.is()) << status; + ASSERT_TRUE(reader.abort_split().ok()); + EXPECT_EQ(fake_state->init_count, 1); + EXPECT_EQ(fake_state->close_count, 1); + + fake_state->not_found_during_init = false; + split_options.current_range.__set_path("existing-fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(fake_state->init_count, 2); + EXPECT_EQ(fake_state->close_count, 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + fake_state->io_ctx = io_ctx; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 3); + EXPECT_EQ(file_reader_stats.read_rows, 3); + EXPECT_EQ(fake_state->close_count, 1); +} + +TEST(TableReaderTest, PushDownCountStopConvertsAggregateEndOfFileToEos) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + auto io_ctx = std::make_shared(); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + fake_state->io_ctx = io_ctx; + fake_state->stop_during_aggregate = true; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(fake_state->close_count, 0); +} + +TEST(TableReaderTest, DebugStringCoversReaderStateAndEnumNames) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + file_schema.push_back(make_file_column(1, "value", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(1, "value", std::make_shared())); + projected_columns[0].name_mapping = {"legacy_id"}; + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->eof_with_first_batch = false; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 0))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = std::make_shared(), + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + + SplitReadOptions split_options; + split_options.partition_values.emplace("dt", Field::create_field("2026-06-29")); + split_options.current_range.__set_path("fake-table-reader-input"); + split_options.current_range.__set_file_size(64); + split_options.current_range.__set_start_offset(7); + split_options.current_range.__set_size(11); + split_options.current_range.__set_modification_time(13); + split_options.current_range.__set_fs_name("local-fs"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto debug = reader.debug_string(); + EXPECT_NE(debug.find("format=PARQUET"), std::string::npos); + EXPECT_NE(debug.find("push_down_agg_type=COUNT"), std::string::npos); + EXPECT_NE(debug.find("current_file=FileDescription{path=fake-table-reader-input"), + std::string::npos); + EXPECT_NE(debug.find("partition_values={dt}"), std::string::npos); + EXPECT_NE(debug.find("table_filters=[TableFilter{conjunct=VExprContext"), std::string::npos); + EXPECT_NE(debug.find("ColumnDefinition{name=id"), std::string::npos); + EXPECT_NE(debug.find("name_mapping=[legacy_id]"), std::string::npos); + EXPECT_NE(debug.find("ColumnMapping{global_index=0"), std::string::npos); + EXPECT_NE(debug.find("FileBlockColumn{file_column_id=0"), std::string::npos); + ASSERT_TRUE(reader.close().ok()); + + const std::vector formats {FileFormat::ORC, FileFormat::CSV, FileFormat::JSON, + FileFormat::TEXT, FileFormat::JNI, FileFormat::NATIVE, + FileFormat::ARROW}; + const std::vector format_names {"ORC", "CSV", "JSON", "TEXT", + "JNI", "NATIVE", "ARROW"}; + for (size_t idx = 0; idx < formats.size(); ++idx) { + TableReader enum_reader; + ASSERT_TRUE(enum_reader + .init({ + .projected_columns = {}, + .conjuncts = {}, + .format = formats[idx], + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + EXPECT_NE(enum_reader.debug_string().find("format=" + format_names[idx]), + std::string::npos); + } + + const std::vector agg_ops {TPushAggOp::type::NONE, TPushAggOp::type::MINMAX, + TPushAggOp::type::MIX, + TPushAggOp::type::COUNT_ON_INDEX}; + const std::vector agg_names {"NONE", "MINMAX", "MIX", "COUNT_ON_INDEX"}; + for (size_t idx = 0; idx < agg_ops.size(); ++idx) { + TableReader enum_reader; + ASSERT_TRUE(enum_reader + .init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = agg_ops[idx], + }) + .ok()); + EXPECT_NE(enum_reader.debug_string().find("push_down_agg_type=" + agg_names[idx]), + std::string::npos); + } +} + +TEST(TableReaderTest, AnnotateProjectedColumnUsesCurrentHistorySchemaForNestedTypes) { + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(200); + + auto profile_field = external_struct_field( + "profile", 20, + {external_array_field("old_scores", 21, external_schema_field("old_score", 22), + {"scores"}), + external_map_field("old_props", 23, external_schema_field("old_key", 24), + external_schema_field("old_value", 25), {"props"})}, + {"user_profile"}); + scan_params.__set_history_schema_info( + {external_schema(100, {external_schema_field("ignored_profile", 10)}), + external_schema(200, {profile_field})}); + + const auto int_type = std::make_shared(); + const auto string_type = std::make_shared(); + auto scores_type = std::make_shared(int_type); + auto props_type = std::make_shared(string_type, string_type); + auto profile_type = std::make_shared(DataTypes {scores_type, props_type}, + Strings {"scores", "props"}); + + ColumnDefinition profile_column = make_table_column(-1, "user_profile", profile_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &profile_column).ok()); + + EXPECT_EQ(profile_column.get_identifier_field_id(), 20); + EXPECT_EQ(profile_column.name_mapping, std::vector({"user_profile"})); + ASSERT_TRUE(context.schema_column.has_value()); + ASSERT_EQ(context.schema_column->children.size(), 2); + EXPECT_EQ(context.schema_column->children[0].name, "old_scores"); + EXPECT_EQ(context.schema_column->children[0].get_identifier_field_id(), 21); + ASSERT_EQ(context.schema_column->children[0].children.size(), 1); + EXPECT_EQ(context.schema_column->children[0].children[0].name, "element"); + EXPECT_EQ(context.schema_column->children[0].children[0].get_identifier_field_id(), 22); + ASSERT_EQ(context.schema_column->children[1].children.size(), 2); + EXPECT_EQ(context.schema_column->children[1].name, "old_props"); + EXPECT_EQ(context.schema_column->children[1].children[0].name, "key"); + EXPECT_EQ(context.schema_column->children[1].children[0].get_identifier_field_id(), 24); + EXPECT_EQ(context.schema_column->children[1].children[1].name, "value"); + EXPECT_EQ(context.schema_column->children[1].children[1].get_identifier_field_id(), 25); +} + +TEST(TableReaderTest, ComplexRematerializeCastsScalarChildToTableType) { + const auto string_type = std::make_shared(); + const auto nullable_string_type = make_nullable(string_type); + const auto file_struct_type = make_nullable(std::make_shared( + DataTypes {nullable_string_type, string_type}, Strings {"country", "city"})); + auto file_struct_column = make_file_column(2, "struct_column", file_struct_type); + file_struct_column.children = {make_file_column(0, "country", nullable_string_type), + make_file_column(1, "city", string_type)}; + std::vector file_schema = {file_struct_column}; + + const auto table_struct_type = make_nullable(std::make_shared( + DataTypes {nullable_string_type, nullable_string_type}, Strings {"country", "city"})); + auto country_child = make_table_column(0, "country", nullable_string_type); + auto city_child = make_table_column(1, "city", nullable_string_type); + auto table_struct_column = make_table_column(2, "struct_column", table_struct_type); + table_struct_column.children = {country_child, city_child}; + std::vector projected_columns = {table_struct_column}; + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(eos); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); + + const auto& result_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& struct_result = + assert_cast(result_nullable.get_nested_column()); + ASSERT_EQ(struct_result.get_columns().size(), 2); + const auto& country_column = assert_cast(struct_result.get_column(0)); + const auto& city_column = assert_cast(struct_result.get_column(1)); + const auto& country_values = + assert_cast(country_column.get_nested_column()); + const auto& city_values = assert_cast(city_column.get_nested_column()); + ASSERT_EQ(city_column.size(), 2); + EXPECT_FALSE(city_column.is_null_at(0)); + EXPECT_FALSE(city_column.is_null_at(1)); + EXPECT_EQ(country_values.get_data_at(0).to_string(), "USA"); + EXPECT_EQ(country_values.get_data_at(1).to_string(), "UK"); + EXPECT_EQ(city_values.get_data_at(0).to_string(), "New York"); + EXPECT_EQ(city_values.get_data_at(1).to_string(), "London"); +} + +TEST(TableReaderTest, ComplexRematerializeCastsNonNullableScalarChildWithNullableFileType) { + const auto int_type = std::make_shared(); + const auto bigint_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto nullable_bigint_type = make_nullable(bigint_type); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto column = ColumnInt32::create(); + column->insert_value(10); + column->insert_value(20); + ColumnPtr result_column = std::move(column); + const auto status = reader._cast_column_to_type(&result_column, nullable_int_type, + nullable_bigint_type, "struct_column.a"); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& result_nullable = assert_cast(*result_column); + const auto& child_values = assert_cast(result_nullable.get_nested_column()); + ASSERT_EQ(result_nullable.size(), 2); + EXPECT_FALSE(result_nullable.is_null_at(0)); + EXPECT_FALSE(result_nullable.is_null_at(1)); + EXPECT_EQ(child_values.get_element(0), 10); + EXPECT_EQ(child_values.get_element(1), 20); +} + +TEST(TableReaderTest, ReopenSplitAfterClose) { + const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const std::vector file_paths = { + (test_dir / "split_1.parquet").string(), + (test_dir / "split_2.parquet").string(), + (test_dir / "split_3.parquet").string(), + }; + write_parquet_file(file_paths[0], 1, "one"); + write_parquet_file(file_paths[1], 2, "two"); + write_parquet_file(file_paths[2], 3, "three"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "value", std::make_shared())); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(1, 1, 0))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + // Simulate the scanner lifecycle for three different splits: + // init() once, then repeat prepare_split() -> get_block() -> close(). + // This verifies TableReader::close() fully releases the previous low-level reader and task + // state, so a later prepare_split() can open and read a new split on the same TableReader. + // The table-level conjunct is also rebuilt for each split. The projection order puts value + // before id, so the pushed conjunct has to be rewritten to the ParquetReader file-local block + // position every time a new split is opened. + std::vector ids; + std::vector values; + for (const auto& file_path : file_paths) { + auto split_options = build_split_options(file_path); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& value_column = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& id_column = + assert_cast(expect_not_null_table_column(block, 1)); + ASSERT_EQ(id_column.size(), 1); + ASSERT_EQ(value_column.size(), 1); + ids.push_back(id_column.get_element(0)); + values.push_back(value_column.get_data_at(0).to_string()); + + ASSERT_TRUE(reader.close().ok()); + } + + EXPECT_EQ(ids, std::vector({1, 2, 3})); + EXPECT_EQ(values, std::vector({"one", "two", "three"})); + + std::filesystem::remove_all(test_dir); +} + +// Scenario: requests without file-local row conjuncts do not produce a row-level survivor bitmap, +// so TableReader must not enable condition cache. +TEST(TableReaderTest, ConditionCacheSkipsRequestWithoutFileLocalConjuncts) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(fake_state->condition_cache_ctx, nullptr); + EXPECT_EQ(reader.condition_cache_hit_count(), 0); + ASSERT_TRUE(reader.close().ok()); +} + +// Scenario: runtime filters can arrive late and are not represented by the stable predicate digest. +// A MISS must not insert a bitmap for `stable predicate AND runtime filter` under the stable digest. +TEST(TableReaderTest, ConditionCacheSkipsRuntimeFilterConjunct) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, runtime_filter_wrapper_expr( + table_int32_greater_than_expr(0, 0, 0)))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(fake_state->condition_cache_ctx, nullptr); + EXPECT_EQ(reader.condition_cache_hit_count(), 0); + ASSERT_TRUE(reader.close().ok()); +} + +// Scenario: table-format delete files/deletion vectors are outside the data-file cache key. When +// TableReader injects delete conjuncts into the file scan request, condition cache must be disabled +// for that split. +TEST(TableReaderTest, ConditionCacheSkipsRequestWithDeleteConjuncts) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->inject_delete_conjunct = true; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 0))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(fake_state->condition_cache_ctx, nullptr); + EXPECT_EQ(reader.condition_cache_hit_count(), 0); + ASSERT_TRUE(reader.close().ok()); +} + +// Scenario: a MISS bitmap is safe to publish only after the physical reader reaches EOF. This test +// returns EOF together with the first batch and verifies TableReader publishes the marked bitmap. +TEST(TableReaderTest, ConditionCacheMissPublishesBitmapAfterReaderEof) { + ScopedConditionCacheForTest cache; + + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->total_rows = ConditionCacheContext::GRANULE_SIZE; + fake_state->condition_cache_base_granule = 7; + fake_state->condition_cache_num_granules = 1; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 0))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->condition_cache_ctx, nullptr); + EXPECT_FALSE(fake_state->condition_cache_ctx->is_hit); + + segment_v2::ConditionCache::ExternalCacheKey legacy_key("fake-table-reader-input", 0, -1, 7, 0, + -1); + segment_v2::ConditionCacheHandle handle; + EXPECT_FALSE(cache.get()->lookup(legacy_key, &handle)); + segment_v2::ConditionCache::ExternalCacheKey key( + "fake-table-reader-input", 0, -1, 7, 0, -1, + segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION); + ASSERT_TRUE(cache.get()->lookup(key, &handle)); + const auto cached_bitmap = handle.get_filter_result(); + ASSERT_NE(cached_bitmap, nullptr); + ASSERT_FALSE(cached_bitmap->empty()); + EXPECT_EQ(cached_bitmap->size(), 1); + EXPECT_TRUE((*cached_bitmap)[0]); + EXPECT_EQ(handle.get_base_granule(), 7); + + ASSERT_TRUE(reader.close().ok()); +} + +// Scenario: LIMIT/cancel can close a reader before it reaches EOF. TableReader must drop the MISS +// bitmap because unvisited granules would still be false and unsafe for future cache hits. +TEST(TableReaderTest, ConditionCacheMissIsDroppedWhenReaderClosesBeforeEof) { + ScopedConditionCacheForTest cache; + + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->total_rows = ConditionCacheContext::GRANULE_SIZE; + fake_state->eof_with_first_batch = false; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 0))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->condition_cache_ctx, nullptr); + EXPECT_FALSE(fake_state->condition_cache_ctx->is_hit); + + ASSERT_TRUE(reader.close().ok()); + segment_v2::ConditionCache::ExternalCacheKey key( + "fake-table-reader-input", 0, -1, 7, 0, -1, + segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION); + segment_v2::ConditionCacheHandle handle; + EXPECT_FALSE(cache.get()->lookup(key, &handle)); +} + +// Scenario: a stop request can arrive while a physical read is in progress. Even if the reader +// converts that stop into eof=true, TableReader must not publish the partially visited MISS bitmap. +TEST(TableReaderTest, ConditionCacheMissIsDroppedWhenStopTurnsReadIntoEof) { + ScopedConditionCacheForTest cache; + + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + auto io_ctx = std::make_shared(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->total_rows = ConditionCacheContext::GRANULE_SIZE * 2; + fake_state->condition_cache_num_granules = 2; + fake_state->stop_during_read = true; + fake_state->io_ctx = io_ctx; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 0))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(io_ctx->should_stop); + EXPECT_EQ(block.rows(), 2); + ASSERT_NE(fake_state->condition_cache_ctx, nullptr); + EXPECT_FALSE(fake_state->condition_cache_ctx->is_hit); + + segment_v2::ConditionCache::ExternalCacheKey key( + "fake-table-reader-input", 0, -1, 7, 0, -1, + segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION); + segment_v2::ConditionCacheHandle handle; + EXPECT_FALSE(cache.get()->lookup(key, &handle)); + + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, PushDownCountFromNewParquetReader) { + const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_count_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 5); + EXPECT_FALSE(is_column_const(*block.get_by_position(0).column)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, TableLevelCountUsesAssignedRowCount) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_table_count_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + TQueryOptions query_options; + query_options.__set_batch_size(2); + RuntimeState state {query_options, TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + auto split_options = build_split_options(file_path); + set_table_level_row_count(&split_options, 5); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.rows(), 2); + + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.rows(), 2); + + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.rows(), 1); + + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 0); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownMinMaxFromNewParquetReader) { + const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_minmax_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {3, 1, 5, 2}, {30, 10, 50, 20}, + {"three", "one", "five", "two"}, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(1, "score", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::MINMAX, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + const auto& score_column = + assert_cast(expect_not_null_table_column(block, 1)); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 5); + EXPECT_EQ(score_column.get_element(0), 10); + EXPECT_EQ(score_column.get_element(1), 50); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownMinMaxFallsBackForFileToTableCast) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_minmax_cast_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {3, 1, 5, 2}, {30, 10, 50, 20}, + {"three", "one", "five", "two"}, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::MINMAX, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status; + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(id_column.get_element(0), 3); + EXPECT_EQ(id_column.get_element(1), 1); + + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& second_id_column = + assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(second_id_column.get_element(0), 5); + EXPECT_EQ(second_id_column.get_element(1), 2); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownMinMaxFromProjectedStructLeaf) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_minmax_struct_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_struct_parquet_file(file_path, {3, 1, 5, 2}, 2); + + const auto int_type = std::make_shared(); + auto id_child = make_table_column(0, "id", int_type); + auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"id"}); + auto struct_column = make_table_column(100, "s", struct_type); + struct_column.children = {id_child}; + std::vector projected_columns = {struct_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::MINMAX, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status; + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& struct_result = + assert_cast(expect_not_null_table_column(block, 0)); + ASSERT_EQ(struct_result.get_columns().size(), 1); + const auto& ids = assert_cast( + expect_not_null_nullable_nested_column(struct_result.get_column(0))); + EXPECT_EQ(ids.get_element(0), 1); + EXPECT_EQ(ids.get_element(1), 5); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownMinMaxFallsBackForProjectedListStructLeaf) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_minmax_list_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_list_struct_parquet_file(file_path); + + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + auto element_type = std::make_shared( + DataTypes {nullable_int_type, nullable_int_type}, Strings {"a", "b"}); + auto nullable_element_type = make_nullable(element_type); + auto list_column = + make_table_column(100, "xs", std::make_shared(nullable_element_type)); + std::vector projected_columns = {list_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::MINMAX, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status; + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 3); + const auto& array_result = + assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(array_result.get_offsets()[0], 2); + EXPECT_EQ(array_result.get_offsets()[1], 3); + EXPECT_EQ(array_result.get_offsets()[2], 4); + const auto& nullable_elements = assert_cast(array_result.get_data()); + for (const auto is_null : nullable_elements.get_null_map_data()) { + EXPECT_EQ(is_null, 0); + } + const auto& element_struct = + assert_cast(nullable_elements.get_nested_column()); + ASSERT_EQ(element_struct.get_columns().size(), 2); + const auto& a_values = assert_cast( + expect_not_null_nullable_nested_column(element_struct.get_column(0))); + EXPECT_EQ(a_values.get_element(0), 10); + EXPECT_EQ(a_values.get_element(1), 20); + EXPECT_EQ(a_values.get_element(2), 30); + EXPECT_EQ(a_values.get_element(3), 40); + const auto& b_values = assert_cast( + expect_not_null_nullable_nested_column(element_struct.get_column(1))); + EXPECT_EQ(b_values.get_element(0), 11); + EXPECT_EQ(b_values.get_element(1), 21); + EXPECT_EQ(b_values.get_element(2), 31); + EXPECT_EQ(b_values.get_element(3), 41); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedListStructReadsSelectedElementChild) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_list_projection_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_list_struct_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto a_child = make_table_column(0, "a", int_type); + auto element_type = std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto nullable_element_type = make_nullable(element_type); + auto element_child = make_table_column(0, "element", nullable_element_type); + element_child.children = {a_child}; + auto list_column = + make_table_column(100, "xs", std::make_shared(nullable_element_type)); + list_column.children = {element_child}; + std::vector projected_columns = {list_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 3); + const auto& array_result = + assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(array_result.get_offsets()[0], 2); + EXPECT_EQ(array_result.get_offsets()[1], 3); + EXPECT_EQ(array_result.get_offsets()[2], 4); + const auto& nullable_elements = assert_cast(array_result.get_data()); + const auto& element_struct = + assert_cast(nullable_elements.get_nested_column()); + ASSERT_EQ(element_struct.get_columns().size(), 1); + const auto& a_values = assert_cast( + expect_not_null_nullable_nested_column(element_struct.get_column(0))); + EXPECT_EQ(a_values.get_element(0), 10); + EXPECT_EQ(a_values.get_element(1), 20); + EXPECT_EQ(a_values.get_element(2), 30); + EXPECT_EQ(a_values.get_element(3), 40); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedListStructReordersRenamedAndMissingElementChildren) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_list_schema_evolution_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_list_struct_parquet_file(file_path); + + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto string_type = std::make_shared(); + auto b_child = make_table_column(1, "renamed_b", nullable_int_type); + b_child.name_mapping = {"b"}; + auto missing_child = make_table_column(99, "missing_child", string_type); + auto a_child = make_table_column(0, "renamed_a", nullable_int_type); + a_child.name_mapping = {"a"}; + auto element_type = std::make_shared( + DataTypes {nullable_int_type, string_type, nullable_int_type}, + Strings {"renamed_b", "missing_child", "renamed_a"}); + auto nullable_element_type = make_nullable(element_type); + auto element_child = make_table_column(0, "element", nullable_element_type); + element_child.children = {b_child, missing_child, a_child}; + auto list_column = + make_table_column(100, "xs", std::make_shared(nullable_element_type)); + list_column.children = {element_child}; + std::vector projected_columns = {list_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 3); + const auto& array_result = + assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(array_result.get_offsets()[0], 2); + EXPECT_EQ(array_result.get_offsets()[1], 3); + EXPECT_EQ(array_result.get_offsets()[2], 4); + const auto& nullable_elements = assert_cast(array_result.get_data()); + const auto& element_struct = + assert_cast(nullable_elements.get_nested_column()); + ASSERT_EQ(element_struct.get_columns().size(), 3); + const auto& b_values = assert_cast( + expect_not_null_nullable_nested_column(element_struct.get_column(0))); + const auto& missing_values = element_struct.get_column(1); + const auto& a_values = assert_cast( + expect_not_null_nullable_nested_column(element_struct.get_column(2))); + EXPECT_EQ(b_values.get_element(0), 11); + EXPECT_EQ(b_values.get_element(1), 21); + EXPECT_EQ(b_values.get_element(2), 31); + EXPECT_EQ(b_values.get_element(3), 41); + expect_nullable_column_all_null(missing_values); + EXPECT_EQ(a_values.get_element(0), 10); + EXPECT_EQ(a_values.get_element(1), 20); + EXPECT_EQ(a_values.get_element(2), 30); + EXPECT_EQ(a_values.get_element(3), 40); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +// Scenario: when every projected array-element struct child is missing/default-only, the reader +// still receives a full element projection and can materialize the default child without crashing. +TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildFallsBackToFullElement) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_list_only_missing_child_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_list_struct_parquet_file(file_path); + + const auto string_type = std::make_shared(); + auto missing_child = make_table_column(99, "missing_child", string_type); + auto element_type = + std::make_shared(DataTypes {string_type}, Strings {"missing_child"}); + auto nullable_element_type = make_nullable(element_type); + auto element_child = make_table_column(0, "element", nullable_element_type); + element_child.children = {missing_child}; + auto list_column = + make_table_column(100, "xs", std::make_shared(nullable_element_type)); + list_column.children = {element_child}; + std::vector projected_columns = {list_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 3); + const auto& array_result = + assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(array_result.get_offsets()[0], 2); + EXPECT_EQ(array_result.get_offsets()[1], 3); + EXPECT_EQ(array_result.get_offsets()[2], 4); + const auto& nullable_elements = assert_cast(array_result.get_data()); + const auto& element_struct = + assert_cast(nullable_elements.get_nested_column()); + ASSERT_EQ(element_struct.get_columns().size(), 1); + expect_nullable_column_all_null(element_struct.get_column(0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownMinMaxFallsBackForProjectedMapValueStructLeaf) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_minmax_map_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_map_struct_parquet_file(file_path); + + const auto key_type = std::make_shared(); + const auto string_type = std::make_shared(); + const auto nullable_string_type = make_nullable(string_type); + auto b_child = make_table_column(1, "b", nullable_string_type); + auto value_type = + std::make_shared(DataTypes {nullable_string_type}, Strings {"b"}); + auto nullable_value_type = make_nullable(value_type); + auto value_child = make_table_column(1, "value", nullable_value_type); + value_child.children = {b_child}; + auto map_column = make_table_column( + 100, "kv", std::make_shared(key_type, nullable_value_type)); + map_column.children = {value_child}; + std::vector projected_columns = {map_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::MINMAX, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 3); + const auto& map_result = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(map_result.get_offsets()[0], 2); + EXPECT_EQ(map_result.get_offsets()[1], 3); + EXPECT_EQ(map_result.get_offsets()[2], 3); + const auto& keys = assert_cast( + expect_not_null_nullable_nested_column(map_result.get_keys())); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(1), 2); + EXPECT_EQ(keys.get_element(2), 3); + const auto& nullable_values = assert_cast(map_result.get_values()); + for (const auto is_null : nullable_values.get_null_map_data()) { + EXPECT_EQ(is_null, 0); + } + const auto& value_struct = + assert_cast(nullable_values.get_nested_column()); + ASSERT_EQ(value_struct.get_columns().size(), 1); + const auto& b_values = assert_cast( + expect_not_null_nullable_nested_column(value_struct.get_column(0))); + EXPECT_EQ(b_values.get_data_at(0).to_string(), "ma"); + EXPECT_EQ(b_values.get_data_at(1).to_string(), "mb"); + EXPECT_EQ(b_values.get_data_at(2).to_string(), "mc"); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedMapValueStructReordersRenamedAndMissingChildren) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_map_schema_evolution_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_map_struct_parquet_file(file_path); + + const auto key_type = std::make_shared(); + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto string_type = std::make_shared(); + const auto nullable_string_type = make_nullable(string_type); + auto b_child = make_table_column(1, "renamed_b", nullable_string_type); + b_child.name_mapping = {"b"}; + auto missing_child = make_table_column(99, "missing_child", string_type); + auto a_child = make_table_column(0, "renamed_a", nullable_int_type); + a_child.name_mapping = {"a"}; + auto value_type = std::make_shared( + DataTypes {nullable_string_type, string_type, nullable_int_type}, + Strings {"renamed_b", "missing_child", "renamed_a"}); + auto nullable_value_type = make_nullable(value_type); + auto value_child = make_table_column(1, "value", nullable_value_type); + value_child.children = {b_child, missing_child, a_child}; + auto map_column = make_table_column( + 100, "kv", std::make_shared(key_type, nullable_value_type)); + map_column.children = {value_child}; + std::vector projected_columns = {map_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 3); + const auto& map_result = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(map_result.get_offsets()[0], 2); + EXPECT_EQ(map_result.get_offsets()[1], 3); + EXPECT_EQ(map_result.get_offsets()[2], 3); + const auto& keys = assert_cast( + expect_not_null_nullable_nested_column(map_result.get_keys())); + EXPECT_EQ(keys.get_element(0), 1); + EXPECT_EQ(keys.get_element(1), 2); + EXPECT_EQ(keys.get_element(2), 3); + const auto& nullable_values = assert_cast(map_result.get_values()); + const auto& value_struct = + assert_cast(nullable_values.get_nested_column()); + ASSERT_EQ(value_struct.get_columns().size(), 3); + const auto& b_values = assert_cast( + expect_not_null_nullable_nested_column(value_struct.get_column(0))); + const auto& missing_values = value_struct.get_column(1); + const auto& a_values = assert_cast( + expect_not_null_nullable_nested_column(value_struct.get_column(2))); + EXPECT_EQ(b_values.get_data_at(0).to_string(), "ma"); + EXPECT_EQ(b_values.get_data_at(1).to_string(), "mb"); + EXPECT_EQ(b_values.get_data_at(2).to_string(), "mc"); + expect_nullable_column_all_null(missing_values); + EXPECT_EQ(a_values.get_element(0), 10); + EXPECT_EQ(a_values.get_element(1), 20); + EXPECT_EQ(a_values.get_element(2), 30); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, MaterializeMapKeyStructReordersRenamedChildren) { + const auto int_type = std::make_shared(); + const auto string_type = std::make_shared(); + const auto file_key_type = + std::make_shared(DataTypes {int_type, string_type}, Strings {"a", "b"}); + const auto table_key_type = std::make_shared( + DataTypes {string_type, int_type}, Strings {"renamed_b", "renamed_a"}); + const auto file_map_type = std::make_shared(file_key_type, int_type); + const auto table_map_type = std::make_shared(table_key_type, int_type); + + ColumnMapping a_mapping; + a_mapping.table_column_name = "renamed_a"; + a_mapping.file_column_name = "a"; + a_mapping.file_local_id = 0; + a_mapping.table_type = int_type; + a_mapping.file_type = int_type; + a_mapping.is_trivial = true; + + ColumnMapping b_mapping; + b_mapping.table_column_name = "renamed_b"; + b_mapping.file_column_name = "b"; + b_mapping.file_local_id = 1; + b_mapping.table_type = string_type; + b_mapping.file_type = string_type; + b_mapping.is_trivial = true; + + ColumnMapping key_mapping; + key_mapping.table_column_name = "key"; + key_mapping.file_column_name = "key"; + key_mapping.file_local_id = 0; + key_mapping.table_type = table_key_type; + key_mapping.file_type = file_key_type; + key_mapping.is_trivial = false; + key_mapping.child_mappings = {b_mapping, a_mapping}; + + ColumnMapping value_mapping; + value_mapping.table_column_name = "value"; + value_mapping.file_column_name = "value"; + value_mapping.file_local_id = 1; + value_mapping.table_type = int_type; + value_mapping.file_type = int_type; + value_mapping.is_trivial = true; + + ColumnMapping map_mapping; + map_mapping.table_column_name = "kv"; + map_mapping.file_column_name = "kv"; + map_mapping.table_type = table_map_type; + map_mapping.file_type = file_map_type; + map_mapping.is_trivial = false; + map_mapping.child_mappings = {key_mapping, value_mapping}; + + auto a_keys = ColumnInt32::create(); + a_keys->insert_value(10); + a_keys->insert_value(20); + a_keys->insert_value(30); + auto b_keys = ColumnString::create(); + b_keys->insert_value("x"); + b_keys->insert_value("y"); + b_keys->insert_value("z"); + MutableColumns key_children; + key_children.push_back(std::move(a_keys)); + key_children.push_back(std::move(b_keys)); + auto key_column = ColumnStruct::create(std::move(key_children)); + + auto value_column = ColumnInt32::create(); + value_column->insert_value(100); + value_column->insert_value(200); + value_column->insert_value(300); + auto offsets_column = ColumnArray::ColumnOffsets::create(); + offsets_column->insert_value(2); + offsets_column->insert_value(3); + ColumnPtr file_column = ColumnMap::create(std::move(key_column), std::move(value_column), + std::move(offsets_column)); + + TableReaderMaterializeTestHelper reader; + ColumnPtr result_column; + ASSERT_TRUE(reader._materialize_map_mapping_column(map_mapping, file_column, 2, &result_column) + .ok()); + + const auto& result_map = assert_cast(*result_column); + EXPECT_EQ(result_map.get_offsets()[0], 2); + EXPECT_EQ(result_map.get_offsets()[1], 3); + const auto& result_key = assert_cast(result_map.get_keys()); + ASSERT_EQ(result_key.get_columns().size(), 2); + const auto& b_result = assert_cast(result_key.get_column(0)); + const auto& a_result = assert_cast(result_key.get_column(1)); + EXPECT_EQ(b_result.get_data_at(0).to_string(), "x"); + EXPECT_EQ(b_result.get_data_at(1).to_string(), "y"); + EXPECT_EQ(b_result.get_data_at(2).to_string(), "z"); + EXPECT_EQ(a_result.get_element(0), 10); + EXPECT_EQ(a_result.get_element(1), 20); + EXPECT_EQ(a_result.get_element(2), 30); + + const auto& result_value = assert_cast(result_map.get_values()); + EXPECT_EQ(result_value.get_element(0), 100); + EXPECT_EQ(result_value.get_element(1), 200); + EXPECT_EQ(result_value.get_element(2), 300); +} + +// Scenario: map value struct materialization follows DataTypeStruct field order even when +// ColumnMapping children arrive in a different order from projected ColumnDefinition children. +TEST(TableReaderTest, MaterializeMapValueStructUsesTableTypeOrder) { + const auto key_type = std::make_shared(); + const auto string_type = std::make_shared(); + const auto file_value_type = std::make_shared( + DataTypes {string_type, string_type}, Strings {"full_name", "gender"}); + const auto table_value_type = std::make_shared( + DataTypes {string_type, string_type}, Strings {"full_name", "gender"}); + const auto file_map_type = std::make_shared(key_type, file_value_type); + const auto table_map_type = std::make_shared(key_type, table_value_type); + + ColumnMapping full_name_mapping; + full_name_mapping.table_column_name = "full_name"; + full_name_mapping.file_column_name = "full_name"; + full_name_mapping.file_local_id = 0; + full_name_mapping.table_type = string_type; + full_name_mapping.file_type = string_type; + full_name_mapping.is_trivial = true; + + ColumnMapping gender_mapping; + gender_mapping.table_column_name = "gender"; + gender_mapping.file_column_name = "gender"; + gender_mapping.file_local_id = 1; + gender_mapping.table_type = string_type; + gender_mapping.file_type = string_type; + gender_mapping.is_trivial = true; + + ColumnMapping value_mapping; + value_mapping.table_column_name = "value"; + value_mapping.file_column_name = "value"; + value_mapping.file_local_id = 1; + value_mapping.table_type = table_value_type; + value_mapping.file_type = file_value_type; + value_mapping.is_trivial = false; + value_mapping.child_mappings = {gender_mapping, full_name_mapping}; + + ColumnMapping key_mapping; + key_mapping.table_column_name = "key"; + key_mapping.file_column_name = "key"; + key_mapping.file_local_id = 0; + key_mapping.table_type = key_type; + key_mapping.file_type = key_type; + key_mapping.is_trivial = true; + + ColumnMapping map_mapping; + map_mapping.table_column_name = "new_map_column"; + map_mapping.file_column_name = "new_map_column"; + map_mapping.table_type = table_map_type; + map_mapping.file_type = file_map_type; + map_mapping.is_trivial = false; + map_mapping.child_mappings = {key_mapping, value_mapping}; + + auto key_column = ColumnString::create(); + key_column->insert_value("person10"); + key_column->insert_value("person20"); + + auto full_name_column = ColumnString::create(); + full_name_column->insert_value("Jack"); + full_name_column->insert_value("James Lee"); + auto gender_column = ColumnString::create(); + gender_column->insert_value("Male"); + gender_column->insert_value("Male"); + MutableColumns value_children; + value_children.push_back(std::move(full_name_column)); + value_children.push_back(std::move(gender_column)); + auto value_column = ColumnStruct::create(std::move(value_children)); + + auto offsets_column = ColumnArray::ColumnOffsets::create(); + offsets_column->insert_value(1); + offsets_column->insert_value(2); + ColumnPtr file_column = ColumnMap::create(std::move(key_column), std::move(value_column), + std::move(offsets_column)); + + TableReaderMaterializeTestHelper reader; + ColumnPtr result_column; + ASSERT_TRUE(reader._materialize_map_mapping_column(map_mapping, file_column, 2, &result_column) + .ok()); + + const auto& result_map = assert_cast(*result_column); + const auto& result_value = assert_cast(result_map.get_values()); + ASSERT_EQ(result_value.get_columns().size(), 2); + const auto& full_name_result = assert_cast(result_value.get_column(0)); + const auto& gender_result = assert_cast(result_value.get_column(1)); + EXPECT_EQ(full_name_result.get_data_at(0).to_string(), "Jack"); + EXPECT_EQ(full_name_result.get_data_at(1).to_string(), "James Lee"); + EXPECT_EQ(gender_result.get_data_at(0).to_string(), "Male"); + EXPECT_EQ(gender_result.get_data_at(1).to_string(), "Male"); +} + +TEST(TableReaderTest, PushDownMinMaxOnlyUsesSelectedRowGroupInFileRange) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_minmax_range_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {10, 1, 100}, {100, 10, 1000}, {"ten", "one", "hundred"}, + 1); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::MINMAX, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options_for_row_group_mid(file_path, 1)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 1); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownCountOnlyUsesSelectedRowGroupInFileRange) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_count_range_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}, 1); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options_for_row_group_mid(file_path, 2)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 1); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownCountFallsBackWithTableConjunct) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_count_conjunct_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 1); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(id_column.get_element(0), 3); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownCountFallsBackWithFilter) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_count_predicate_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}, 1); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 1); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + EXPECT_EQ(id_column.get_element(0), 3); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, PushDownMinMaxFallsBackWithoutDirectFileMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_minmax_missing_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + projected_columns.push_back( + make_table_column(99, "missing_id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::MINMAX, + }) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 1); + expect_nullable_column_all_null(*block.get_by_position(0).column); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, OpenReaderBuildsTableFiltersFromConjuncts) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_conjunct_filter_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 3, "three"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "value", std::make_shared())); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(1, 1, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + // open_reader() should convert the table-level conjunct on projected column id 1 into + // _table_filters before ColumnMapper creates the FileScanRequest. ColumnMapper then rewrites + // the conjunct's slot ref from table column id 1 to the file-local block position used by + // ParquetReader. The projection order intentionally puts value before id, so the id filter + // column is not at position 0 in the file block. + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); + ASSERT_EQ(id_column.size(), 1); + EXPECT_EQ(id_column.get_element(0), 3); + + ASSERT_TRUE(reader.close().ok()); + + TableReader filtered_reader; + ASSERT_TRUE(filtered_reader + .init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(1, 1, 4))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + ASSERT_TRUE(filtered_reader.prepare_split(build_split_options(file_path)).ok()); + + block = build_table_block(projected_columns); + eos = false; + ASSERT_TRUE(filtered_reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.get_by_position(1).column->size(), 0); + + ASSERT_TRUE(filtered_reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, OpenReaderPushesVExprPredicateToParquetReader) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_column_predicate_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + // Keep one row per row group so the VExpr ZoneMap path can prune the first two row groups and + // leave only id = 3. + write_int_pair_parquet_file(file_path, {1, 2, 3}, {1, 5, 8}, {"one", "two", "three"}, 1); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(2, "value", std::make_shared())); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(1, 1, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& value_column = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); + ASSERT_EQ(id_column.size(), 1); + ASSERT_EQ(value_column.size(), 1); + EXPECT_EQ(id_column.get_element(0), 3); + EXPECT_EQ(value_column.get_data_at(0).to_string(), "three"); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, VExprPredicateSurvivesReopenSplit) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_predicate_reopen_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const std::vector file_paths = { + (test_dir / "split_1.parquet").string(), + (test_dir / "split_2.parquet").string(), + }; + write_int_pair_parquet_file(file_paths[0], {1, 3}, {10, 30}, {"one", "three"}, 1); + write_int_pair_parquet_file(file_paths[1], {2, 4}, {20, 40}, {"two", "four"}, 1); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + std::vector ids; + for (const auto& file_path : file_paths) { + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& id_column = + assert_cast(expect_not_null_table_column(block, 0)); + ASSERT_EQ(id_column.size(), 1); + ids.push_back(id_column.get_element(0)); + + ASSERT_TRUE(reader.close().ok()); + } + + EXPECT_EQ(ids, std::vector({3, 4})); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) { + const auto int_type = std::make_shared(); + const std::vector projected_columns = { + make_table_column(0, "a", int_type), + make_table_column(1, "b", int_type), + make_table_column(2, "c", int_type), + make_table_column(3, "value", std::make_shared()), + }; + const std::vector file_schema = { + make_file_column(0, "a", int_type), + make_file_column(1, "b", int_type), + make_file_column(2, "c", int_type), + make_file_column(3, "value", std::make_shared()), + }; + + TableColumnMapper mapper; + ASSERT_TRUE(mapper.create_mapping(projected_columns, {}, file_schema).ok()); + + std::vector table_filters; + table_filters.push_back({ + // This test only needs the referenced global indices to drive predicate-column + // placement. Keep the conjunct empty so the assertion focuses on scan-column + // de-duplication rather than expression rewrite/prepare behavior. + .conjunct = nullptr, + .global_indices = {GlobalIndex(0), GlobalIndex(1)}, + }); + table_filters.push_back({ + .conjunct = nullptr, + .global_indices = {GlobalIndex(0), GlobalIndex(2)}, + }); + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request(table_filters, projected_columns, &file_request).ok()); + + // Both filters reference column a. It must still be read once as a predicate column, and a + // predicate column must not be repeated as a non-predicate column. + EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0, 1, 2})); + EXPECT_EQ(projection_ids(file_request.non_predicate_columns), std::vector({3})); + ASSERT_EQ(file_request.local_positions.size(), 4); + EXPECT_EQ(file_request.local_positions.at(LocalColumnId(3)).value(), 0); + EXPECT_EQ(file_request.local_positions.at(LocalColumnId(0)).value(), 1); + EXPECT_EQ(file_request.local_positions.at(LocalColumnId(1)).value(), 2); + EXPECT_EQ(file_request.local_positions.at(LocalColumnId(2)).value(), 3); + const auto predicate_column_ids = projection_ids(file_request.predicate_columns); + const auto non_predicate_column_ids = projection_ids(file_request.non_predicate_columns); + for (const auto predicate_column_id : predicate_column_ids) { + EXPECT_TRUE(std::find(non_predicate_column_ids.begin(), non_predicate_column_ids.end(), + predicate_column_id) == non_predicate_column_ids.end()); + } +} + +TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) { + const auto int_type = std::make_shared(); + const std::vector projected_columns = { + make_table_column(0, "id", int_type), + make_table_column(1, "score", int_type), + }; + const std::vector file_schema = { + make_file_column(0, "id", int_type), + make_file_column(1, "score", int_type), + }; + + TableColumnMapper mapper; + ASSERT_TRUE(mapper.create_mapping(projected_columns, {}, file_schema).ok()); + + TableFilter table_filter { + .conjunct = VExprContext::create_shared(table_int32_greater_than_expr(0, 0, 1)), + .global_indices = {GlobalIndex(0)}, + }; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request).ok()); + + EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); + EXPECT_EQ(projection_ids(file_request.non_predicate_columns), std::vector({1})); + ASSERT_EQ(file_request.local_positions.size(), 2); + EXPECT_EQ(file_request.local_positions.at(LocalColumnId(0)).value(), 1); + EXPECT_EQ(file_request.local_positions.at(LocalColumnId(1)).value(), 0); +} + +TEST(TableReaderTest, CreateScanRequestUsesColumnNameForByNamePredicateMapping) { + const auto int_type = std::make_shared(); + std::vector projected_columns = { + make_table_column(10, "id", int_type), + make_table_column(11, "score", int_type), + }; + const std::vector file_schema = { + make_file_column(0, "ID", int_type), + make_file_column(1, "score", int_type), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + set_name_identifiers(&projected_columns); + ASSERT_TRUE(mapper.create_mapping(projected_columns, {}, file_schema).ok()); + + TableFilter table_filter { + .conjunct = VExprContext::create_shared(table_int32_greater_than_expr(0, 0, 1)), + .global_indices = {GlobalIndex(0)}, + }; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request).ok()); + + EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); + EXPECT_EQ(projection_ids(file_request.non_predicate_columns), std::vector({1})); + ASSERT_EQ(file_request.conjuncts.size(), 1); + const auto* localized_slot = + assert_cast(file_request.conjuncts[0]->root()->children()[0].get()); + EXPECT_EQ(localized_slot->slot_id(), 0); + EXPECT_EQ(localized_slot->column_id(), 1); +} + +TEST(TableReaderTest, OpenReaderPushesMultiColumnConjunctToParquetReader) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_multi_conjunct_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {1, 5, 8}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(2, "value", std::make_shared())); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(1, "score", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_sum_greater_than_expr(1, 1, 2, 2, 8))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + // The conjunct references both id and score, so ColumnMapper must put both file columns into + // predicate_columns and rewrite both slot refs to ParquetReader's file-local block positions. + // ParquetReader then evaluates the expression after all predicate columns have been read. + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& value_column = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); + const auto& score_column = + assert_cast(expect_not_null_table_column(block, 2)); + ASSERT_EQ(id_column.size(), 1); + ASSERT_EQ(score_column.size(), 1); + ASSERT_EQ(value_column.size(), 1); + EXPECT_EQ(id_column.get_element(0), 3); + EXPECT_EQ(score_column.get_element(0), 8); + EXPECT_EQ(value_column.get_data_at(0).to_string(), "three"); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedColumnsFillDefaultForParquetSchemaMismatch) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_schema_mismatch_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + projected_columns.push_back( + make_table_column(99, "missing_value", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + // The table projection asks for field id 99, but the ParquetReader exposes only file-local + // fields 0 and 1. Missing columns are allowed by the current mapper options, so TableReader + // should still use the Parquet row count and fill a default column in table schema. + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(0).column->size(), 1); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, DefaultExprResultMatchesNullableTableType) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_nullable_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + const auto int_type = std::make_shared(); + auto missing_column = make_table_column(99, "c_new", make_nullable(int_type)); + missing_column.default_expr = VExprContext::create_shared( + VLiteral::create_shared(int_type, Field::create_field(42))); + std::vector projected_columns; + projected_columns.push_back(std::move(missing_column)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(eos); + + const auto& result = block.get_by_position(0); + ASSERT_TRUE(result.check_type_and_column_match().ok()); + EXPECT_TRUE(result.type->is_nullable()); + ASSERT_TRUE(result.column->is_nullable()); + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(nullable_column.size(), 1); + EXPECT_EQ(nullable_column.get_null_map_data()[0], 0); + const auto& values = assert_cast(nullable_column.get_nested_column()); + EXPECT_EQ(values.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, DefaultExprAlignsNestedNullableArrayTableType) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_nested_nullable_array_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + const auto bigint_type = std::make_shared(); + const auto array_type = std::make_shared(make_nullable(bigint_type)); + const auto table_type = make_nullable(array_type); + auto missing_column = make_table_column(99, "single_element_groups", table_type); + missing_column.default_expr = VExprContext::create_shared( + std::make_shared(table_type)); + std::vector projected_columns; + projected_columns.push_back(std::move(missing_column)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(eos); + + const auto& result = block.get_by_position(0); + ASSERT_TRUE(result.check_type_and_column_match().ok()); + ASSERT_TRUE(result.column->is_nullable()); + const auto& nullable_column = assert_cast(*result.column); + ASSERT_EQ(nullable_column.size(), 1); + EXPECT_EQ(nullable_column.get_null_map_data()[0], 0); + + const auto& array_column = assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(array_column.size(), 1); + EXPECT_EQ(array_column.get_offsets()[0], 1); + ASSERT_TRUE(array_column.get_data().is_nullable()); + const auto& nested_nullable = assert_cast(array_column.get_data()); + ASSERT_EQ(nested_nullable.size(), 1); + EXPECT_EQ(nested_nullable.get_null_map_data()[0], 0); + const auto& values = assert_cast(nested_nullable.get_nested_column()); + EXPECT_EQ(values.get_element(0), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedColumnsFillMissingParquetColumnWithDefault) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_schema_mismatch_reject_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + projected_columns.push_back( + make_table_column(99, "missing_value", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(eos); + + const auto& result = block.get_by_position(0); + ASSERT_TRUE(result.check_type_and_column_match().ok()); + // A missing scalar column without an explicit default is materialized as a default-value + // column. It may stay constant, so verify through the IColumn interface instead of assuming a + // concrete ColumnString instance. + ASSERT_EQ(result.column->size(), 1); + EXPECT_EQ(result.column->get_data_at(0).to_string(), ""); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_struct_missing_child_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_struct_parquet_file(file_path, 7); + + const auto int_type = std::make_shared(); + const auto string_type = std::make_shared(); + auto id_child = make_table_column(0, "id", int_type); + auto missing_child = make_table_column(99, "missing_child", string_type); + auto struct_type = std::make_shared(DataTypes {int_type, string_type}, + Strings {"id", "missing_child"}); + auto struct_column = make_table_column(100, "s", struct_type); + struct_column.children = {id_child, missing_child}; + std::vector projected_columns = {struct_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& struct_result = + assert_cast(expect_not_null_table_column(block, 0)); + ASSERT_EQ(struct_result.get_columns().size(), 2); + const auto& ids = assert_cast( + expect_not_null_nullable_nested_column(struct_result.get_column(0))); + ASSERT_EQ(struct_result.size(), 1); + EXPECT_EQ(ids.get_element(0), 7); + expect_nullable_column_all_null(struct_result.get_column(1)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ReusedBlockClearsProjectedStructWithNullableChild) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_struct_nullable_child_reuse_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_struct_with_nullable_child_parquet_file(file_path); + + const auto int_type = std::make_shared(); + const auto string_type = std::make_shared(); + const auto nullable_string_type = make_nullable(string_type); + auto id_child = make_table_column(0, "id", int_type); + auto note_child = make_table_column(1, "note", nullable_string_type); + auto missing_child = make_table_column(99, "missing_child", string_type); + auto struct_type = std::make_shared( + DataTypes {int_type, nullable_string_type, string_type}, + Strings {"id", "note", "missing_child"}); + auto struct_column = make_table_column(100, "s", struct_type); + struct_column.children = {id_child, note_child, missing_child}; + std::vector projected_columns = {struct_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + const auto& struct_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& notes = assert_cast(struct_result.get_column(1)); + EXPECT_FALSE(notes.is_null_at(0)); + EXPECT_TRUE(notes.is_null_at(1)); + + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 0); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedRenamedStructPreservesParentDeclaredChildNullability) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_struct_parent_child_nullability_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_struct_with_nullable_child_parquet_file(file_path); + + const auto string_type = std::make_shared(); + const auto nullable_string_type = make_nullable(string_type); + auto renamed_note = make_table_column(1, "renamed_note", nullable_string_type); + renamed_note.name_mapping = {"note"}; + // Iceberg nested schema metadata can omit nullability on this child while the parent DataType + // remains authoritative and declares it nullable. + renamed_note.type = string_type; + auto struct_type = std::make_shared(DataTypes {nullable_string_type}, + Strings {"renamed_note"}); + auto struct_column = make_table_column(100, "s", struct_type); + struct_column.children = {renamed_note}; + std::vector projected_columns = {struct_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(eos); + + const auto& struct_result = + assert_cast(expect_not_null_table_column(block, 0)); + ASSERT_EQ(struct_result.get_columns().size(), 1); + const auto& notes = assert_cast(struct_result.get_column(0)); + EXPECT_FALSE(notes.is_null_at(0)); + EXPECT_EQ(notes.get_data_at(0).to_string(), "seven"); + EXPECT_TRUE(notes.is_null_at(1)); + ASSERT_TRUE(block.get_by_position(0).check_type_and_column_match().ok()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedPartitionColumnUsesSplitPartitionValue) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_partition_value_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + auto partition_column = make_table_column(1, "value", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.partition_values.emplace("value", Field::create_field("p1")); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + // The file has a physical column with the same id/name. The split partition value should still + // take precedence and be materialized by TableReader. + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto partition_value = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& partition_value_data = assert_cast( + expect_not_null_nullable_nested_column(*partition_value)); + ASSERT_EQ(partition_value_data.size(), 1); + EXPECT_EQ(partition_value_data.get_data_at(0).to_string(), "p1"); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedNullPartitionColumnPreservesNull) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_null_partition_value_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + auto partition_column = make_table_column(1, "value", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.partition_values.emplace("value", Field::create_field(Null())); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + expect_nullable_column_all_null(*block.get_by_position(0).column); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ConstantPartitionFilterSkipsSplitWhenFalse) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_constant_partition_filter_skip_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + auto partition_column = make_table_column(0, "part", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 10))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.partition_values.emplace("part", Field::create_field(7)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.get_by_position(0).column->size(), 0); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ConstantPartitionFilterKeepsSplitWhenTrue) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_constant_partition_filter_keep_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + auto partition_column = make_table_column(0, "part", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 1))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.partition_values.emplace("part", Field::create_field(7)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + expect_int32_column_values(*block.get_by_position(0).column, {7}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, RuntimeFilterOnConstantPartitionIsNotPreExecuted) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_constant_runtime_filter"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + auto partition_column = make_table_column(0, "part", std::make_shared()); + partition_column.is_partition_key = true; + projected_columns.push_back(std::move(partition_column)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, runtime_filter_wrapper_expr( + table_int32_greater_than_expr(0, 0, 1)))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.partition_values.emplace("part", Field::create_field(7)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(eos); + expect_int32_column_values(*block.get_by_position(0).column, {7}); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ParquetReaderReadsOnlyRowGroupsInFileRange) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_file_range_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, + {"range_group_one", "range_group_two", "range_group_three"}, 1); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(2, "value", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options_for_row_group_mid(file_path, 1)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + const auto& value_column = + assert_cast(expect_not_null_table_column(block, 1)); + ASSERT_EQ(block.rows(), 1); + EXPECT_EQ(id_column.get_element(0), 2); + EXPECT_EQ(value_column.get_data_at(0).to_string(), "range_group_two"); + + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 0); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedColumnsUseMapperExpressionForSameNameDifferentIdParquetSchema) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_same_name_diff_id_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 1, "one"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(99, "id", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + // The table column has the same name as the Parquet field, but a different field id. + // ColumnMapper should still resolve it by name and build a SlotRef projection from the file + // column into the requested table column. + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + ASSERT_EQ(id_column.size(), 1); + EXPECT_EQ(id_column.get_element(0), 1); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ProjectedColumnsUseMapperExpressionsForParquetSchemaMismatch) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_table_reader_mapper_expr_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "seven"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(1, "value", std::make_shared())); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + // The table projection requests id as BIGINT instead of the file INT, so ColumnMapper should + // build a Cast expression. The second field has the same type and should build a SlotRef + // projection. Both columns should still materialize in table schema order. + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + ASSERT_EQ(block.get_by_position(0).name, "id"); + ASSERT_EQ(block.get_by_position(1).name, "value"); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 0)); + const auto& value_column = + assert_cast(expect_not_null_table_column(block, 1)); + ASSERT_EQ(id_column.size(), 1); + ASSERT_EQ(value_column.size(), 1); + EXPECT_EQ(id_column.get_element(0), 7); + EXPECT_EQ(value_column.get_data_at(0).to_string(), "seven"); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/timestamp_statistics_test.cpp b/be/test/format_v2/timestamp_statistics_test.cpp new file mode 100644 index 00000000000000..6c27b680dc2896 --- /dev/null +++ b/be/test/format_v2/timestamp_statistics_test.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/timestamp_statistics.h" + +#include +#include + +namespace doris::format { + +TEST(TimestampStatisticsTest, DetectsBackwardTimezoneTransitionsInUtcRange) { + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + + // The 2021 fall transition at 06:00 UTC makes local civil time move backward by one hour. + EXPECT_FALSE(utc_timestamp_range_is_monotonic(1636263000, 1636266600, new_york)); + EXPECT_FALSE(utc_timestamp_range_is_monotonic(1636263000, 1636264800, new_york)); + EXPECT_TRUE(utc_timestamp_range_is_monotonic(1636264800, 1636266600, new_york)); + + // A range beginning before the spring-forward transition must continue scanning and find the + // later rollback in the same year. + EXPECT_FALSE(utc_timestamp_range_is_monotonic(1609477200, 1641013200, new_york)); + + // The 2021 spring transition at 07:00 UTC skips civil values but preserves ordering. + EXPECT_TRUE(utc_timestamp_range_is_monotonic(1615703400, 1615707000, new_york)); + + // Corrupt external statistics must disable pruning instead of asserting in the BE. + EXPECT_FALSE(utc_timestamp_range_is_monotonic(2, 1, new_york)); +} + +TEST(TimestampStatisticsTest, FloorsNegativeEpochFractions) { + EXPECT_EQ(floor_epoch_seconds(1001, 1000), 1); + EXPECT_EQ(floor_epoch_seconds(-1, 1000), -1); + EXPECT_EQ(floor_epoch_seconds(-1001, 1000), -2); +} + +} // namespace doris::format diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl index e4e922d16665dd..b481e3a5829803 100644 --- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl +++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl @@ -101,6 +101,23 @@ services: - /usr/lib/iceberg-rest/iceberg-rest-adapter.jar:/opt/jdbc/postgresql.jar - org.apache.iceberg.rest.RESTCatalogServer + trino: + image: trinodb/trino:482 + container_name: doris--iceberg-trino + depends_on: + rest: + condition: service_started + mc: + condition: service_completed_successfully + user: root + networks: + - doris--iceberg + healthcheck: + test: ["CMD-SHELL", "test -d /etc/trino/catalog && command -v trino >/dev/null"] + interval: 5s + timeout: 60s + retries: 120 + minio: image: minio/minio:RELEASE.2025-01-20T14-49-07Z container_name: doris--minio diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run28.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run28.sql index adbfafcf976b33..31d75a8d50be29 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run28.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run28.sql @@ -34,6 +34,56 @@ where batch = 2 and id >= 14; delete from dv_test where id % 2 = 1; +drop table if exists dv_test_orc; +create table dv_test_orc ( + id int, + batch int, + data string +) +using iceberg +tblproperties ( + 'format-version' = '3', + 'write.format.default' = 'orc', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' +); + +insert into dv_test_orc values + (1, 1, 'a'), (2, 1, 'b'), (3, 1, 'c'), (4, 1, 'd'), + (5, 1, 'e'), (6, 1, 'f'), (7, 1, 'g'), (8, 1, 'h'); + +delete from dv_test_orc +where batch = 1 and id in (3, 4, 5); + +insert into dv_test_orc values + (9, 2, 'i'), (10, 2, 'j'), (11, 2, 'k'), (12, 2, 'l'), + (13, 2, 'm'), (14, 2, 'n'), (15, 2, 'o'), (16, 2, 'p'); + +delete from dv_test_orc +where batch = 2 and id >= 14; + +delete from dv_test_orc +where id % 2 = 1; + +drop table if exists dv_test_no_delete; +create table dv_test_no_delete ( + id int, + batch int, + data string +) +using iceberg +tblproperties ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' +); + +insert into dv_test_no_delete values + (1, 1, 'a'), (2, 1, 'b'), (3, 1, 'c'), (4, 1, 'd'), + (5, 1, 'e'), (6, 1, 'f'), (7, 1, 'g'), (8, 1, 'h'); + drop table if exists dv_test_v2; create table dv_test_v2 ( id int, diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md new file mode 100644 index 00000000000000..b05ea798ee46b0 --- /dev/null +++ b/docs/file-scanner-v2-code-review-guide.md @@ -0,0 +1,173 @@ +# FileScannerV2 Code Review Guide + +This guide contains the detailed checklists referenced by +`be/src/format_v2/AGENTS.md`. Read the common checklist for every FileReader review, then apply the +format-specific checklist when reviewing Parquet or ORC. + +## Common FileReader: Indexes and Predicate Filtering + +- Inventory the reader's actual pruning capabilities before evaluating a change: metadata or + statistics, dictionary information, Bloom filters, page/stripe/row indexes, partition/Split + ranges, and format-specific encodings. Record the granularity, supported predicate/type set, + exactness, I/O cost, and conservative fallback for each capability. +- A FileReader consumes only predicates already localized by `TableColumnMapper` in + `FileScanRequest`. It may translate those predicates into format-native indexes or SDK filters, + but it must not reinterpret table-schema identity, defaults, partitions, or table-format + semantics. +- Every index may discard a candidate only when it proves that the candidate cannot match. Missing, + malformed, stale, truncated, unsupported, writer-incompatible, or unsafe metadata must retain the + candidate or return the format's explicit correctness-preserving error. +- Check logical-to-physical identity at every index boundary: file-local root and nested column IDs, + physical leaf IDs, row-group/stripe/page ordinals, byte ranges, file-global row offsets, and + selected row ranges. Index results for one column or unit must never be applied to another. +- Verify metadata semantics for NULL/all-NULL, empty units, NaN, signedness, truncated bounds, + decimal precision/scale, date/timestamp/timezone, string/binary ordering, CHAR padding, and + external-writer differences before trusting min/max or membership information. +- Preserve a cheap-to-expensive pruning order. Do not read or parse a finer index for a file, + row group, stripe, or page already eliminated by a cheaper layer. Measure index read/parse/build + cost as well as the I/O, decompression, decoding, and materialization it avoids. +- Trace each predicate through index pruning, exact format-native filtering, Doris residual VExpr, + delete predicates, and final materialization. A predicate not exactly covered by an earlier layer + must remain in the residual path. +- Preserve SQL three-valued logic and error behavior across AND/OR/NOT, comparisons, IN/NOT IN, + IS NULL, null-safe equality, casts, functions, stateful expressions, and exception-sensitive + operations. Splitting or reordering predicates requires proof of equivalence. +- Predicate columns and lazily read non-predicate columns must refer to the same original rows after + all skips and filters. Skipping must advance every physical reader consistently, including nested + definition/repetition state, offsets, row positions, and subsequent batches. +- Keep row-level deletes, equality deletes, position deletes, table filters, and query predicates in + their specified order. An index optimization must not bypass a delete or use post-filter row + numbering where file-global numbering is required. +- Readers without a native index or lazy-read capability must declare that boundary and preserve + correctness through residual evaluation. Do not add an imitation index in a generic layer merely + to make formats appear uniform. +- Require differential tests that compare exact results and errors with each index/filter + optimization enabled and disabled. Cover missing/invalid indexes, all/none/partially filtered + units, multiple files/Splits/batches, NULL and type boundaries, nested data, deletes, and + external-writer fixtures. + +## Common FileReader: Data and Condition Caches + +- Distinguish the cache layers and their value semantics: remote `FileCache` stores file bytes, + format metadata/page caches store format-specific serialized ranges or parsed metadata, + `ConditionCache` stores predicate survivor granules, and table-format caches may store deletion + vectors or decoded objects. Never reuse an entry as a different representation. +- A cache key must include every input that can change the value: filesystem and canonical path, + stable object/file version, size or mtime where reliable, byte/Split range, format/encoding + context, and predicate digest for filter results. Disable the cache when a stable identity cannot + be established; never trade stale rows for a hit. +- Validate hit, miss, partial coverage, overlapping/subrange reads, eviction, concurrent access, + cancellation, and error paths. A partial cache hit must read or conservatively retain uncovered + data rather than treating it as absent. +- `ConditionCache` can skip only file-global granules explicitly known to contain no surviving row. + Disable or expand the key when Runtime Filters, delete files/vectors, table snapshots, or other + changing semantics are not represented. Publish a miss result only after the physical reader + reaches EOF successfully so unvisited granules cannot become false negatives. +- Cache admission, prefetch, and range merging must follow pruning and lazy materialization. Do not + prefetch output columns or pruned units merely to improve hit rate, and account for read + amplification, request count, memory ownership, and cache pollution. +- Preserve resource accounting and source attribution across local, peer, and remote hits. Require + counters for hit/miss/write/eviction, bytes by source, wait/download time, requests, and avoided + reads so performance claims are diagnosable. +- Require warm/cold, enabled/disabled, overwrite/version-change, partial-range, concurrent, and + cancellation tests. Cached and uncached execution must return identical rows and errors. + +## Common FileReader: Virtual Columns + +- Keep file-coordinate virtual columns distinct from table-format virtual columns. FileReaders may + synthesize reserved file-local `ROW_POSITION` and `GLOBAL_ROWID`; `TableReader` and + `TableColumnMapper` own table semantics such as Iceberg `_row_id`, + `_last_updated_sequence_number`, and Doris Iceberg row locators. +- `ROW_POSITION` is the absolute zero-based physical row in the file, not an output, batch, + selected-row, row-group, stripe, or Split-local ordinal. It must advance across pruned units, + skipped pages/granules, rejected batches, lazy filters, and deletes without renumbering survivors. +- `GLOBAL_ROWID` must be stable and unique for its documented context. Review context version, + backend/file identity, serialization, physical row position, cross-file collisions, and retries; + filtering and batching must not change the generated ID for the same source row. +- Generate virtual values only when requested as output or needed by a predicate/delete. Support + virtual-only scans with no physical projected column, predicate-only virtual columns, selected-row + materialization, and EOF without forcing unrelated file I/O. +- Preserve declared type, nullability, nested shape, and `LocalColumnId`/`LocalIndex` mapping. Do not + let reserved negative IDs collide with invalid IDs, physical columns, table IDs, or block + positions. +- Require tests across multiple files, Splits, row groups/stripes/pages, batches, all rows filtered, + no rows filtered, index/cache skips, lazy materialization, deletes, and virtual-only projection. + Compare virtual values with the same scan when pruning, caching, and lazy reads are disabled. + +## Common FileReader: Performance and Observability + +- Keep index construction, predicate translation, cache lookup, and virtual-column setup out of + per-row and repeated batch paths unless the work is inherently row-local. Avoid repeated schema + traversal, expression cloning, metadata parsing, allocation, and conversion. +- Require format readers to populate the common `ReaderStatistics` accurately where applicable: + filtered/read row groups, Bloom and min/max pruning, filtered group/page/lazy rows, read rows and + bytes, metadata/footer/cache timing, page-index work, predicate time, dictionary rewrite, and + Bloom read time. +- Evaluate performance with representative format versions, writers, data ordering, predicate + selectivity, nested width, remote storage, batch sizes, and warm/cold caches. Report both the + optimization overhead and the avoided work; a low pruning ratio alone is not a defect. + +## Parquet Multi-Level Filtering + +- Use [FileScannerV2 Parquet Scan Design](file-scanner-v2-parquet-scan-design.md) as the detailed + architecture reference. Trace each affected predicate through localization, Row Group planning, + Page ranges, row-level residual evaluation, and final selected-column materialization. +- At Row Group level, check Split ownership and file-global row offsets, then verify Statistics, + Dictionary, and Bloom pruning independently. Dictionary pruning requires complete compatible + encoding. Bloom may prove absence only; a hit is never a matching row. +- Preserve the cost order from cheap to expensive. Footer Statistics should reduce candidates before + Dictionary/Bloom I/O, and ColumnIndex/OffsetIndex should be read only for surviving Row Groups. +- At Page level, require compatible ColumnIndex and OffsetIndex semantics. Check page-to-row mapping, + first/last row boundaries, empty or all-null pages, multi-column range intersection, and conversion + from logical `selected_ranges` to each leaf reader's physical `page_skip_plan`. +- Page skipping must keep every column reader aligned. Skipping values or pages must advance value, + definition, and repetition state consistently, especially for nested/repeated columns whose Page + boundaries do not align across leaves. +- At Row/Batch level, keep SelectionVector positions aligned with original Row Group rows across + dictionary-ID filters, incremental predicates, residual expressions, deletes, and output + materialization. Physical row positions must not be renumbered after pruning. +- Verify lazy materialization avoids reading and decoding non-predicate columns for rejected rows + while advancing all readers correctly. Predicate columns should be read/prefetched first; output + prefetch should wait for survivors when filtering is active. +- Register Parquet Page Cache ranges only for surviving projected Column Chunks, require a stable + file-version key, and assess FileCache, MergeRange, prefetch, requests, and read amplification + together. +- Require counters for Statistics/Dictionary/Bloom pruning, Page Index selected ranges and skipped + rows/pages, raw and filtered rows, dictionary-row filtering, lazy-read savings, cache sources, and + remote I/O. +- Differential tests must cover absent/invalid statistics, missing or partial Page Index, mixed + dictionary/plain encoding, Bloom false positives, NULL/NaN/type conversion, cross-Page batches, + nested/repeated columns, multiple Row Groups/Splits, and all/none filtered. + +## ORC SARG and Index Filtering + +- Trace every pushed predicate from localized `FileScanRequest` through + `build_orc_search_argument()`, ORC `SearchArgumentBuilder`, Stripe selection, SDK RowReader index + pruning, lazy callback filtering, and residual Doris VExpr. +- SARG conversion must be equivalent to the original Doris predicate for every value, including + NULL. Preserve AND/OR/NOT grouping, literal-on-left comparison direction, comparison/IN/NULL + semantics, and wrappers for Runtime Filter, direct-IN, and TopN predicates. +- Verify ORC predicate-domain and literal conversion for integer, floating-point, boolean, string, + binary, varchar, date, decimal, timestamp, and timestamp-instant, including overflow, non-finite + values, signed boundaries, precision/scale, CHAR/VARCHAR, timezone, and NULL. +- Treat schema-evolution casts as SARGable only when truth is preserved in the ORC domain. Review + numeric exactness, decimal widening, date-to-datetime boundary normalization, timestamp precision, + and string/binary casts. Lossy or timezone-changing casts must remain residual. +- For nested predicates, verify struct field name/ordinal traversal and the final ORC type ID. + Unsupported array/map/repeated/missing paths must not target another primitive child. +- Intersect the Split byte window with Stripe ownership before SARG selection, then let ORC RowReader + use row indexes and Bloom filters inside surviving Stripes. SARG must not reintroduce an + out-of-Split Stripe. +- Validate non-adjacent Stripe ranges, all-pruned/no-Stripe cases, file-global row positions, + deletes, and Condition Cache granules after every skipped Stripe or row group. +- Keep SDK filtering and Doris lazy materialization aligned: include the correct filter columns, + preserve selected-row indexes, and decode non-predicate columns only for survivors without + desynchronizing nested vectors or later batches. +- Review SARG cost for large IN lists, deep trees, many Runtime Filters, repeated literal conversion, + Stripe-statistics reads, and SDK index initialization. Build once per reader/Split setup and keep + expensive work out of batch loops. +- Require counters for evaluated/selected groups or Stripes, filtered rows/bytes, groups read, + lazy-filtered rows, I/O, decompression, and decoding. Explain pruning benefit and SARG/index cost. +- Differential tests must cover NULL truth tables, literal-on-left, nested AND/OR/NOT, IN/NOT IN + with NULL, casts, all literal domains, nested structs, unsupported arrays/maps, non-adjacent + Stripes, Split boundaries, row-index strides, Bloom present/absent, and all/none filtered. diff --git a/docs/file-scanner-v2-design.md b/docs/file-scanner-v2-design.md new file mode 100644 index 00000000000000..66b96de1204d14 --- /dev/null +++ b/docs/file-scanner-v2-design.md @@ -0,0 +1,325 @@ +# FileScannerV2 Scan Pipeline Design + +> **Core conclusion:** FileScannerV2 is not primarily about rewriting a file reader. Its purpose is +> to establish stable layer boundaries: Operator/Scheduler owns the control plane, Scanner owns +> query integration and the Split lifecycle, TableReader owns table semantics, and format readers +> own physical reads. All optimizations aim to eliminate unnecessary I/O as early as possible, +> control batch cost, preserve consistent semantics across formats, and make state reusable and +> observable. + +## 1. Design Goals and Boundaries + +FileScannerV2 targets external-data scans. It separates query execution, table-format semantics, +and file-format details into layers that can evolve independently. The design prioritizes durable +boundaries rather than isolated acceleration for a particular format. + +### Core goals + +- Unify the read pipeline for Parquet, ORC, text, JSON, JNI, and other formats. +- Complete pruning and short-circuit evaluation before file I/O whenever possible. +- Isolate table-level semantics from file-local schemas. +- Reuse heavyweight Scanner state across multiple Splits. +- Maintain consistent resource accounting and Profile conventions. + +### Non-goals + +- Do not reimplement every file format inside Scanner. +- Do not force every optimization onto every reader. +- Do not expose file-local column positions to the query layer. +- Do not sacrifice error semantics in order to continue execution. +- Full support for the Load path is currently out of scope. + +> **Design placement rule:** The correct layer for a capability depends on whether it manages the +> query, manages a Split, restores table semantics, or interprets a physical file. Layer boundaries +> take priority over code reuse. + +## 2. Overall Architecture + +V2 divides the scan pipeline into four layers. Upper layers depend only on stable contracts, while +lower layers may evolve independently for each format. + +```mermaid +flowchart TB + Q[Query Plan and Scan Operator] --> S[Scanner Scheduler and Split Source] + S --> F[FileScannerV2 Query Integration] + F --> T[TableReader Table-Semantics Orchestration] + T --> N[Native FileReader] + T --> J[JNI Reader] + N --> P[Parquet / ORC / CSV / Text / JSON] + J --> C[Paimon / Hudi / JDBC / Trino / MaxCompute] + F -. "Profile and Resource Accounting" .-> O[Query Observability] + N -. "FileCache and I/O Statistics" .-> O +``` + +| Layer | Primary responsibilities | Intentionally isolated concerns | +| --- | --- | --- | +| Operator / Scheduler | Select V1 or V2, control concurrency, distribute Splits, and apply late Runtime Filters | Does not understand file-schema mapping or interpret format metadata | +| FileScannerV2 | Maintain Scanner lifecycle, advance Splits, connect query context, predict batch size, handle errors, and collect statistics | Does not decode specific formats or implement table-format delete semantics | +| TableReader | Restore table-level column semantics and manage partition constants, predicates, deletes, Split state, and reader open order | Does not depend on Scanner scheduling | +| Format Reader | Interpret physical formats, metadata, encodings, pages, row groups, and JNI protocols | Does not control query-level concurrency or resource governance | + +> **Primary benefit:** Add a new format by extending a reader, add new table semantics by extending +> TableReader, and add query-level governance in Scanner/Operator. Each change remains in the layer +> that owns it. + +## 3. Core Scan Pipeline + +One Scanner consumes multiple Splits sequentially. The main pipeline advances through a loop rather +than reconstructing the entire scan object for every file. + +```mermaid +sequenceDiagram + participant O as FileScanOperator + participant S as ScannerScheduler + participant X as SplitSource + participant F as FileScannerV2 + participant T as TableReader + participant R as Format Reader + participant U as Upstream Operator + O->>S: Create and schedule Scanner + S->>F: prepare / open + F->>X: Fetch first or next Split + X-->>F: Split descriptor and partition values + S->>F: Refresh late Runtime Filters + F->>T: prepare split + alt Split is pruned early + T-->>F: pruned + F->>X: Continue with next Split + else Split must be read + F->>T: get block + T->>R: Lazily create and open concrete Reader + R-->>T: File-local Block + T-->>F: Table-level Block + F-->>U: Deliver upstream + loop Current Split is not finished + F->>T: get block + T->>R: read next batch + T-->>F: Table-level Block + F-->>U: Deliver upstream + end + F->>X: Advance to next Split + end +``` + +1. **Selection and scheduling:** Operator selects V2 from feature flags, the scan scenario, and the + complete format-support matrix. Multiple Scanners dynamically fetch work from one SplitSource. +2. **One-time initialization:** Expressions, projected columns, I/O Context, and TableReader are + reused throughout the Scanner lifecycle. +3. **Per-Split preparation:** Update only the current file, partition values, delete information, + and the latest available filters. +4. **Open on demand:** Construct the format reader only when data must actually be read, preserving + the opportunity for early pruning. +5. **Repeated delivery:** TableReader produces stable table-level Blocks. Scanner then applies the + common upstream filtering, projection, and statistics path. + +> **Core invariant:** Upstream operators always observe table-level column order and types. Split +> transitions, file-schema differences, cache sources, and concrete formats remain hidden below. + +## 4. Split Lifecycle and Early Pruning + +Split is the most important state-isolation unit in V2. Every transition clears the previous +Split's local state before deciding whether the current Split warrants reader construction. + +```mermaid +stateDiagram-v2 + [*] --> FetchSplit + FetchSplit --> PrepareSplit: Range acquired + PrepareSplit --> Pruned: Partition predicates reject all rows + PrepareSplit --> Ready: Read required + PrepareSplit --> Ignored: Ignorable NOT_FOUND + Ready --> Reading: First get block + Reading --> Reading: Return non-empty Block + Reading --> Finished: Current Split EOF + Reading --> Ignored: NOT_FOUND while reading + Pruned --> FetchSplit + Ignored --> FetchSplit + Finished --> FetchSplit + FetchSplit --> [*]: No more Splits or stopped +``` + +### Why pruning happens during prepare split + +```mermaid +sequenceDiagram + participant S as Scheduler + participant F as FileScannerV2 + participant T as TableReader + participant R as Concrete Reader + S->>F: Inject latest Runtime Filters + F->>T: Current Split and latest filter snapshot + T->>T: Build one-row semantics from partition constants + T->>T: Select only predicates fully answerable by partitions + alt All rows are filtered + T-->>F: Mark Split as pruned + Note over R: No construction, open, or file I/O + else Split may contain matches + T-->>F: Split ready + F->>T: get block + T->>R: Create and open Reader + end +``` + +### Benefits + +- Late Runtime Filters can affect subsequent Splits. +- Unnecessary object-storage requests and metadata reads are avoided. +- Delete-file parsing can also be skipped after pruning. +- All formats share the same Split-pruning semantics. + +### Required constraints + +- Make a pruning decision only when the current partition values fully determine the expression. +- Conservatively retain a Split when the result cannot be determined. +- Pruning, normal completion, and ignored errors must all advance the finished range consistently. +- Cleanup must cover native, JNI, and hybrid child readers. + +## 5. Block Reading and Table-Semantics Restoration + +A file reader returns a file-local Block, while query execution requires a table-level Block. V2 +models the conversion explicitly so schema evolution, partition columns, and virtual columns do not +leak into format readers. + +```mermaid +flowchart LR + A[Table Projection and Predicates] --> B[Global Column Semantics] + B --> C[Column Mapper] + D[File Schema and Local Column Positions] --> C + E[Partition Values / Defaults / Virtual Columns] --> C + C --> F[File Scan Request] + F --> G[Format Reader Reads Required Columns] + G --> H[File-local Block] + H --> I[Type Conversion and Nested-Column Reconstruction] + I --> J[Delete Semantics and Table-level Filtering] + J --> K[Stable Table-level Block] +``` + +| Design object | Problem addressed | Optimization enabled | +| --- | --- | --- | +| Global Index | Expressions use stable table-level positions independent of file-column order | Predicates can be relocated for different file schemas | +| Column Mapper | Handles names, positions, field IDs, missing columns, partition columns, and nested projection uniformly | Reads only required physical columns and enables nested-field pruning | +| File Scan Request | Translates table intent into a local request understood by a format reader | Predicate pushdown, lazy materialization, and dictionary/page/row-group pruning | +| Finalize | Restores file columns to the types, order, and virtual semantics required by the query | Upstream layers remain unaware of file-format differences | + +> **Tradeoff:** The mapping layer adds orchestration cost, but enables cross-format consistency, +> schema evolution, and fine-grained projection and filter optimizations. It is core V2 +> infrastructure. + +## 6. Key Optimizations + +V2 optimization is a continuous pipeline: eliminate work, control the cost of each remaining unit, +and reuse work already performed. + +```mermaid +flowchart TB + A[Eliminate Irrelevant Splits] --> A1[Runtime Filter Partition Pruning] + A --> A2[Constant-Predicate Short Circuit] + B[Eliminate Irrelevant Data] --> B1[Column and Subfield Projection] + B --> B2[Predicate Pushdown and Format-level Pruning] + B --> B3[Delete-Semantics Pushdown] + C[Control Per-batch Cost] --> C1[Small Probe Batch] + C --> C2[Adapt from Materialized Bytes per Row] + D[Reuse and Caching] --> D1[Scanner / TableReader Reuse Across Splits] + D --> D2[FileCache] + D --> D3[Condition Cache and Metadata Cache] + E[Avoid Materialization] --> E1[COUNT / MIN / MAX Aggregate Pushdown] +``` + +| Optimization | Design motivation | Key consideration | +| --- | --- | --- | +| Shared SplitSource with dynamic work assignment | Prevent a Scanner from binding to fixed files and reduce long-tail imbalance | Control concurrency by execution resources, not simply by file count | +| Lazy reader open | Allow pruning before remote I/O and format initialization | Define clear state contracts between prepare and read | +| Adaptive batches | A fixed row count cannot bound memory for wide or nested rows | Sample the final table-level Block's bytes per row; use a small probe without history | +| Projection and predicate localization | Translate table intent into the minimum physical read set | Pushdown must not change final query semantics | +| Layered caches | Reuse remote data and stabilize object-storage access cost | Attribute cache sources accurately to local, remote, and peer reads | +| Aggregate pushdown | Avoid data-page materialization when metadata can answer the query | Disable conservatively when filters or deletes may change the result | + +> **Optimization rule:** First prove that data need not be read, then decide what must be read, and +> finally optimize how much to read at once. Earlier optimizations usually provide greater benefit +> and require stricter correctness boundaries. + +## 7. Format Extension and Hybrid Readers + +V2 does not require every data source to use one physical execution mechanism. TableReader provides +uniform table semantics, while each Split can use native execution, JNI, or a hybrid reader that +dispatches between them. + +```mermaid +flowchart TB + S[Current Split] --> D{Table Format and Split Type} + D -->|Regular File| N[Native TableReader] + D -->|Java Connector| J[JNI TableReader] + D -->|Mixed Splits in One Table| H[Hybrid Reader] + H --> HN[Native Child] + H --> HJ[JNI Child] + N --> U[Uniform Table-level Block] + J --> U + HN --> U + HJ --> U + P[Pruning State / abort split / Profile] -. "Uniform Contract" .-> N + P -. "Uniform Contract" .-> J + P -. "Forward to Active Child" .-> H +``` + +### Adding a new file format + +- Implement schema discovery, reads, and format-level Profile reporting. +- Reuse TableReader mapping, deletes, constants, and finalize logic. +- Declare a capability matrix and select V2 only when every Split is supported. + +### Adding a new table format + +- Add field identity, historical schema, and delete semantics. +- Select native or JNI execution per Split. +- Ensure state queries and cleanup reach the actual child reader. + +> **Incremental migration:** V2 protects compatibility through a capability matrix instead of +> assuming that every format migrates at once. Coverage can expand gradually while retaining the V1 +> fallback path. + +## 8. Observability and Failure Semantics + +Scan optimization remains maintainable only when costs are visible, sources are distinguishable, +and failure semantics are explicit. V2 provides three complementary views: Query Profile, query +resource context, and global metrics. + +```mermaid +flowchart LR + R[FileReader and FileCache Raw Statistics] --> P[Query Profile] + R --> Q[Query Resource Context] + R --> M[Doris Metrics] + P --> P1[Per-query Layer Timings and Counts] + Q --> Q1[Resource Governance and Local/Remote I/O Attribution] + M --> M1[Long-term Node Trends] + C[Condition Cache / Pruning / NOT_FOUND] --> P + C --> Q +``` + +| Failure category | Default semantics | Design rationale | +| --- | --- | --- | +| Query cancellation / should stop | Stop Reader and Scanner loops promptly | Propagate the stop signal through I/O Context to avoid further remote-resource use | +| NOT_FOUND | Return an error by default; skip the current Split only when explicitly configured | Clean reader state and update counters before skipping; do not disguise another error as a missing file | +| Schema / decode / delete-semantics error | Fail immediately | These errors can affect result correctness and must not be swallowed defensively | +| Pruning | Complete the current Split normally | Pruning is an optimization result, not an error, and must be observed separately from Empty/NOT_FOUND | + +> **Observability rule:** Profile explains why one query is slow, ResourceContext explains what that +> query consumed, and DorisMetrics describes overall node health. Their measurements are related but +> not interchangeable. + +## 9. Summary of Design Tradeoffs + +| Design choice | Primary benefit | Cost and constraint | +| --- | --- | --- | +| Scanner, TableReader, and Format Reader layering | Stable responsibilities, extensible formats, and clear test boundaries | Adds translation and state contracts | +| One Scanner consumes multiple Splits | Reuses expressions, caches, and reader-orchestration state | Requires complete isolation of Split-local state | +| Separate table-global and file-local semantics | Supports schema evolution, field mapping, and complex-column pruning | Makes Column Mapper and finalize logic more complex | +| Prune before opening a reader | Maximizes avoided remote I/O and initialization | Can evaluate only predicates that are safe to decide early | +| Adapt batches from actual bytes | Controls memory peaks for wide and nested rows | Requires an initial probe and uses a dynamic estimate | +| Capability matrix with V1 fallback | Enables incremental migration without exposing incomplete format paths | Requires both paths to preserve equivalent semantics during migration | + +> **In one sentence:** FileScannerV2 separates whether to read, what to read, how to read, how to +> restore table semantics, and how to account for cost, allowing correctness, performance, and +> extensibility to evolve independently. + +## Further Reading + +- [FileScannerV2 profiling and pruning PR](https://github.com/apache/doris/pull/65449) diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md new file mode 100644 index 00000000000000..63f63819670dd2 --- /dev/null +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -0,0 +1,503 @@ +# FileScannerV2 Parquet Scan Pipeline Design + +> **Reading goal:** Understand how the FileScannerV2 Parquet Reader progressively pushes +> table-level predicates down to Split, Row Group, Page, and Row granularity, then uses indexes, +> lazy materialization, and layered caches to reduce unnecessary I/O and decoding. + +## 1. Design Goals and Core Conclusions + +Parquet V2 is not simply a replacement decoder. It divides a file scan into a **planning phase** and +an **execution phase**: first eliminate impossible matches with lightweight metadata, then read only +predicate columns for surviving ranges, and finally defer output-column reads until matches exist. + +> **In one sentence:** Scan cost contracts through File and Split → Row Group → Page → Row → Column. +> The earlier a non-match is established, the more remote I/O, decompression, decoding, and +> materialization can be avoided. + +- **Uniform entry point:** TableReader maps table semantics to file semantics. ParquetReader handles + only localized columns and predicates. +- **Planning first:** After opening a file, read footer/schema and build `RowGroupReadPlan` objects + instead of making ad hoc decisions during reads. +- **Multi-level predicates:** The same table predicate may be reused at several granularities, but + each layer eliminates data only when it can do so safely. Uncertain cases remain candidates. +- **Predicate columns first:** Read filter columns first and maintain a SelectionVector. Read output + columns only for surviving rows. +- **Layered caches:** File-block cache, Parquet page cache, condition-result cache, and merged small + I/O solve different problems and are not interchangeable. + +**Scope:** This document focuses on the FileScannerV2 Parquet Reader design and core pipeline. It +does not cover Arrow decoder internals, complex-type reconstruction, or expression implementation. + +## 2. Overall Architecture + +Responsibilities are divided across scan orchestration, table-semantic adaptation, format planning, +Row Group execution, column decoding, and I/O. Upper layers own correctness semantics; lower layers +own format-aware pruning and reads. + +```mermaid +flowchart TB + A[FileScanOperator / ScannerScheduler
Schedule Scanners and Runtime Filters] --> B[FileScannerV2
Split Fetching, Batch Control, Profile Aggregation] + B --> C[TableReader
Schema Mapping, Partition/Default Values, Predicate Localization] + C --> D[ParquetReader
Footer/Schema and Row Group Scan Planning] + D --> E[ParquetScanScheduler
Row Group Lifecycle and Batch Reads] + E --> F[ParquetColumnReader
Page Skipping, Decompression, Decoding, Materialization] + F --> G[ParquetFileContext / Arrow RandomAccessFile
Page Cache, MergeRange, Prefetch] + G --> H[Doris FileReader / FileCache / Remote FS] +``` + +| Layer | Core responsibilities | Responsibilities intentionally excluded | +| --- | --- | --- | +| FileScannerV2 | Split lifecycle, reader reuse, dynamic batches, and unified Profile | Does not understand Parquet pages or encodings | +| TableReader | Map table columns, partition columns, missing columns, defaults, and conjuncts into file-local coordinates | Does not parse the Parquet footer directly | +| ParquetReader | Build file context, plan Row Groups, and aggregate format-level statistics | Does not implement table-level schema-evolution semantics | +| ParquetScanScheduler | Open planned Row Groups and order predicate/output column reads | Does not repeat global predicate analysis | +| ColumnReader | Locate and skip pages, decompress, decode, and materialize by Selection | Does not decide whether a Row Group is a candidate | +| FileContext / FileReader | Provide random reads, caches, merged reads, and remote access | Does not interpret SQL predicates | + +> **Design benefit:** Table format, file format, and storage medium remain decoupled. The Parquet +> layer can use footer, page index, dictionary, and other format knowledge while upper layers retain +> uniform scan semantics. + +## 3. From File Open to Scan Plan + +After a reader receives a Split, it opens the file and builds the scan plan. This phase determines +which Row Groups, row ranges, column chunks, and Page Skip Plans will be used later. + +```mermaid +sequenceDiagram + participant FS as FileScannerV2 + participant TR as TableReader + participant PR as ParquetReader + participant FC as ParquetFileContext + participant META as Footer/Metadata + participant PLAN as RowGroup Planner + participant SCH as ScanScheduler + FS->>TR: prepare/open split + TR->>TR: Map schema and localize predicates + TR->>PR: FileScanRequest + PR->>FC: Open FileReader + FC->>META: Read footer and schema + META-->>PR: Row Group / Column Chunk metadata + PR->>PLAN: Candidate Row Groups and local predicates + PLAN->>PLAN: Select by Split range + PLAN->>PLAN: Prune by Statistics/Dictionary/Bloom + PLAN->>PLAN: Prune pages by ColumnIndex+OffsetIndex + PLAN-->>PR: RowGroupReadPlan list + PR->>FC: Register Page Cache ranges for surviving chunks + PR->>SCH: Install plans and column-read request + SCH-->>FS: ready / EOF +``` + +### Key planning objects + +- **FileScanRequest:** Contains `predicate_columns`, `non_predicate_columns`, localized conjuncts, + delete conjuncts, and local column-position mappings. +- **RowGroupReadPlan:** Records the Row Group, its file-global starting row, `selected_ranges` + produced by page-index pruning, and the `page_skip_plan` for each leaf column. +- **ParquetFileContext:** Adapts Doris FileReader to Arrow RandomAccessFile and owns Page Cache, + FileCache prefetch, and MergeRange routing. + +> Planning intentionally proceeds from cheap to expensive. Split and metadata pruning reduce the +> candidate set before finer indexes are read for surviving Row Groups, avoiding index I/O for data +> that is already known to be irrelevant. + +## 4. Predicate Pushdown Design + +Predicate pushdown does not begin by passing table expressions directly to Parquet. TableReader and +ColumnMapper first translate a table expression into an expression understood by the current file. + +```mermaid +flowchart LR + A[Table Conjunct / Runtime Filter] --> B[Resolve Column References] + B --> C{Column Present in Current File?} + C -- "File Column" --> D[Map to LocalColumnId / Block Position] + C -- "Partition Column" --> E[Evaluate with Constant Value] + C -- "Missing Column" --> F[Apply Default or NULL Semantics] + D --> G[Separate Predicate and Output Columns] + E --> G + F --> G + G --> H[FileScanRequest] + H --> I[Reuse Localized Predicates at Parquet Index Levels] +``` + +### Design principles + +1. **Semantics before optimization:** Resolve partition constants, missing columns, defaults, and + type mappings before deciding whether pushdown is safe. +2. **Local coordinates:** Parquet sees only the current file's column IDs and block positions, so it + does not repeatedly interpret table-schema evolution. +3. **Capability checks:** ZoneMap, Dictionary, and Bloom use only expressions they can interpret + safely. All others remain row-level residual predicates. +4. **Prefer safe single-column predicates:** Single-column predicates can drive indexes and staged + filtering. Multi-column, stateful, or error-sensitive expressions retain whole-expression + evaluation. +5. **Runtime Filters can refresh:** ScannerScheduler refreshes late Runtime Filters before reading. + TableReader handles partition-range pruning during Split preparation, and passes file-pushable + parts as localized conjuncts. + +> Pushdown is not merely avoiding another expression evaluation. It projects deterministic facts +> from the expression onto cheaper data summaries. Any case that cannot prove a non-match must +> continue scanning. + +## 5. Predicate Evaluation at Different Granularities + +The same predicate may be attempted at several granularities. Each layer produces a smaller +candidate set that becomes the next layer's input. + +```mermaid +flowchart TB + A[Query / Runtime Filter] --> B[Split / Partition
Skip Entire File or Fragment] + B --> C[Row Group
Statistics / Dictionary / Bloom] + C --> D[Page
ColumnIndex + OffsetIndex] + D --> E[Batch / Row
Dictionary ID + VExpr + Delete Predicate] + E --> F[Column
Materialize Output Only for Surviving Rows] + style B fill:#e8f3ff + style C fill:#eaf7ea + style D fill:#fff5d6 + style E fill:#f4eaff + style F fill:#fce8e6 +``` + +| Granularity | Input information | Main cost avoided | Conservative fallback | +| --- | --- | --- | --- | +| Split / Partition | Partition values, Runtime Filter range, scan byte range | Opening and reading an entire file or fragment | Retain the Split when uncertain | +| Row Group | Footer statistics, dictionary, Bloom filter | I/O and decoding for all column chunks in the group | Retain the Row Group when an index is missing or incompatible | +| Page | ColumnIndex min/max/null data and OffsetIndex | Page I/O, decompression, and decoding | Read the affected range when page indexes are incomplete | +| Row / Batch | Actual column values, dictionary IDs, residual conjuncts | Later predicate-column and output-column materialization | Evaluate full VExpr semantics | +| Column | SelectionVector | Reads, decoding, and memory writes for non-predicate columns | Read all projected columns sequentially when no filtering applies | + +> **Key distinction:** Row Group and Page indexes generally eliminate impossible candidates; they +> do not produce final query results. Row-level predicates determine whether individual rows match. + +## 6. Row Group Planning and Index Coordination + +The Row Group Planner combines physical layout from the footer, the Split byte range, and predicate +index capabilities into an executable plan. The key property is a stable candidate-reduction order. + +```mermaid +sequenceDiagram + participant P as Planner + participant M as RowGroup Metadata + participant S as Statistics + participant D as Dictionary Page + participant B as Bloom Filter + participant I as Page Index + P->>M: Enumerate footer Row Groups + P->>M: Assign Split by Row Group midpoint + loop Each candidate Row Group + P->>S: Evaluate ZoneMap(min/max/null) + alt Proven non-match + S-->>P: Prune Row Group + else Match remains possible + P->>D: Read and evaluate available dictionary + alt Dictionary domain cannot match + D-->>P: Prune Row Group + else Match remains possible + P->>B: Probe Bloom Filter + B-->>P: Prune or retain + end + end + end + P->>I: Read ColumnIndex/OffsetIndex for survivors + I-->>P: selected_ranges + page_skip_plans +``` + +### Why this order is used + +- **Statistics:** Usually already in the footer, making them the lowest-cost option for range and + null semantics. +- **Dictionary:** Requires reading the dictionary page, but can prove a complete non-match for + low-cardinality string columns. +- **Bloom:** Requires Bloom data I/O and is useful for negative membership tests. A positive result + may be a false positive. +- **Page Index:** Builds page-level row ranges only for surviving Row Groups, avoiding index cost for + groups already eliminated. + +### How the plan drives physical skips + +ColumnIndex provides min/max/null semantics for each page. OffsetIndex maps pages to Row Group row +numbers and file offsets. Candidate ranges from multiple predicate columns are intersected into +`selected_ranges`; a `page_skip_plan` is then built for each leaf so its column reader can skip pages +that do not overlap surviving rows. + +> `selected_ranges` represents logical row ranges, while `page_skip_plan` represents physical page +> reads. Keeping them separate allows the scheduler to advance by row batch while each column skips +> according to its own page boundaries. + +## 7. Batch Reads, Dictionary Filtering, and Lazy Materialization + +Execution follows a filter-first, materialize-later strategy. The scheduler advances through +`selected_ranges`, asks column readers to skip gaps, and then reads the current batch. + +```mermaid +sequenceDiagram + participant S as ParquetScanScheduler + participant PC as Predicate Column Readers + participant SEL as SelectionVector + participant EX as Residual Expressions + participant OC as Output Column Readers + S->>S: Open next Row Group + S->>S: Skip row ranges rejected by Page Index + S->>PC: Read first predicate column set + PC->>SEL: Filter with dictionary IDs or actual values + loop Remaining safe single-column predicates + S->>PC: Read/materialize only surviving rows + PC->>SEL: Further reduce selection + end + S->>EX: Evaluate residual multi-column predicates and deletes + EX->>SEL: Produce final survivors + alt Rows survive + S->>OC: Prefetch and read non-predicate columns + OC->>OC: Materialize by Selection + S-->>S: Assemble output Block + else No rows survive + S->>S: Do not read deferred output columns + end +``` + +### Row-level dictionary filtering + +```mermaid +flowchart LR + A[Single-column Predicate] --> B{Column Fully Dictionary Encoded?} + B -- "No" --> F[Read Actual Values and Execute VExpr] + B -- "Yes" --> C[Read Dictionary Page] + C --> D[Evaluate Predicate on Dictionary Values
Build Dictionary-ID Bitmap] + D --> E[Decode Data-page Dictionary IDs
Update SelectionVector Directly] + E --> G[Materialize Only Survivors] +``` + +- Applies to non-repeated primitive, string-like BYTE_ARRAY / FIXED_LEN_BYTE_ARRAY columns whose + complete Column Chunk uses dictionary data encoding. +- Safe AND subexpressions may remove components exactly covered by dictionary evaluation. OR or + non-equivalent expressions are not rewritten aggressively. +- Stateful, potentially throwing, or whole-batch-sensitive expressions disable staged + single-column scheduling and fall back to reading required columns before whole-expression + evaluation. + +> **Optimization loop:** The earlier SelectionVector shrinks, the fewer values later predicate and +> output columns must decode and copy. This is the main benefit of lazy materialization in a +> columnar format. + +## 8. Supported Indexes and Their Boundaries + +V2 uses native Parquet metadata and encoding information. It does not construct Doris-internal +storage indexes for external Parquet files. + +| Capability | Granularity | Suitable predicates | Result property | Main limitations | +| --- | --- | --- | --- | --- | +| Footer Statistics / ZoneMap | Row Group | Ranges, comparisons, IS NULL/IS NOT NULL, and expressions safely convertible to ZoneMap | Can prove the entire group cannot match | Requires valid min/max/null_count and safe type conversion | +| Dictionary Pruning | Row Group | Single-column predicates exactly evaluable over the dictionary domain | Can prove the entire group cannot match | Low-cardinality string-like primitive with complete dictionary encoding | +| Parquet Bloom Filter | Row Group / Column Chunk | Equality and IN membership-negation predicates | Negative result can prune; positive result requires verification | Controlled by configuration; file must contain Bloom data; false positives are possible | +| ColumnIndex | Page | Predicates evaluable from min/max/null | Produces candidate pages and row ranges | Requires an index and decodable compatible types | +| OffsetIndex | Page → Row Range | Does not evaluate predicates directly | Maps page results to row numbers and physical skip plans | Normally used with ColumnIndex | +| Dictionary-ID Filter | Row / Batch | Safe single-column string-like predicates | Exact filtering of actual rows | Complete dictionary encoding and non-repeated primitive only | +| Condition Cache Bitmap | File-global granule | Stable cacheable conditions | Reuses previous filtering to reduce row ranges | Not a native Parquet index; uncovered ranges remain candidates | + +### Index-selection overview + +```mermaid +flowchart TD + A[Localized Predicate] --> B{ZoneMap Evaluatable?} + B -- "Yes" --> C[Row Group Statistics / Page ColumnIndex] + B -- "No" --> D{Single-column Dictionary Evaluatable?} + D -- "Yes" --> E[Row Group Dictionary + Row Dictionary-ID] + D -- "No" --> F{Bloom Negative-membership Test?} + F -- "Yes" --> G[Parquet Bloom Filter] + F -- "No" --> H[Retain as Row-level Residual VExpr] + C --> H + E --> H + G --> H +``` + +> Indexes are layered rather than mutually exclusive. An index may remove only ranges already +> proven impossible; residual predicates still guarantee final correctness. + +## 9. Cache and I/O Optimization + +Parquet V2 has four complementary cache and I/O paths: cache remote file blocks, cache serialized +Parquet ranges, cache predicate results, and merge small random reads. + +```mermaid +flowchart TB + A[Parquet Column Reader ReadAt] --> B{Parquet Page Cache Hit?} + B -- "Yes" --> C[Return Cached Serialized Range Bytes] + B -- "No" --> D{MergeRange Active?} + D -- "Yes" --> E[MergeRangeFileReader
Merge Adjacent Small I/O] + D -- "No" --> F[Base FileReader] + E --> G[CachedRemoteFileReader / FileCache] + F --> G + G --> H[Local Block / Peer / Remote Object Storage] + H --> I[Populate FileCache] + I --> J[Populate Page Cache for Registered Ranges] +``` + +| Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | +| --- | --- | --- | --- | +| FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | +| Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | +| Condition Cache | Condition-surviving granule bitmap | Managed by condition and file-range context | Reuse filtering results before reading columns | +| MergeRangeFileReader | Not a cache; merges small ranges into larger slices | Installed temporarily for projected chunks of the current Row Group | Reduce remote small-I/O count and request overhead | + +### Why Page Cache registers only surviving chunks + +The footer is read before Row Group planning and before Page Cache ranges are registered, so +footer/metadata bytes never enter the Parquet Page Cache. After planning, only projected Column +Chunks from surviving Row Groups are registered, limiting pollution and key count. + +### Relationship between prefetch and MergeRange + +- When the base reader is CachedRemoteFileReader, predicate/output ranges for the current Row Group + may be prefetched into FileCache. +- When average projected chunks are small and the reader is not in-memory, install + MergeRangeFileReader so subsequent Arrow `ReadAt` calls actually use merged reads. +- With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after + at least one row survives, avoiding unnecessary bandwidth. + +## 10. Other Key Optimizations + +### 10.1 Condition Cache: Move Historical Filter Results Earlier + +```mermaid +sequenceDiagram + participant T as TableReader / Cache Context + participant S as ParquetScanScheduler + participant P as RowGroup Plans + participant R as Row Filter + alt Cache Hit + T->>S: Bitmap + base granule + S->>P: Intersect with selected_ranges + P-->>S: Smaller pending row ranges + else Cache Miss + T->>S: Empty bitmap context + S->>R: Execute normal row predicates + R-->>S: SelectionVector + S->>S: Mark granules containing survivors + S-->>T: Publish to Condition Cache later + end +``` + +On a hit, only granules explicitly proven unnecessary by the bitmap are removed. Rows outside +bitmap coverage remain candidates. On a miss, granules containing surviving rows are marked, +trading granularity for reuse and smaller cache entries. + +### 10.2 Adaptive Batches + +FileScannerV2 uses a small probe batch to measure bytes per row in the final table Block. It derives +later batch rows from a target Block size, bounded by the system batch-size limit. Wide rows use +smaller batches to reduce memory peaks; narrow rows use larger batches for throughput. + +```mermaid +flowchart LR + A[Small Probe Batch] --> B[Read and Complete Table-level Materialization] + B --> C[Estimate Bytes per Row] + C --> D[Target Block Bytes / Bytes per Row] + D --> E[Choose Next Row Cap] + E --> F[Bound by batch_size and Selected Range] +``` + +### 10.3 Aggregate Pushdown + +When TableReader proves that no filter or delete semantics can change the result, COUNT / MIN / MAX +may use Parquet metadata to compute all or part of an aggregate without scanning data pages. This is +a metadata aggregation optimization and is distinct from Row Group index pruning. + +### 10.4 Staged Prefetch + +Without row-level filtering, output columns may be warmed together. With filtering, warm predicate +columns first and defer non-predicate columns until at least one row survives, aligning network +bandwidth with lazy materialization. + +## 11. Correctness, Fallback, and Capability Boundaries + +V2 follows a prove-before-skip rule. Missing indexes, unsupported types, expressions that cannot be +split safely, or read anomalies must never change query semantics. + +> **Correctness baseline:** Index results only reduce candidate sets. Every expression not exactly +> covered remains a residual conjunct evaluated against actual data. + +| Scenario | V2 behavior | +| --- | --- | +| Missing Statistics or unsafe min/max conversion | Treat the column's ZoneMap as unavailable and retain the Row Group/Page | +| Bloom missing, disabled, or unreadable | Skip Bloom pruning and continue with later scan stages | +| Incomplete dictionary page, mixed non-dictionary encoding, complex/repeated column | Disable dictionary pruning and Dictionary-ID Filter; use actual values | +| Missing or inconsistent ColumnIndex/OffsetIndex | Disable fine-grained page pruning and read the full candidate range | +| Multi-column, OR, stateful, or error-order-sensitive expression | Preserve whole-expression evaluation to avoid changing SQL short-circuit or error semantics | +| No stable file-version identity for Page Cache | Disable Parquet Page Cache to prevent stale-byte reads | +| Incomplete Condition Cache coverage | Retain and recompute uncovered ranges | + +### Capability boundaries + +- Parquet Reader uses indexes and encoding metadata already present in the file; it does not build + new indexes for external files. +- Page boundaries and definition/repetition levels are more complex for nested/repeated columns, so + some dictionary and page-level optimizations conservatively fall back. +- Bloom is probabilistic and is safe only for proving absence. A positive Bloom result is not a row + match. +- Page Index benefit depends on whether the writer produced indexes, data ordering, and predicate + selectivity. + +## 12. Profile Observation and Troubleshooting + +Troubleshoot in this order: verify planning effectiveness, row filtering, lazy materialization, and +then I/O/cache health. Total ScanTime alone does not identify the cause. + +```mermaid +flowchart TD + A[Slow Scan] --> B{Many Row Groups Pruned?} + B -- "No" --> C[Check Statistics/Dictionary/Bloom Availability and Predicate Shape] + B -- "Yes" --> D{Page selected_ranges Shrink Significantly?} + D -- "No" --> E[Check ColumnIndex/OffsetIndex and Data Ordering] + D -- "Yes" --> F{Many Rows Filtered by Predicates?} + F -- "Yes" --> G[Check Deferred Prefetch and Selection-based Output Materialization] + F -- "No" --> H[Low Selectivity: Inspect Decode and I/O Throughput] + G --> I{Cache Hits and Small I/O Reasonable?} + H --> I + I -- "No" --> J[Check FileCache, Page Cache, MergeRange, and Remote Reads] + I -- "Yes" --> K[Check Type Conversion, Complex Columns, and Downstream Operators] +``` + +### Important metric families + +| Metric family | Question answered | +| --- | --- | +| Row Group pruning | How many total Row Groups were pruned by Statistics/Dictionary/Bloom, and how much time did each stage take? | +| Page index pruning | How many indexes were checked, pages/rows were pruned, ranges selected, and pages skipped? | +| Dictionary row filter | How often were predicates rewritten, dictionaries read, bitmaps built, and attempts successful or rejected? | +| Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | +| Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | +| FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | +| Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | +| Condition Cache | How many rows were skipped early after a cache hit? | + +> Interpret pruning ratios in the context of write layout. Unsorted data produces wide min/max +> ranges, so Row Group/Page pruning may be ineffective even when the reader and indexes work +> correctly. + +## 13. Summary + +The FileScannerV2 Parquet scan pipeline has three primary threads: + +1. **Semantic thread:** TableReader maps table schema and predicates into stable file-local + semantics, preserving schema evolution, partition columns, and missing columns. +2. **Pruning thread:** Split → Row Group → Page → Row progressively applies Runtime Filters, + Statistics, Dictionary, Bloom, Page Index, and actual-value filters. +3. **I/O thread:** Predicate-first reads, SelectionVector, lazy materialization, adaptive batches, + FileCache/Page Cache/Condition Cache, and MergeRange reduce read amplification together. + +```mermaid +flowchart LR + A[Table-level Semantics] --> B[File-local Predicates] --> C[RowGroupReadPlan] --> D[selected_ranges] --> E[SelectionVector] --> F[Final Block] + G[Statistics / Dictionary / Bloom] --> C + H[ColumnIndex / OffsetIndex] --> D + I[FileCache / Page Cache / MergeRange] --> C + I --> D + I --> E +``` + +> **Final design criterion:** V2 turns format knowledge into an explicit scan plan and requires the +> executor to perform only the minimum necessary reads. Indexes safely reduce candidates, caches +> reuse cost, and lazy materialization avoids reading irrelevant columns for rejected rows. + +This document reflects the current code pipeline and is intended as a common reference for +architecture reviews, performance analysis, and Profile troubleshooting. diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 8f64a51dc9b15b..0749720840ad5b 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -24,27 +24,52 @@ import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache; import com.google.common.base.Preconditions; +import org.apache.paimon.CoreOptions; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.disk.IOManagerImpl; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableRead; import org.apache.paimon.types.DataType; import org.apache.paimon.types.TimestampType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; public class PaimonJniScanner extends JniScanner { private static final Logger LOG = LoggerFactory.getLogger(PaimonJniScanner.class); private static final String HADOOP_OPTION_PREFIX = "hadoop."; + private static final String PAIMON_OPTION_PREFIX = "paimon."; + private static final String ASYNC_READER_THREAD_NAME_PREFIX = "paimon-reader-async-thread"; + private static final String FILE_READER_ASYNC_THRESHOLD = "file-reader-async-threshold"; + static final String ENABLE_JNI_IO_MANAGER = "paimon.doris.enable_jni_io_manager"; + static final String JNI_IO_MANAGER_TMP_DIR = "paimon.doris.jni_io_manager.tmp_dir"; + static final String JNI_IO_MANAGER_IMPL_CLASS = "paimon.doris.jni_io_manager.impl_class"; + private static final AtomicInteger ACTIVE_SCANNERS = new AtomicInteger(); private final Map params; private final Map hadoopOptionParams; @@ -52,12 +77,20 @@ public class PaimonJniScanner extends JniScanner { private final String paimonPredicate; private Table table; private RecordReader reader; + private IOManager ioManager; + private String ioManagerTempDirs; private final PaimonColumnValue columnValue = new PaimonColumnValue(); private List paimonAllFieldNames; private List paimonDataTypeList; private RecordReader.RecordIterator recordIterator = null; private final ClassLoader classLoader; private PreExecutionAuthenticator preExecutionAuthenticator; + private boolean scannerCounted; + private long openTimeNanos; + private long readBatchTimeNanos; + private long readBatchCalls; + private long emptyReadBatchCalls; + private long rowsRead; public PaimonJniScanner(int batchSize, Map params) { this.classLoader = this.getClass().getClassLoader(); @@ -65,8 +98,11 @@ public PaimonJniScanner(int batchSize, Map params) { LOG.debug("params:{}", params); } this.params = params; - String[] requiredFields = params.get("required_fields").split(","); - String[] requiredTypes = params.get("columns_types").split("#"); + String[] requiredFields = splitParam(params.get("required_fields"), ","); + String[] requiredTypes = splitParam(params.get("columns_types"), "#"); + Preconditions.checkArgument(requiredFields.length == requiredTypes.length, + "required_fields size %s does not match columns_types size %s", + requiredFields.length, requiredTypes.length); ColumnType[] columnTypes = new ColumnType[requiredTypes.length]; for (int i = 0; i < requiredTypes.length; i++) { columnTypes[i] = ColumnType.parseType(requiredFields[i], requiredTypes[i]); @@ -85,6 +121,8 @@ public PaimonJniScanner(int batchSize, Map params) { @Override public void open() throws IOException { + markScannerOpenedForMetrics(); + long startTime = System.nanoTime(); try { // When the user does not specify hive-site.xml, Paimon will look for the file from the classpath: // org.apache.paimon.hive.HiveCatalog.createHiveConf: @@ -100,8 +138,15 @@ public void open() throws IOException { resetDatetimeV2Precision(); } catch (Throwable e) { + try { + close(); + } catch (IOException closeException) { + e.addSuppressed(closeException); + } LOG.warn("Failed to open paimon_scanner: " + e.getMessage(), e); throw new RuntimeException(e); + } finally { + openTimeNanos += System.nanoTime() - startTime; } } @@ -117,13 +162,111 @@ private void initReader() throws IOException { int[] projected = getProjected(); readBuilder.withProjection(projected); readBuilder.withFilter(getPredicates()); - reader = readBuilder.newRead().executeFilter().createReader(getSplit()); + reader = newReadWithOptionalIOManager(readBuilder).executeFilter().createReader(getSplit()); paimonDataTypeList = Arrays.stream(projected).mapToObj(i -> table.rowType().getTypeAt(i)).collect(Collectors.toList()); } + private TableRead newReadWithOptionalIOManager(ReadBuilder readBuilder) throws IOException { + TableRead tableRead = readBuilder.newRead(); + if (!isIOManagerEnabled(params)) { + return tableRead; + } + ioManagerTempDirs = getIOManagerTempDirs(params); + ioManager = createIOManager(ioManagerTempDirs, getIOManagerImplClass(params)); + LOG.info("Enable Paimon JNI IOManager with temp dirs: {}, implementation: {}", + ioManagerTempDirs, ioManager.getClass().getName()); + return tableRead.withIOManager(ioManager); + } + + static boolean isIOManagerEnabled(Map params) { + return Boolean.parseBoolean(params.getOrDefault(ENABLE_JNI_IO_MANAGER, "false")); + } + + static String getIOManagerTempDirs(Map params) throws IOException { + String tempDirs = params.get(JNI_IO_MANAGER_TMP_DIR); + if (tempDirs == null || tempDirs.trim().isEmpty()) { + throw new IOException("Paimon JNI IOManager is enabled but " + JNI_IO_MANAGER_TMP_DIR + " is not set"); + } + return tempDirs.trim(); + } + + static String getIOManagerImplClass(Map params) { + String implClass = params.get(JNI_IO_MANAGER_IMPL_CLASS); + return implClass == null || implClass.trim().isEmpty() ? null : implClass.trim(); + } + + static IOManager createIOManager(String tempDirs) throws IOException { + return createIOManager(tempDirs, null); + } + + static IOManager createIOManager(String tempDirs, String implClassName) throws IOException { + String[] splitDirs = IOManagerImpl.splitPaths(tempDirs); + if (splitDirs.length == 0) { + throw new IOException("Paimon JNI IOManager temp dirs are empty"); + } + for (String splitDir : splitDirs) { + Files.createDirectories(Paths.get(splitDir)); + } + if (implClassName == null) { + return IOManager.create(splitDirs); + } + return createCustomIOManager(implClassName, splitDirs, tempDirs); + } + + private static IOManager createCustomIOManager(String implClassName, String[] splitDirs, String tempDirs) + throws IOException { + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + if (loader == null) { + loader = PaimonJniScanner.class.getClassLoader(); + } + try { + Class implClass = Class.forName(implClassName, true, loader); + if (!IOManager.class.isAssignableFrom(implClass)) { + throw new IOException("Paimon JNI IOManager implementation " + implClassName + + " does not implement " + IOManager.class.getName()); + } + return (IOManager) instantiateCustomIOManager(implClass, splitDirs, tempDirs); + } catch (ClassNotFoundException e) { + throw new IOException("Failed to find Paimon JNI IOManager implementation: " + implClassName, e); + } catch (ReflectiveOperationException e) { + throw new IOException("Failed to create Paimon JNI IOManager implementation: " + implClassName, e); + } + } + + private static Object instantiateCustomIOManager(Class implClass, String[] splitDirs, String tempDirs) + throws ReflectiveOperationException { + try { + Constructor constructor = implClass.getConstructor(String[].class); + return constructor.newInstance((Object) splitDirs); + } catch (NoSuchMethodException e) { + try { + Constructor constructor = implClass.getConstructor(String.class); + return constructor.newInstance(tempDirs); + } catch (NoSuchMethodException stringConstructorMissing) { + Constructor constructor = implClass.getConstructor(); + return constructor.newInstance(); + } + } catch (InvocationTargetException e) { + throw e; + } + } + private int[] getProjected() { - return Arrays.stream(fields).mapToInt(paimonAllFieldNames::indexOf).toArray(); + return Arrays.stream(fields).mapToInt(fieldName -> { + int index = getFieldIndex(paimonAllFieldNames, fieldName); + Preconditions.checkArgument(index >= 0, "RequiredField %s not found in schema", fieldName); + return index; + }).toArray(); + } + + static int getFieldIndex(List fieldNames, String fieldName) { + for (int i = 0; i < fieldNames.size(); i++) { + if (fieldNames.get(i).equalsIgnoreCase(fieldName)) { + return i; + } + } + return -1; } private List getPredicates() { @@ -147,7 +290,7 @@ private void resetDatetimeV2Precision() { if (types[i].isDateTimeV2()) { // paimon support precision > 6, but it has been reset as 6 in FE // try to get the right precision for datetimev2 - int index = paimonAllFieldNames.indexOf(fields[i]); + int index = getFieldIndex(paimonAllFieldNames, fields[i]); if (index != -1) { DataType dataType = table.rowType().getTypeAt(index); if (dataType instanceof TimestampType) { @@ -160,8 +303,50 @@ private void resetDatetimeV2Precision() { @Override public void close() throws IOException { - if (reader != null) { - reader.close(); + IOException exception = null; + try { + try { + releaseRecordIterator(); + } catch (RuntimeException e) { + exception = new IOException("Failed to release Paimon record iterator", e); + } + if (reader != null) { + try { + reader.close(); + reader = null; + } catch (IOException e) { + if (exception == null) { + exception = e; + } else { + exception.addSuppressed(e); + } + } + } + if (ioManager != null) { + try { + ioManager.close(); + ioManager = null; + } catch (Exception e) { + LOG.warn("Failed to close Paimon JNI IOManager, temp dirs: {}", ioManagerTempDirs, e); + if (exception == null) { + exception = new IOException(e); + } else { + exception.addSuppressed(e); + } + } + } + } finally { + markScannerClosedForMetrics(); + } + if (exception != null) { + throw exception; + } + } + + private void releaseRecordIterator() { + if (recordIterator != null) { + recordIterator.releaseBatch(); + recordIterator = null; } } @@ -169,7 +354,7 @@ private int readAndProcessNextBatch() throws IOException { int rows = 0; try { if (recordIterator == null) { - recordIterator = reader.readBatch(); + recordIterator = readBatchWithMetrics(); } while (recordIterator != null) { @@ -184,15 +369,23 @@ private int readAndProcessNextBatch() throws IOException { appendData(i, columnValue); } if (rows >= batchSize) { + if (fields.length == 0) { + vectorTable.appendVirtualData(rows); + } appendDataTime += System.nanoTime() - startTime; + rowsRead += rows; return rows; } } appendDataTime += System.nanoTime() - startTime; - recordIterator.releaseBatch(); - recordIterator = reader.readBatch(); + releaseRecordIterator(); + recordIterator = readBatchWithMetrics(); } + if (fields.length == 0 && rows > 0) { + vectorTable.appendVirtualData(rows); + } + rowsRead += rows; } catch (Exception e) { close(); LOG.warn("Failed to get the next batch of paimon. " @@ -203,6 +396,20 @@ private int readAndProcessNextBatch() throws IOException { return rows; } + private RecordReader.RecordIterator readBatchWithMetrics() throws IOException { + long startTime = System.nanoTime(); + try { + RecordReader.RecordIterator iterator = reader.readBatch(); + if (iterator == null) { + emptyReadBatchCalls++; + } + return iterator; + } finally { + readBatchCalls++; + readBatchTimeNanos += System.nanoTime() - startTime; + } + } + @Override protected int getNext() { try { @@ -218,13 +425,164 @@ protected TableSchema parseTableSchema() throws UnsupportedOperationException { return null; } + @Override + public Map getStatistics() { + Map statistics = new HashMap<>(); + statistics.put("gauge:PaimonJniIOManagerEnabled", ioManager != null ? "1" : "0"); + statistics.put("gauge:PaimonJniActiveScannerCount", String.valueOf(ACTIVE_SCANNERS.get())); + statistics.put("gauge:PaimonJniAsyncReaderThreadCount", + String.valueOf(currentAsyncReaderThreadCount())); + statistics.put("gauge:PaimonJniRequiredFieldCount", String.valueOf(fields.length)); + statistics.put("counter:PaimonJniSplitEncodedLength", String.valueOf(lengthOfParam("paimon_split"))); + statistics.put("counter:PaimonJniPredicateEncodedLength", String.valueOf(lengthOfParam("paimon_predicate"))); + statistics.put("gauge:PaimonJniAsyncThresholdConfigured", + hasPaimonOption(FILE_READER_ASYNC_THRESHOLD) ? "1" : "0"); + parseDataSizeBytes(paimonOption(FILE_READER_ASYNC_THRESHOLD)).ifPresent( + bytes -> statistics.put("bytes_gauge:PaimonJniAsyncThresholdBytes", String.valueOf(bytes))); + statistics.put("counter:PaimonJniReadBatchCalls", String.valueOf(readBatchCalls)); + statistics.put("counter:PaimonJniEmptyReadBatchCalls", String.valueOf(emptyReadBatchCalls)); + statistics.put("counter:PaimonJniRowsRead", String.valueOf(rowsRead)); + statistics.put("timer:PaimonJniScannerOpenTime", String.valueOf(openTimeNanos)); + statistics.put("timer:PaimonJniReadBatchTime", String.valueOf(readBatchTimeNanos)); + putMemoryStatistics(statistics); + return statistics; + } + + private int lengthOfParam(String key) { + String value = params.get(key); + return value == null ? 0 : value.length(); + } + + private boolean hasPaimonOption(String key) { + return paimonOption(key) != null; + } + + private String paimonOption(String key) { + if (table != null) { + String tableOption = table.options().get(key); + if (tableOption != null) { + return tableOption; + } + } + return params.get(PAIMON_OPTION_PREFIX + key); + } + + private static void putMemoryStatistics(Map statistics) { + MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); + MemoryUsage heapUsage = memoryMXBean.getHeapMemoryUsage(); + MemoryUsage nonHeapUsage = memoryMXBean.getNonHeapMemoryUsage(); + statistics.put("bytes_gauge:PaimonJniJvmHeapUsed", String.valueOf(nonNegative(heapUsage.getUsed()))); + statistics.put("bytes_gauge:PaimonJniJvmHeapCommitted", String.valueOf(nonNegative(heapUsage.getCommitted()))); + statistics.put("bytes_gauge:PaimonJniJvmHeapMax", String.valueOf(nonNegative(heapUsage.getMax()))); + statistics.put("bytes_gauge:PaimonJniJvmNonHeapUsed", String.valueOf(nonNegative(nonHeapUsage.getUsed()))); + statistics.put("bytes_gauge:PaimonJniJvmNonHeapCommitted", + String.valueOf(nonNegative(nonHeapUsage.getCommitted()))); + statistics.put("bytes_gauge:PaimonJniJvmNonHeapMax", String.valueOf(nonNegative(nonHeapUsage.getMax()))); + } + + private static long nonNegative(long value) { + return Math.max(value, 0L); + } + + private static int currentAsyncReaderThreadCount() { + return countThreadsByNamePrefix(ASYNC_READER_THREAD_NAME_PREFIX); + } + + static int countThreadsByNamePrefix(String threadNamePrefix) { + int count = 0; + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(threadMXBean.getAllThreadIds(), 0); + for (ThreadInfo threadInfo : threadInfos) { + if (threadInfo != null && threadInfo.getThreadName().startsWith(threadNamePrefix)) { + count++; + } + } + return count; + } + + private void markScannerOpenedForMetrics() { + if (!scannerCounted) { + scannerCounted = true; + ACTIVE_SCANNERS.incrementAndGet(); + } + } + + private void markScannerClosedForMetrics() { + if (scannerCounted) { + scannerCounted = false; + ACTIVE_SCANNERS.decrementAndGet(); + } + } + + static Optional parseDataSizeBytes(String value) { + if (value == null || value.trim().isEmpty()) { + return Optional.empty(); + } + String normalized = value.trim().toLowerCase(Locale.ROOT).replace("_", "").replace(" ", ""); + int unitStart = 0; + while (unitStart < normalized.length() + && (Character.isDigit(normalized.charAt(unitStart)) || normalized.charAt(unitStart) == '.')) { + unitStart++; + } + if (unitStart == 0) { + return Optional.empty(); + } + try { + double number = Double.parseDouble(normalized.substring(0, unitStart)); + String unit = normalized.substring(unitStart); + long multiplier; + switch (unit) { + case "": + case "b": + case "byte": + case "bytes": + multiplier = 1L; + break; + case "k": + case "kb": + case "kib": + multiplier = 1024L; + break; + case "m": + case "mb": + case "mib": + multiplier = 1024L * 1024L; + break; + case "g": + case "gb": + case "gib": + multiplier = 1024L * 1024L * 1024L; + break; + case "t": + case "tb": + case "tib": + multiplier = 1024L * 1024L * 1024L * 1024L; + break; + default: + return Optional.empty(); + } + return Optional.of((long) (number * multiplier)); + } catch (NumberFormatException e) { + return Optional.empty(); + } + } + private void initTable() { Preconditions.checkState(params.containsKey("serialized_table")); table = PaimonUtils.deserialize(params.get("serialized_table")); + table = table.copy(Collections.singletonMap( + CoreOptions.READ_BATCH_SIZE.key(), String.valueOf(batchSize))); paimonAllFieldNames = PaimonUtils.getFieldNames(this.table.rowType()); if (LOG.isDebugEnabled()) { LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames); } } + private static String[] splitParam(String value, String delimiter) { + if (value == null || value.isEmpty()) { + return new String[0]; + } + return value.split(delimiter); + } + } diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java new file mode 100644 index 00000000000000..d43f251f58deb1 --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -0,0 +1,349 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.disk.BufferFileReader; +import org.apache.paimon.disk.BufferFileWriter; +import org.apache.paimon.disk.FileIOChannel; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.disk.IOManagerImpl; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.Table; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class PaimonJniScannerTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testConstructorAcceptsEmptyProjection() { + new PaimonJniScanner(128, createBaseParams()); + } + + @Test + public void testIOManagerOptionHelpers() throws Exception { + Map params = createBaseParams(); + Assert.assertFalse(PaimonJniScanner.isIOManagerEnabled(params)); + + params.put(PaimonJniScanner.ENABLE_JNI_IO_MANAGER, "true"); + File tempDir = new File(temporaryFolder.getRoot(), "paimon-io-manager"); + params.put(PaimonJniScanner.JNI_IO_MANAGER_TMP_DIR, tempDir.getAbsolutePath()); + + Assert.assertTrue(PaimonJniScanner.isIOManagerEnabled(params)); + Assert.assertEquals(tempDir.getAbsolutePath(), PaimonJniScanner.getIOManagerTempDirs(params)); + Assert.assertNull(PaimonJniScanner.getIOManagerImplClass(params)); + PaimonJniScanner.createIOManager(tempDir.getAbsolutePath()).close(); + Assert.assertTrue(tempDir.exists()); + } + + @Test + public void testCreateDefaultAndCustomIOManager() throws Exception { + File tempDir = new File(temporaryFolder.getRoot(), "paimon-io-manager-impl"); + IOManager defaultIOManager = PaimonJniScanner.createIOManager(tempDir.getAbsolutePath()); + Assert.assertTrue(defaultIOManager instanceof IOManagerImpl); + defaultIOManager.close(); + + Map params = createBaseParams(); + params.put(PaimonJniScanner.JNI_IO_MANAGER_IMPL_CLASS, TestIOManager.class.getName()); + Assert.assertEquals(TestIOManager.class.getName(), PaimonJniScanner.getIOManagerImplClass(params)); + IOManager customIOManager = PaimonJniScanner.createIOManager( + tempDir.getAbsolutePath(), PaimonJniScanner.getIOManagerImplClass(params)); + Assert.assertTrue(customIOManager instanceof TestIOManager); + Assert.assertArrayEquals(new String[] {tempDir.getAbsolutePath()}, customIOManager.tempDirs()); + } + + @Test + public void testCloseCleansIOManagerTempDirectory() throws Exception { + File tempDir = temporaryFolder.newFolder("paimon-io-manager-clean"); + IOManager ioManager = PaimonJniScanner.createIOManager(tempDir.getAbsolutePath()); + FileIOChannel.ID channel = ioManager.createChannel(); + File spillFile = channel.getPathFile(); + Assert.assertTrue(spillFile.createNewFile()); + File spillDir = spillFile.getParentFile(); + Assert.assertTrue(spillDir.exists()); + + PaimonJniScanner scanner = new PaimonJniScanner(128, createBaseParams()); + Field ioManagerField = PaimonJniScanner.class.getDeclaredField("ioManager"); + ioManagerField.setAccessible(true); + ioManagerField.set(scanner, ioManager); + Assert.assertEquals("1", scanner.getStatistics().get("gauge:PaimonJniIOManagerEnabled")); + + scanner.close(); + Assert.assertFalse(spillDir.exists()); + } + + @Test + public void testStatisticsIncludePaimonDiagnostics() throws Exception { + Map params = createBaseParams(); + params.put("paimon_split", "encoded-split"); + params.put("paimon_predicate", "encoded-predicate"); + PaimonJniScanner scanner = new PaimonJniScanner(128, params); + setTableOptions(scanner, Collections.singletonMap("file-reader-async-threshold", "10 MiB")); + + Map statistics = scanner.getStatistics(); + + Assert.assertEquals("0", statistics.get("gauge:PaimonJniIOManagerEnabled")); + Assert.assertEquals("0", statistics.get("gauge:PaimonJniRequiredFieldCount")); + Assert.assertEquals("13", statistics.get("counter:PaimonJniSplitEncodedLength")); + Assert.assertEquals("17", statistics.get("counter:PaimonJniPredicateEncodedLength")); + Assert.assertEquals("1", statistics.get("gauge:PaimonJniAsyncThresholdConfigured")); + Assert.assertEquals(String.valueOf(10L * 1024L * 1024L), + statistics.get("bytes_gauge:PaimonJniAsyncThresholdBytes")); + Assert.assertTrue(statistics.containsKey("gauge:PaimonJniAsyncReaderThreadCount")); + Assert.assertTrue(statistics.containsKey("gauge:PaimonJniActiveScannerCount")); + Assert.assertFalse(statistics.containsKey("peak:PaimonJniActiveScannerPeakCount")); + Assert.assertFalse(statistics.containsKey("peak:PaimonJniAsyncReaderThreadPeakCount")); + Assert.assertTrue(statistics.containsKey("counter:PaimonJniReadBatchCalls")); + Assert.assertTrue(statistics.containsKey("timer:PaimonJniScannerOpenTime")); + Assert.assertTrue(statistics.containsKey("timer:PaimonJniReadBatchTime")); + Assert.assertTrue(Long.parseLong(statistics.get("bytes_gauge:PaimonJniJvmHeapUsed")) > 0); + Assert.assertTrue(Long.parseLong(statistics.get("bytes_gauge:PaimonJniJvmHeapCommitted")) > 0); + } + + @Test + public void testCountThreadsByNamePrefix() throws Exception { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Thread thread = new Thread(() -> { + started.countDown(); + try { + release.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }, "paimon-reader-async-thread-test"); + + thread.start(); + try { + Assert.assertTrue(started.await(5, TimeUnit.SECONDS)); + Assert.assertTrue(PaimonJniScanner.countThreadsByNamePrefix("paimon-reader-async-thread") >= 1); + } finally { + release.countDown(); + thread.join(5000); + } + } + + @Test + public void testParseDataSizeBytes() { + Assert.assertEquals(Long.valueOf(1024L), PaimonJniScanner.parseDataSizeBytes("1 KiB").get()); + Assert.assertEquals(Long.valueOf(10L * 1024L * 1024L), + PaimonJniScanner.parseDataSizeBytes("10 MiB").get()); + Assert.assertEquals(Long.valueOf(2L * 1024L * 1024L * 1024L), + PaimonJniScanner.parseDataSizeBytes("2GB").get()); + Assert.assertFalse(PaimonJniScanner.parseDataSizeBytes("unknown").isPresent()); + } + + @Test + public void testCloseReleasesActiveRecordIterator() throws Exception { + PaimonJniScanner scanner = new PaimonJniScanner(128, createBaseParams()); + AtomicBoolean released = new AtomicBoolean(false); + RecordReader.RecordIterator recordIterator = + new RecordReader.RecordIterator() { + @Override + public InternalRow next() { + return null; + } + + @Override + public void releaseBatch() { + released.set(true); + } + }; + + Field recordIteratorField = PaimonJniScanner.class.getDeclaredField("recordIterator"); + recordIteratorField.setAccessible(true); + recordIteratorField.set(scanner, recordIterator); + + scanner.close(); + + Assert.assertTrue(released.get()); + Assert.assertNull(recordIteratorField.get(scanner)); + } + + @Test + public void testFailedCloseRetainsResourcesForRetry() throws Exception { + PaimonJniScanner scanner = new PaimonJniScanner(128, createBaseParams()); + AtomicInteger iteratorCloseCalls = new AtomicInteger(); + RecordReader.RecordIterator recordIterator = + new RecordReader.RecordIterator() { + @Override + public InternalRow next() { + return null; + } + + @Override + public void releaseBatch() { + if (iteratorCloseCalls.incrementAndGet() == 1) { + throw new RuntimeException("injected iterator close failure"); + } + } + }; + AtomicInteger readerCloseCalls = new AtomicInteger(); + RecordReader reader = new RecordReader() { + @Override + public RecordIterator readBatch() { + return null; + } + + @Override + public void close() throws IOException { + if (readerCloseCalls.incrementAndGet() == 1) { + throw new IOException("injected reader close failure"); + } + } + }; + RetryableIOManager ioManager = new RetryableIOManager(); + + Field recordIteratorField = PaimonJniScanner.class.getDeclaredField("recordIterator"); + recordIteratorField.setAccessible(true); + recordIteratorField.set(scanner, recordIterator); + Field readerField = PaimonJniScanner.class.getDeclaredField("reader"); + readerField.setAccessible(true); + readerField.set(scanner, reader); + Field ioManagerField = PaimonJniScanner.class.getDeclaredField("ioManager"); + ioManagerField.setAccessible(true); + ioManagerField.set(scanner, ioManager); + + try { + scanner.close(); + Assert.fail("expected the first close to fail"); + } catch (IOException expected) { + Assert.assertEquals("Failed to release Paimon record iterator", expected.getMessage()); + } + Assert.assertSame(recordIterator, recordIteratorField.get(scanner)); + Assert.assertSame(reader, readerField.get(scanner)); + Assert.assertSame(ioManager, ioManagerField.get(scanner)); + + scanner.close(); + Assert.assertNull(recordIteratorField.get(scanner)); + Assert.assertNull(readerField.get(scanner)); + Assert.assertNull(ioManagerField.get(scanner)); + Assert.assertEquals(2, iteratorCloseCalls.get()); + Assert.assertEquals(2, readerCloseCalls.get()); + Assert.assertEquals(2, ioManager.closeCalls.get()); + } + + private Map createBaseParams() { + Map params = new HashMap<>(); + params.put("required_fields", ""); + params.put("columns_types", ""); + params.put("paimon_split", ""); + params.put("paimon_predicate", ""); + return params; + } + + private void setTableOptions(PaimonJniScanner scanner, Map options) throws Exception { + Table table = (Table) Proxy.newProxyInstance( + Table.class.getClassLoader(), new Class[] {Table.class}, (proxy, method, args) -> { + if ("options".equals(method.getName())) { + return options; + } + if ("toString".equals(method.getName())) { + return "TestPaimonTable"; + } + throw new UnsupportedOperationException(method.getName()); + }); + Field tableField = PaimonJniScanner.class.getDeclaredField("table"); + tableField.setAccessible(true); + tableField.set(scanner, table); + } + + public static class TestIOManager implements IOManager { + private final String[] tempDirs; + + public TestIOManager(String[] tempDirs) { + this.tempDirs = tempDirs; + } + + @Override + public FileIOChannel.ID createChannel() { + throw new UnsupportedOperationException(); + } + + @Override + public FileIOChannel.ID createChannel(String channelName) { + throw new UnsupportedOperationException(); + } + + @Override + public String[] tempDirs() { + return tempDirs; + } + + @Override + public FileIOChannel.Enumerator createChannelEnumerator() { + throw new UnsupportedOperationException(); + } + + @Override + public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channel) { + throw new UnsupportedOperationException(); + } + + @Override + public BufferFileReader createBufferFileReader(FileIOChannel.ID channel) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + } + } + + public static class RetryableIOManager extends TestIOManager { + private final AtomicInteger closeCalls = new AtomicInteger(); + + public RetryableIOManager() { + super(new String[0]); + } + + @Override + public void close() { + if (closeCalls.incrementAndGet() == 1) { + throw new RuntimeException("injected IO manager close failure"); + } + } + } + + @Test + public void testGetFieldIndexMatchesMixedCaseColumns() { + Assert.assertEquals(1, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"), + "mixed_col")); + Assert.assertEquals(2, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"), + "part")); + Assert.assertEquals(-1, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"), + "missing_col")); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java index 6e4f44add930f0..76baa5fec5b3cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java @@ -75,6 +75,24 @@ public class BrokerUtil { private static final int READ_BUFFER_SIZE_B = 1024 * 1024; + public static final class ParsedColumnsFromPath { + private final List values; + private final List isNull; + + private ParsedColumnsFromPath(List values, List isNull) { + this.values = values; + this.isNull = isNull; + } + + public List getValues() { + return values; + } + + public List getIsNull() { + return isNull; + } + } + /** * Parse file status in path with broker, except directory * @param path @@ -203,6 +221,21 @@ public static List parseColumnsFromPath( return Lists.newArrayList(columns); } + public static ParsedColumnsFromPath normalizeColumnsFromPath(List columnsFromPath) { + if (columnsFromPath == null || columnsFromPath.isEmpty()) { + return new ParsedColumnsFromPath(Collections.emptyList(), Collections.emptyList()); + } + List values = new ArrayList<>(columnsFromPath.size()); + List isNull = new ArrayList<>(columnsFromPath.size()); + for (String value : columnsFromPath) { + boolean nullValue = value == null || FeConstants.null_string.equals(value) + || HiveExternalMetaCache.HIVE_DEFAULT_PARTITION.equals(value); + values.add(nullValue ? "" : value); + isNull.add(nullValue); + } + return new ParsedColumnsFromPath(values, isNull); + } + /** * Read binary data from path with broker * @param path diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 784315fc15c7bd..6a986da45303d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -34,6 +34,7 @@ import org.apache.doris.thrift.schema.external.TStructField; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,6 +46,9 @@ private static TField getExternalSchema(Column column) { root.setId(column.getUniqueId()); root.setIsOptional(column.isAllowNull()); root.setType(column.getType().toColumnTypeThrift()); + if (column.getDefaultValue() != null) { + root.setInitialDefaultValue(column.getDefaultValue()); + } TNestedField nestedField = new TNestedField(); if (column.getType().isStructType()) { @@ -120,7 +124,8 @@ private static TStructField getExternalSchemaForPrunedColumn(List columns, Map> nameMapping) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, Collections.emptyMap()); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, + Map base64InitialDefaults) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); - tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping)); + tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping, base64InitialDefaults)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, - Map> nameMapping) { + Map> nameMapping, Map base64InitialDefaults) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); - fieldPtr.setFieldPtr(getExternalSchema(child.getType(), child, nameMapping)); + fieldPtr.setFieldPtr(getExternalSchema( + child.getType(), child, nameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } return structField; @@ -149,11 +161,22 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { + return getExternalSchema(columnType, dorisColumn, nameMapping, Collections.emptyMap()); + } + + private static TField getExternalSchema(Type columnType, Column dorisColumn, + Map> nameMapping, Map base64InitialDefaults) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); root.setIsOptional(dorisColumn.isAllowNull()); root.setType(dorisColumn.getType().toColumnTypeThrift()); + if (base64InitialDefaults.containsKey(dorisColumn.getUniqueId())) { + root.setInitialDefaultValue(base64InitialDefaults.get(dorisColumn.getUniqueId())); + root.setInitialDefaultValueIsBase64(true); + } else if (dorisColumn.getDefaultValue() != null) { + root.setInitialDefaultValue(dorisColumn.getDefaultValue()); + } if (nameMapping != null && nameMapping.containsKey(dorisColumn.getUniqueId())) { // for iceberg set name mapping. @@ -175,7 +198,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, for (StructField subField : dorisStructType.getFields()) { TFieldPtr fieldPtr = new TFieldPtr(); Column subColumn = subNameToSubColumn.get(subField.getName()); - fieldPtr.setFieldPtr(getExternalSchema(subField.getType(), subColumn, nameMapping)); + fieldPtr.setFieldPtr(getExternalSchema( + subField.getType(), subColumn, nameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } @@ -187,7 +211,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TArrayField listField = new TArrayField(); TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping)); + dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, + base64InitialDefaults)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -197,12 +222,14 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TMapField mapField = new TMapField(); TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( - dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping)); + dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, + base64InitialDefaults)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( - dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping)); + dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, + base64InitialDefaults)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); @@ -210,4 +237,3 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, return root; } } - diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index e99db552c6b686..6f201e5ae66a10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -449,11 +449,14 @@ private TScanRangeLocations splitToScanRange( HiveSplit hiveSplit = (HiveSplit) fileSplit; isACID = hiveSplit.isACID(); } - List partitionValuesFromPath = fileSplit.getPartitionValues() == null + List rawPartitionValues = fileSplit.getPartitionValues() == null ? BrokerUtil.parseColumnsFromPath(fileSplit.getPathString(), pathPartitionKeys, false, isACID) : fileSplit.getPartitionValues(); + BrokerUtil.ParsedColumnsFromPath partitionValues = + BrokerUtil.normalizeColumnsFromPath(rawPartitionValues); - TFileRangeDesc rangeDesc = createFileRangeDesc(fileSplit, partitionValuesFromPath, pathPartitionKeys); + TFileRangeDesc rangeDesc = createFileRangeDesc(fileSplit, partitionValues.getValues(), + pathPartitionKeys, partitionValues.getIsNull()); TFileCompressType fileCompressType = getFileCompressType(fileSplit); rangeDesc.setCompressType(fileCompressType); // set file format type, and the type might fall back to native format in setScanParams @@ -530,7 +533,8 @@ private TScanRangeLocations newLocations() { } private TFileRangeDesc createFileRangeDesc(FileSplit fileSplit, List columnsFromPath, - List columnsFromPathKeys) { + List columnsFromPathKeys, + List columnsFromPathIsNull) { TFileRangeDesc rangeDesc = new TFileRangeDesc(); rangeDesc.setStartOffset(fileSplit.getStart()); rangeDesc.setSize(fileSplit.getLength()); @@ -540,6 +544,7 @@ private TFileRangeDesc createFileRangeDesc(FileSplit fileSplit, List col if (!columnsFromPathKeys.isEmpty()) { rangeDesc.setColumnsFromPath(columnsFromPath); rangeDesc.setColumnsFromPathKeys(columnsFromPathKeys); + rangeDesc.setColumnsFromPathIsNull(columnsFromPathIsNull); } rangeDesc.setFileType(fileSplit.getLocationType()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index e67e8f432011d4..05d78c37a649e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -1246,6 +1246,9 @@ public Map getSupportedSysTables() { public TFileFormatType getFileFormatType(SessionVariable sessionVariable) throws UserException { TFileFormatType type = null; Table table = getRemoteTable(); + // now hive self only support mixed with orc/parquet files in table and different partitions + // But if mixed with orc/parquet files in table and same partition, will failed when read. + // now here hive used table format, so BE will regrard all files in table is same format. String inputFormatName = table.getSd().getInputFormat(); String hiveFormat = HiveMetaStoreClientHelper.HiveFileFormat.getFormat(inputFormatName); if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.PARQUET.getDesc())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index bed07db31bdb5c..ad7539af255150 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -57,6 +57,7 @@ import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; @@ -132,6 +133,7 @@ protected void doInitialize() throws UserException { super.doInitialize(); if (hmsTable.isHiveTransactionalTable()) { + markTransactionalHiveScanParams(params); this.hiveTransaction = new HiveTransaction(DebugUtil.printId(ConnectContext.get().queryId()), ConnectContext.get().getQualifiedUser(), hmsTable, hmsTable.isFullAcidTable()); Env.getCurrentHiveTransactionMgr().register(hiveTransaction); @@ -139,6 +141,14 @@ protected void doInitialize() throws UserException { } } + static void markTransactionalHiveScanParams(TFileScanRangeParams scanParams) { + // BE selects the scanner before remote batch splits are fetched, so expose the table format + // in scan-level params as well as in each range. + TTableFormatFileDesc tableFormatParams = new TTableFormatFileDesc(); + tableFormatParams.setTableFormatType(TableFormatType.TRANSACTIONAL_HIVE.value()); + scanParams.setTableFormatParams(tableFormatParams); + } + protected List getPartitions() throws AnalysisException { List resPartitions = Lists.newArrayList(); HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() @@ -573,7 +583,7 @@ protected TFileAttributes getFileAttributes() throws UserException { textParams.setNullFormat(""); fileAttributes.setTextParams(textParams); fileAttributes.setHeaderType(""); - if (textParams.isSetEnclose()) { + if (shouldTrimDoubleQuotes(textParams)) { fileAttributes.setTrimDoubleQuotes(true); } fileAttributes.setEnableTextValidateUtf8( @@ -630,6 +640,10 @@ protected TFileAttributes getFileAttributes() throws UserException { return fileAttributes; } + static boolean shouldTrimDoubleQuotes(TFileTextScanRangeParams textParams) { + return textParams.isSetEnclose() && textParams.getEnclose() == (byte) '"'; + } + @Override protected TFileCompressType getFileCompressType(FileSplit fileSplit) throws UserException { TFileCompressType compressType = super.getFileCompressType(fileSplit); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java index 56fa03120d9f37..de19d90728d606 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java @@ -29,6 +29,7 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import java.util.Collections; import java.util.List; @@ -37,14 +38,21 @@ */ public class DorisTypeToIcebergType extends DorisTypeVisitor { private final StructType root; + private final List rootFieldNames; private int nextId = 0; public DorisTypeToIcebergType() { this.root = null; + this.rootFieldNames = Collections.emptyList(); } public DorisTypeToIcebergType(StructType root) { + this(root, Collections.emptyList()); + } + + public DorisTypeToIcebergType(StructType root, List rootFieldNames) { this.root = root; + this.rootFieldNames = rootFieldNames; // the root struct's fields use the first ids this.nextId = root.getFields().size(); } @@ -65,10 +73,11 @@ public Type struct(StructType struct, List types) { Type type = types.get(i); int id = isRoot ? i : getNextId(); + String fieldName = isRoot && !rootFieldNames.isEmpty() ? rootFieldNames.get(i) : field.getName(); if (field.getContainsNull()) { - newFields.add(Types.NestedField.optional(id, field.getName(), type, field.getComment())); + newFields.add(Types.NestedField.optional(id, fieldName, type, field.getComment())); } else { - newFields.add(Types.NestedField.required(id, field.getName(), type, field.getComment())); + newFields.add(Types.NestedField.required(id, fieldName, type, field.getComment())); } } return Types.StructType.of(newFields); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 8276bf71dfdf3a..db98bb283d41ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -356,8 +356,8 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx .map(col -> new StructField(col.getName(), col.getType(), col.getComment(), col.isAllowNull())) .collect(Collectors.toList()); StructType structType = new StructType(new ArrayList<>(collect)); - Type visit = - DorisTypeVisitor.visit(structType, new DorisTypeToIcebergType(structType)); + List rootFieldNames = columns.stream().map(Column::getName).collect(Collectors.toList()); + Type visit = DorisTypeVisitor.visit(structType, new DorisTypeToIcebergType(structType, rootFieldNames)); Schema schema = new Schema(visit.asNestedType().asStructType().fields()); Map properties = createTableInfo.getProperties(); properties.put(ExternalCatalog.DORIS_VERSION, ExternalCatalog.DORIS_VERSION_VALUE); @@ -1234,7 +1234,7 @@ private org.apache.iceberg.SortOrder buildSortOrder(List sortFiel org.apache.iceberg.SortOrder.Builder builder = org.apache.iceberg.SortOrder.builderFor(schema); for (SortFieldInfo sortField : sortFields) { - String columnName = sortField.getColumnName(); + String columnName = getIcebergColumnName(schema, sortField.getColumnName()); if (sortField.isAscending()) { if (sortField.isNullFirst()) { builder.asc(columnName, org.apache.iceberg.NullOrder.NULLS_FIRST); @@ -1251,4 +1251,9 @@ private org.apache.iceberg.SortOrder buildSortOrder(List sortFiel } return builder.build(); } + + private static String getIcebergColumnName(Schema schema, String columnName) { + NestedField field = schema.caseInsensitiveFindField(columnName); + return field == null ? columnName : field.name(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 676dfd258f1cb9..638b0474771146 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -106,6 +106,7 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -133,10 +134,11 @@ import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; +import java.util.Base64; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -541,34 +543,44 @@ public static PartitionSpec solveIcebergPartitionSpec(PartitionDesc partitionDes PartitionSpec.Builder builder = PartitionSpec.builderFor(schema); for (Expr expr : partitionExprs) { if (expr instanceof SlotRef) { - builder.identity(((SlotRef) expr).getColumnName()); + builder.identity(getIcebergColumnName(schema, ((SlotRef) expr).getColumnName())); } else if (expr instanceof FunctionCallExpr) { String exprName = expr.getExprName(); List params = ((FunctionCallExpr) expr).getParams().exprs(); switch (exprName.toLowerCase()) { case "bucket": - builder.bucket(params.get(1).getExprName(), Integer.parseInt(params.get(0).getStringValue())); + builder.bucket( + getIcebergColumnName(schema, + params.get(1).getExprName()), + Integer.parseInt(params.get(0).getStringValue())); break; case "year": case "years": - builder.year(params.get(0).getExprName()); + builder.year(getIcebergColumnName(schema, + params.get(0).getExprName())); break; case "month": case "months": - builder.month(params.get(0).getExprName()); + builder.month(getIcebergColumnName(schema, + params.get(0).getExprName())); break; case "date": case "day": case "days": - builder.day(params.get(0).getExprName()); + builder.day(getIcebergColumnName(schema, + params.get(0).getExprName())); break; case "date_hour": case "hour": case "hours": - builder.hour(params.get(0).getExprName()); + builder.hour(getIcebergColumnName(schema, + params.get(0).getExprName())); break; case "truncate": - builder.truncate(params.get(1).getExprName(), Integer.parseInt(params.get(0).getStringValue())); + builder.truncate( + getIcebergColumnName(schema, + params.get(1).getExprName()), + Integer.parseInt(params.get(0).getStringValue())); break; default: throw new UserException("unsupported partition for " + exprName); @@ -578,6 +590,11 @@ public static PartitionSpec solveIcebergPartitionSpec(PartitionDesc partitionDes return builder.build(); } + private static String getIcebergColumnName(Schema schema, String columnName) { + Types.NestedField field = schema.caseInsensitiveFindField(columnName); + return field == null ? columnName : field.name(); + } + private static Type icebergPrimitiveTypeToDorisType(org.apache.iceberg.types.Type.PrimitiveType primitive, boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { switch (primitive.typeId()) { @@ -714,6 +731,56 @@ public static Map getPartitionInfoMap(PartitionData partitionDat return partitionInfoMap; } + public static List getIdentityPartitionColumns(Table table) { + LinkedHashSet partitionColumns = new LinkedHashSet<>(); + for (PartitionSpec spec : table.specs().values()) { + for (PartitionField partitionField : spec.fields()) { + if (!partitionField.transform().isIdentity()) { + continue; + } + String columnName = table.schema().findColumnName(partitionField.sourceId()); + if (columnName != null) { + partitionColumns.add(columnName); + } + } + } + return new ArrayList<>(partitionColumns); + } + + public static Map getIdentityPartitionInfoMap(PartitionData partitionData, + PartitionSpec partitionSpec, Table table, String timeZone) { + Map partitionInfoMap = Maps.newLinkedHashMap(); + List fields = partitionData.getPartitionType().asNestedType().fields(); + List partitionFields = partitionSpec.fields(); + Preconditions.checkArgument(fields.size() == partitionFields.size(), + "PartitionData fields size does not match PartitionSpec fields size"); + + for (int i = 0; i < fields.size(); i++) { + NestedField field = fields.get(i); + PartitionField partitionField = partitionFields.get(i); + if (!partitionField.transform().isIdentity()) { + continue; + } + TypeID partitionTypeId = field.type().typeId(); + if (partitionTypeId == TypeID.BINARY || partitionTypeId == TypeID.FIXED) { + continue; + } + + String columnName = table.schema().findColumnName(partitionField.sourceId()); + if (columnName == null) { + continue; + } + Object value = partitionData.get(i); + try { + partitionInfoMap.put(columnName, serializePartitionValue(field.type(), value, timeZone)); + } catch (UnsupportedOperationException e) { + LOG.warn("Failed to serialize Iceberg table partition value for field {}: {}", field.name(), + e.getMessage()); + } + } + return partitionInfoMap; + } + public static List getPartitionValues(PartitionData partitionData, PartitionSpec partitionSpec, String timeZone) { List fields = partitionData.getPartitionType().asNestedType().fields(); @@ -1047,10 +1114,14 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi List columns = schema.columns(); List resSchema = Lists.newArrayListWithCapacity(columns.size()); for (Types.NestedField field : columns) { - Column column = new Column(field.name().toLowerCase(Locale.ROOT), + String initialDefault = null; + if (field.initialDefault() != null) { + initialDefault = serializeInitialDefault(field.type(), field.initialDefault(), + enableMappingTimestampTz); + } + Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), - true, null, - true, field.doc(), true, -1); + true, null, true, initialDefault, field.doc(), true, -1); updateIcebergColumnUniqueId(column, field); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); @@ -1063,6 +1134,62 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi return resSchema; } + private static String serializeInitialDefault(org.apache.iceberg.types.Type type, Object value, + boolean enableMappingTimestampTz) { + String humanValue = Transforms.identity(type).toHumanString(type, value); + if (type.typeId() == TypeID.TIMESTAMP) { + // Iceberg formats timestamps as ISO-8601 (for example 2024-01-01T00:00:00), while + // Doris' DATETIMEV2 default parser requires a space between the date and time. + String dorisValue = humanValue.replace('T', ' '); + Types.TimestampType timestampType = (Types.TimestampType) type; + if (timestampType.shouldAdjustToUTC() && !enableMappingTimestampTz) { + // Iceberg timestamptz human values carry a trailing offset. DATETIMEV2 has no + // offset carrier, so retain the displayed UTC wall time and remove the suffix. + return dorisValue.replaceFirst("(Z|[+-]\\d{2}:\\d{2})$", ""); + } + return dorisValue; + } + if (isBinaryLike(type)) { + // Always use the lossless Base64 carrier. Binary-like Iceberg fields may map to either + // VARBINARY or STRING/CHAR, and the scan schema marker tells BE to decode both forms + // back to the raw bytes stored in equality-delete files. + return serializeBinaryInitialDefault(type, value); + } + return humanValue; + } + + /** + * Return binary-like initial defaults in a lossless transport representation. These defaults + * cannot be carried as raw Java strings and their Doris type is insufficient to identify them + * when varbinary mapping is disabled, because UUID/BINARY/FIXED then map to STRING/CHAR. + */ + public static Map getBase64EncodedInitialDefaults(Schema schema) { + Map result = Maps.newHashMap(); + for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) { + if (field.initialDefault() == null || !isBinaryLike(field.type())) { + continue; + } + result.put(field.fieldId(), serializeBinaryInitialDefault(field.type(), field.initialDefault())); + } + return result; + } + + private static boolean isBinaryLike(org.apache.iceberg.types.Type type) { + return type.typeId() == TypeID.UUID || type.typeId() == TypeID.BINARY + || type.typeId() == TypeID.FIXED; + } + + private static String serializeBinaryInitialDefault(org.apache.iceberg.types.Type type, Object value) { + if (type.typeId() != TypeID.UUID) { + return Transforms.identity(type).toHumanString(type, value); + } + UUID uuid = (UUID) value; + ByteBuffer bytes = ByteBuffer.allocate(16); + bytes.putLong(uuid.getMostSignificantBits()); + bytes.putLong(uuid.getLeastSignificantBits()); + return Base64.getEncoder().encodeToString(bytes.array()); + } + /** * Decide whether a row count can be read from an Iceberg snapshot summary. * Returns {@link TableIf#UNKNOWN_ROW_COUNT} when required counters are absent diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index bb1110687fc459..a158d5c6ddbba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -18,11 +18,9 @@ package org.apache.doris.datasource.iceberg.source; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; -import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; @@ -80,6 +78,7 @@ import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; +import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; @@ -415,20 +414,26 @@ public void createScanRangeLocations() throws UserException { // Extract name mapping from Iceberg table properties Map> nameMapping = extractNameMapping(); - boolean haveTopnLazyMatCol = false; - for (SlotDescriptor slot : desc.getSlots()) { - String colName = slot.getColumn().getName(); - if (colName.startsWith(Column.GLOBAL_ROWID_COL)) { - haveTopnLazyMatCol = true; - break; - } - } - if (haveTopnLazyMatCol) { - ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), nameMapping); - } else { - // Use new initSchemaInfo method that only includes needed columns based on slots and pruned type - ExternalUtil.initSchemaInfoForPrunedColumn(params, -1L, desc.getSlots(), nameMapping); - } + // Equality-delete keys are hidden scan dependencies and need not appear in the query + // projection. Both scanners need the complete current schema to resolve field ids, + // historical names, types, and initial defaults when an old data file lacks such a key. + ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), + nameMapping, getBase64EncodedInitialDefaultsForScan()); + } + + @VisibleForTesting + Map getBase64EncodedInitialDefaultsForScan() throws UserException { + TableScan tableScan = createTableScan(); + Snapshot snapshot = tableScan.snapshot(); + // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. + // Resolve the selected snapshot's schema id explicitly so this metadata describes the same + // snapshot as source.getTargetTable().getColumns(). Otherwise a later type change can make + // BE decode a historical non-binary default as Base64, or fail to decode a binary default. + Schema scanSchema = snapshot == null + ? tableScan.schema() + : tableScan.table().schemas().get(snapshot.schemaId()); + return IcebergUtils.getBase64EncodedInitialDefaults( + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan snapshot is null")); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 1775a984f2cd5f..6a744f765e8e2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -346,7 +346,7 @@ public Optional initSchema(SchemaCacheKey key) { Set partitionColumnNames = Sets.newHashSet(tableSchema.partitionKeys()); List partitionColumns = Lists.newArrayList(); for (DataField field : columns) { - Column column = new Column(field.name().toLowerCase(), + Column column = new Column(field.name(), PaimonUtil.paimonTypeToDorisType(field.type(), getCatalog().getEnableMappingVarbinary(), getCatalog().getEnableMappingTimestampTz()), true, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java index e6c8177edcae47..bd3245f0f9f431 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java @@ -57,6 +57,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -218,7 +219,9 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx col.getComment(), col.isNullable())) .collect(Collectors.toList()); StructType structType = new StructType(new ArrayList<>(collect)); - Schema schema = toPaimonSchema(structType, createTableInfo.getPartitionDesc(), createTableInfo.getProperties()); + List rootFieldNames = columns.stream().map(ColumnDefinition::getName).collect(Collectors.toList()); + Schema schema = toPaimonSchema(structType, rootFieldNames, createTableInfo.getPartitionDesc(), + createTableInfo.getProperties()); try { catalog.createTable(new Identifier(createTableInfo.getDbName(), createTableInfo.getTableName()), schema, createTableInfo.isIfNotExists()); @@ -228,7 +231,8 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx return false; } - private Schema toPaimonSchema(StructType structType, PartitionDesc partitionDesc, Map properties) { + private Schema toPaimonSchema(StructType structType, List rootFieldNames, PartitionDesc partitionDesc, + Map properties) { Map normalizedProperties = new HashMap<>(properties); normalizedProperties.remove(PRIMARY_KEY_IDENTIFIER); normalizedProperties.remove(PROP_COMMENT); @@ -242,19 +246,31 @@ private Schema toPaimonSchema(StructType structType, PartitionDesc partitionDesc .map(String::trim) .collect(Collectors.toList()); List partitionKeys = partitionDesc == null ? new ArrayList<>() : partitionDesc.getPartitionColNames(); + primaryKeys = getPaimonColumnNames(rootFieldNames, primaryKeys); + partitionKeys = getPaimonColumnNames(rootFieldNames, partitionKeys); Schema.Builder schemaBuilder = Schema.newBuilder() .options(normalizedProperties) .primaryKey(primaryKeys) .partitionKeys(partitionKeys) .comment(properties.getOrDefault(PROP_COMMENT, null)); - for (StructField field : structType.getFields()) { - schemaBuilder.column(field.getName(), + List fields = structType.getFields(); + for (int i = 0; i < fields.size(); i++) { + StructField field = fields.get(i); + schemaBuilder.column(rootFieldNames.get(i), toPaimontype(field.getType()).copy(field.getContainsNull()), field.getComment()); } return schemaBuilder.build(); } + private List getPaimonColumnNames(List paimonColumnNames, List dorisColumnNames) { + Map paimonColumnNameMap = paimonColumnNames.stream() + .collect(Collectors.toMap(name -> name.toLowerCase(Locale.ROOT), name -> name)); + return dorisColumnNames.stream() + .map(name -> paimonColumnNameMap.getOrDefault(name.toLowerCase(Locale.ROOT), name)) + .collect(Collectors.toList()); + } + private DataType toPaimontype(Type type) { return DorisTypeVisitor.visit(type, new DorisToPaimonTypeVisitor()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java index db972c6b2b6a1f..2ce78387d04135 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java @@ -137,16 +137,19 @@ public Table getSysPaimonTable() { @Override public List getFullSchema() { Table sysTable = getSysPaimonTable(); - List fields = sysTable.rowType().getFields(); + return buildFullSchema(sysTable.rowType().getFields(), getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz()); + } + + static List buildFullSchema(List fields, boolean enableMappingVarbinary, + boolean enableMappingTimestampTz) { List columns = Lists.newArrayListWithCapacity(fields.size()); for (DataField field : fields) { Column column = new Column( - field.name().toLowerCase(), + field.name(), PaimonUtil.paimonTypeToDorisType( - field.type(), - getCatalog().getEnableMappingVarbinary(), - getCatalog().getEnableMappingTimestampTz()), + field.type(), enableMappingVarbinary, enableMappingTimestampTz), true, null, true, @@ -253,4 +256,5 @@ public Map getTableProperties() { public String getComment() { return "Paimon system table: " + sysTableType + " for " + sourceTable.getName(); } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 2a738a0ac8da23..1c71ac3dd248a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -503,7 +503,7 @@ public static List parseSchema(RowType rowType, List primaryKeys boolean enableTimestampTzMapping) { List resSchema = Lists.newArrayListWithCapacity(rowType.getFields().size()); rowType.getFields().forEach(field -> { - resSchema.add(new Column(field.name().toLowerCase(), + resSchema.add(new Column(field.name(), PaimonUtil.paimonTypeToDorisType(field.type(), enableVarbinaryMapping, enableTimestampTzMapping), primaryKeys.contains(field.name()), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java index 73a3c72ddcc7c3..ae45c2427184e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java @@ -46,7 +46,7 @@ public class PaimonPredicateConverter { public PaimonPredicateConverter(RowType rowType) { this.builder = new PredicateBuilder(rowType); - this.fieldNames = rowType.getFields().stream().map(f -> f.name().toLowerCase()).collect(Collectors.toList()); + this.fieldNames = rowType.getFields().stream().map(DataField::name).collect(Collectors.toList()); this.paimonFieldTypes = rowType.getFields().stream().map(DataField::type).collect(Collectors.toList()); } @@ -99,7 +99,7 @@ private Predicate doInPredicate(InPredicate predicate) { return null; } String colName = slotRef.getColumnName(); - int idx = fieldNames.indexOf(colName); + int idx = getFieldIndex(colName); DataType dataType = paimonFieldTypes.get(idx); List valueList = new ArrayList<>(); for (int i = 1; i < predicate.getChildren().size(); i++) { @@ -132,7 +132,7 @@ private Predicate binaryExprDesc(Expr dorisExpr) { return null; } String colName = slotRef.getColumnName(); - int idx = fieldNames.indexOf(colName); + int idx = getFieldIndex(colName); DataType dataType = paimonFieldTypes.get(idx); Object value = dataType.accept(new PaimonValueConverter(literalExpr)); if (value == null) { @@ -174,6 +174,15 @@ private Predicate binaryExprDesc(Expr dorisExpr) { } + private int getFieldIndex(String colName) { + for (int i = 0; i < fieldNames.size(); i++) { + if (fieldNames.get(i).equalsIgnoreCase(colName)) { + return i; + } + } + return fieldNames.indexOf(colName); + } + public static SlotRef convertDorisExprToSlotRef(Expr expr) { SlotRef slotRef = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index f465e3c9534fe3..a8e331f9e468f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -48,6 +48,7 @@ import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TPaimonDeletionFileDesc; import org.apache.doris.thrift.TPaimonFileDesc; +import org.apache.doris.thrift.TPaimonReaderType; import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; @@ -68,6 +69,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -93,6 +95,14 @@ public class PaimonScanNode extends FileQueryScanNode { private static final String DORIS_START_TIMESTAMP = "startTimestamp"; private static final String DORIS_END_TIMESTAMP = "endTimestamp"; private static final String DORIS_INCREMENTAL_BETWEEN_SCAN_MODE = "incrementalBetweenScanMode"; + private static final String PAIMON_PROPERTY_PREFIX = "paimon."; + private static final String DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; + private static final String DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; + private static final String DORIS_JNI_IO_MANAGER_IMPL_CLASS = "doris.jni_io_manager.impl_class"; + private static final List BACKEND_PAIMON_OPTIONS = Arrays.asList( + DORIS_ENABLE_JNI_IO_MANAGER, + DORIS_JNI_IO_MANAGER_TMP_DIR, + DORIS_JNI_IO_MANAGER_IMPL_CLASS); private static final String PAIMON_BINLOG_SYSTEM_TABLE_TYPE = "binlog"; private static final String PAIMON_AUDIT_LOG_SYSTEM_TABLE_TYPE = "audit_log"; @@ -218,6 +228,19 @@ private void setScanLevelPaimonOptions() { } } + private List getOrderedPathPartitionKeys() { + if (source == null) { + return Collections.emptyList(); + } + ExternalTable externalTable = source.getExternalTable(); + if (externalTable instanceof PaimonSysExternalTable + && !((PaimonSysExternalTable) externalTable).isDataTable()) { + return Collections.emptyList(); + } + Table paimonTable = source.getPaimonTable(); + return paimonTable == null ? Collections.emptyList() : paimonTable.partitionKeys(); + } + private void putHistorySchemaInfo(Long schemaId) { if (currentQuerySchema.putIfAbsent(schemaId, Boolean.TRUE) == null) { ExternalTable targetTable = source.getExternalTable(); @@ -235,20 +258,6 @@ private void putHistorySchemaInfo(Long schemaId) { } } - private List getOrderedPathPartitionKeys() { - if (source == null) { - return Collections.emptyList(); - } - ExternalTable externalTable = source.getExternalTable(); - if (externalTable instanceof PaimonSysExternalTable - && !((PaimonSysExternalTable) externalTable).isDataTable()) { - return Collections.emptyList(); - } - return source.getPaimonTable().partitionKeys().stream() - .map(key -> key.toLowerCase(Locale.ROOT)) - .collect(Collectors.toList()); - } - @VisibleForTesting void setPartitionValues(TFileRangeDesc rangeDesc, Map partitionValues) { rangeDesc.unsetColumnsFromPathKeys(); @@ -270,9 +279,10 @@ void setPartitionValues(TFileRangeDesc rangeDesc, Map partitionV List fromPathValues = new ArrayList<>(orderedPartitionKeys.size()); List fromPathIsNull = new ArrayList<>(orderedPartitionKeys.size()); for (String partitionKey : orderedPartitionKeys) { - Preconditions.checkState(normalizedPartitionValues.containsKey(partitionKey), + String normalizedPartitionKey = partitionKey.toLowerCase(Locale.ROOT); + Preconditions.checkState(normalizedPartitionValues.containsKey(normalizedPartitionKey), "Missing partition value for Paimon partition key: %s", partitionKey); - String partitionValue = normalizedPartitionValues.get(partitionKey); + String partitionValue = normalizedPartitionValues.get(normalizedPartitionKey); fromPathValues.add(partitionValue == null ? "" : partitionValue); fromPathIsNull.add(partitionValue == null); } @@ -293,8 +303,10 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI); // Use Paimon native serialization for paimon-cpp reader if (sessionVariable.isEnablePaimonCppReader() && split instanceof DataSplit) { + fileDesc.setReaderType(TPaimonReaderType.PAIMON_CPP); fileDesc.setPaimonSplit(PaimonUtil.encodeDataSplitToString((DataSplit) split)); } else { + fileDesc.setReaderType(TPaimonReaderType.PAIMON_JNI); fileDesc.setPaimonSplit(PaimonUtil.encodeObjectToString(split)); } // Set table location for paimon-cpp reader @@ -305,6 +317,7 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) rangeDesc.setSelfSplitWeight(paimonSplit.getSelfSplitWeight()); } else { // use native reader + fileDesc.setReaderType(TPaimonReaderType.PAIMON_NATIVE); if (fileFormat.equals("orc")) { rangeDesc.setFormatType(TFileFormatType.FORMAT_ORC); } else if (fileFormat.equals("parquet")) { @@ -457,7 +470,7 @@ public List getSplits(int numBackends) throws UserException { file.length(), -1, !applyCountPushdown, - null, + Collections.emptyList(), PaimonSplit.PaimonSplitCreator.DEFAULT); for (Split dorisSplit : dorisSplits) { PaimonSplit paimonSplit = (PaimonSplit) dorisSplit; @@ -520,12 +533,24 @@ Map getBackendPaimonOptions() { return Collections.emptyMap(); } PaimonExternalCatalog catalog = (PaimonExternalCatalog) source.getCatalog(); + Map backendOptions = new HashMap<>(); + Map catalogProperties = catalog.getCatalogProperty().getProperties(); + if (catalogProperties == null) { + catalogProperties = Collections.emptyMap(); + } + for (String option : BACKEND_PAIMON_OPTIONS) { + String catalogProperty = PAIMON_PROPERTY_PREFIX + option; + if (catalogProperties.containsKey(catalogProperty)) { + backendOptions.put(option, catalogProperties.get(catalogProperty)); + } + } if (!(catalog.getCatalogProperty().getMetastoreProperties() instanceof PaimonJdbcMetaStoreProperties)) { - return Collections.emptyMap(); + return backendOptions; } PaimonJdbcMetaStoreProperties jdbcMetaStoreProperties = (PaimonJdbcMetaStoreProperties) catalog.getCatalogProperty().getMetastoreProperties(); - return jdbcMetaStoreProperties.getBackendPaimonOptions(); + backendOptions.putAll(jdbcMetaStoreProperties.getBackendPaimonOptions()); + return backendOptions; } @VisibleForTesting @@ -588,13 +613,9 @@ public Map getIncrReadParams() throws UserException { @VisibleForTesting public List getPaimonSplitFromAPI() throws UserException { Table paimonTable = getProcessedTable(); + List fieldNames = paimonTable.rowType().getFieldNames(); int[] projected = desc.getSlots().stream().mapToInt( - slot -> paimonTable.rowType() - .getFieldNames() - .stream() - .map(String::toLowerCase) - .collect(Collectors.toList()) - .indexOf(slot.getColumn().getName())) + slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) .filter(i -> i >= 0) .toArray(); ReadBuilder readBuilder = paimonTable.newReadBuilder(); @@ -613,6 +634,16 @@ public List getPaimonSplitFromAPI() throws return splits; } + @VisibleForTesting + static int getFieldIndex(List fieldNames, String columnName) { + for (int i = 0; i < fieldNames.size(); i++) { + if (fieldNames.get(i).equalsIgnoreCase(columnName)) { + return i; + } + } + return -1; + } + private String getFileFormat(String path) { return FileFormatUtils.getFileFormatBySuffix(path).orElse(source.getFileFormatFromTableProperties()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c9020b6c9dc37d..fda1b7da86e054 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -94,6 +94,7 @@ public class SessionVariable implements Serializable, Writable { public static final String SCAN_QUEUE_MEM_LIMIT = "scan_queue_mem_limit"; public static final String MAX_SCANNERS_CONCURRENCY = "max_scanners_concurrency"; public static final String MAX_FILE_SCANNERS_CONCURRENCY = "max_file_scanners_concurrency"; + public static final String ENABLE_FILE_SCANNER_V2 = "enable_file_scanner_v2"; public static final String MIN_SCANNERS_CONCURRENCY = "min_scanners_concurrency"; public static final String MIN_FILE_SCANNERS_CONCURRENCY = "min_file_scanners_concurrency"; public static final String MIN_SCAN_SCHEDULER_CONCURRENCY = "min_scan_scheduler_concurrency"; @@ -1116,6 +1117,11 @@ public static double getHotValueThreshold() { "FileScanNode 扫描数据的最大并发，默认为 16", "The max threads to read data of FileScanNode, default 16"}) public int maxFileScannersConcurrency = 16; + @VariableMgr.VarAttr(name = ENABLE_FILE_SCANNER_V2, needForward = true, description = { + "开启后 FileScanNode 会在支持的查询场景使用 FileScannerV2，默认开启", + "When enabled, FileScanNode uses FileScannerV2 for supported query scans. Enabled by default."}) + public boolean enableFileScannerV2 = true; + @VariableMgr.VarAttr(name = LOCAL_EXCHANGE_FREE_BLOCKS_LIMIT) public int localExchangeFreeBlocksLimit = 4; @@ -5446,6 +5452,7 @@ public TQueryOptions toThrift() { tResult.setScanQueueMemLimit(maxScanQueueMemByte); tResult.setMaxScannersConcurrency(maxScannersConcurrency); tResult.setMaxFileScannersConcurrency(maxFileScannersConcurrency); + tResult.setEnableFileScannerV2(enableFileScannerV2); tResult.setMaxColumnReaderNum(maxColumnReaderNum); tResult.setParallelPrepareThreshold(parallelPrepareThreshold); tResult.setMinScannersConcurrency(minScannersConcurrency); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java index 13bbd66fab1d6f..0c24316cdc4c6f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java @@ -22,8 +22,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FsBroker; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.GenericPool; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.hive.HiveExternalMetaCache; import org.apache.doris.thrift.TBrokerCloseReaderRequest; import org.apache.doris.thrift.TBrokerCloseWriterRequest; import org.apache.doris.thrift.TBrokerDeletePathRequest; @@ -156,6 +158,16 @@ public void parseColumnsFromPath() { } + @Test + public void normalizeColumnsFromPathPreservesNullInfo() { + BrokerUtil.ParsedColumnsFromPath parsed = BrokerUtil.normalizeColumnsFromPath( + Lists.newArrayList("p1", FeConstants.null_string, + HiveExternalMetaCache.HIVE_DEFAULT_PARTITION, null)); + + Assert.assertEquals(Lists.newArrayList("p1", "", "", ""), parsed.getValues()); + Assert.assertEquals(Lists.newArrayList(false, true, true, true), parsed.getIsNull()); + } + @Test public void testReadFile(@Mocked TPaloBrokerService.Client client, @Mocked Env env, @Injectable BrokerMgr brokerMgr) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 5ce09be0456e5f..3ecb3a72d6b023 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -219,7 +219,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { TFileScanRangeParams params = new TFileScanRangeParams(); Long schemaId = 500L; - Column col1 = new Column("c1", Type.INT, true); + Column col1 = new Column("c1", Type.INT, false, null, true, "7", ""); col1.setUniqueId(101); Column col2 = new Column("c2", Type.VARCHAR, false); col2.setUniqueId(102); @@ -230,7 +230,10 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { nameMapping.put(col1.getUniqueId(), Arrays.asList("m_c1")); nameMapping.put(col2.getUniqueId(), Arrays.asList("m_c2_a", "m_c2_b")); - ExternalUtil.initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping); + Map base64InitialDefaults = new HashMap<>(); + base64InitialDefaults.put(col2.getUniqueId(), "AAEC/w=="); + ExternalUtil.initSchemaInfoForAllColumn( + params, schemaId, columns, nameMapping, base64InitialDefaults); Assert.assertEquals(schemaId.longValue(), params.getCurrentSchemaId()); List history = params.getHistorySchemaInfo(); @@ -251,12 +254,15 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); + Assert.assertEquals("7", field1.getInitialDefaultValue()); + Assert.assertFalse(field1.isSetInitialDefaultValueIsBase64()); Assert.assertEquals(col2.getName(), field2.getName()); Assert.assertEquals(col2.getUniqueId(), field2.getId()); Assert.assertEquals(col2.isAllowNull(), field2.isIsOptional()); Assert.assertEquals(col2.getType().toColumnTypeThrift(), field2.getType()); Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); + Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); + Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } } - diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java index abb48bcae0b3bc..38aaceda1739d0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java @@ -19,12 +19,15 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.datasource.TableFormatType; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveExternalMetaCache; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TFileTextScanRangeParams; import org.junit.Assert; import org.junit.Test; @@ -86,4 +89,40 @@ public void testDetermineTargetFileSplitSizeKeepsInitialSize() throws Exception long target = (long) method.invoke(node, caches, false); Assert.assertEquals(32 * MB, target); } + + @Test + public void testMarkTransactionalHiveScanParams() { + TFileScanRangeParams scanParams = new TFileScanRangeParams(); + HiveScanNode.markTransactionalHiveScanParams(scanParams); + + Assert.assertTrue(scanParams.isSetTableFormatParams()); + Assert.assertEquals(TableFormatType.TRANSACTIONAL_HIVE.value(), + scanParams.getTableFormatParams().getTableFormatType()); + } + + @Test + public void testTrimDoubleQuotesOnlyForDoubleQuoteEnclose() { + TFileTextScanRangeParams textParams = new TFileTextScanRangeParams(); + textParams.setEnclose((byte) '"'); + Assert.assertTrue(HiveScanNode.shouldTrimDoubleQuotes(textParams)); + + textParams.setEnclose((byte) '\''); + Assert.assertFalse(HiveScanNode.shouldTrimDoubleQuotes(textParams)); + } + + private HiveScanNode createHiveScanNode() { + return createHiveScanNode(false); + } + + private HiveScanNode createHiveScanNode(boolean partitioned) { + SessionVariable sv = new SessionVariable(); + TupleDescriptor desc = new TupleDescriptor(new TupleId(0)); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.bindBrokerName()).thenReturn(""); + Mockito.when(table.isPartitionedTable()).thenReturn(partitioned); + desc.setTable(table); + return new HiveScanNode(new PlanNodeId(0), desc, false, sv, null, ScanContext.EMPTY); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 909359146eb55e..6962f911538f81 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -214,6 +214,45 @@ public void testPartition() throws UserException { Assert.assertEquals("b", table.properties().get("a")); } + @Test + public void testPartitionPreservesNonLowercaseColumnNames() throws UserException { + TableIdentifier tb = TableIdentifier.of(dbName, getTableName()); + String sql = "create table " + tb + " (" + + "data int, " + + "`PART` int, " + + "`mIxEd_COL` int" + + ") engine = iceberg " + + "partition by (`PART`, bucket(2, `mIxEd_COL`)) ()"; + createTable(sql); + Table table = ops.getCatalog().loadTable(tb); + Schema schema = table.schema(); + + Assert.assertEquals("PART", schema.columns().get(1).name()); + Assert.assertEquals("mIxEd_COL", schema.columns().get(2).name()); + PartitionSpec spec = PartitionSpec.builderFor(schema) + .identity("PART") + .bucket("mIxEd_COL", 2) + .build(); + Assert.assertEquals(spec, table.spec()); + } + + @Test + public void testSortOrderResolvesNonLowercaseColumnNamesCaseInsensitively() throws UserException { + TableIdentifier tb = TableIdentifier.of(dbName, getTableName()); + String sql = "create table " + tb + " (" + + "data int, " + + "`mIxEd_COL` int" + + ") engine = iceberg " + + "order by (`mixed_col` asc)"; + createTable(sql); + Table table = ops.getCatalog().loadTable(tb); + Schema schema = table.schema(); + + Assert.assertEquals("mIxEd_COL", schema.columns().get(1).name()); + Assert.assertEquals(1, table.sortOrder().fields().size()); + Assert.assertEquals(schema.findField("mIxEd_COL").fieldId(), table.sortOrder().fields().get(0).sourceId()); + } + public void createTable(String sql) throws UserException { LogicalPlan plan = new NereidsParser().parseSingle(sql); Assertions.assertTrue(plan instanceof CreateTableCommand); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index 143438a71bbb70..69506a505cb21e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -197,6 +197,10 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedTableScan).when(mockedTableScan).filter(ArgumentMatchers.any()); Mockito.doReturn(mockedTableScan).when(mockedTableScan).planWith(ArgumentMatchers.any()); Mockito.doReturn(null).when(mockedTableScan).snapshot(); + // Keep the scan schema aligned with the mocked table schema. IcebergScanNode reads the + // selected scan schema when serializing initial defaults, and several tests temporarily + // replace the table schema to exercise partition transforms. + Mockito.doAnswer(invocation -> mockedIcebergTable.schema()).when(mockedTableScan).schema(); Mockito.doReturn(CloseableIterable.withNoopClose(java.util.Collections.emptyList())) .when(mockedTableScan).planFiles(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java index 6bb96bf0d43c75..0a5a4ab11d4621 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java @@ -321,6 +321,13 @@ public void testGetPartitionSpecSqlIdentity() { Assertions.assertEquals("PARTITION BY LIST (`d_year`) ()", spy.getPartitionSpecSql()); } + @Test + public void testGetPartitionSpecSqlPreservesNonLowercaseColumnName() { + IcebergExternalTable spy = createSpyTable(); + setupSingleField(mockTransform("identity"), "mIxEd_COL"); + Assertions.assertEquals("PARTITION BY LIST (`mIxEd_COL`) ()", spy.getPartitionSpecSql()); + } + @Test public void testGetPartitionSpecSqlBucket() { IcebergExternalTable spy = createSpyTable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 49b05f3c1bb520..5c669623c412cb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -58,12 +58,15 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; public class IcebergUtilsTest { @Test @@ -202,6 +205,60 @@ public void testAppendRowLineageFieldsForV3AddsMetadataFields() { Assert.assertNotNull(schemaWithRowLineage.findField(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId())); } + @Test + public void testParseSchemaPreservesNonLowercaseColumnNames() { + Schema schema = new Schema( + Types.NestedField.required(1, "mIxEd_COL", Types.IntegerType.get()), + Types.NestedField.required(2, "PART", Types.StringType.get())); + + List columns = IcebergUtils.parseSchema(schema, false, false); + + Assert.assertEquals("mIxEd_COL", columns.get(0).getName()); + Assert.assertEquals("PART", columns.get(1).getName()); + } + + @Test + public void testParseSchemaPreservesInitialDefault() { + Schema schema = new Schema( + Types.NestedField.optional("added_column") + .withId(1) + .ofType(Types.IntegerType.get()) + .withInitialDefault(7) + .build(), + Types.NestedField.optional("added_timestamp") + .withId(2) + .ofType(Types.TimestampType.withoutZone()) + .withInitialDefault(1_704_067_200_123_456L) + .build(), + Types.NestedField.optional("added_uuid") + .withId(3) + .ofType(Types.UUIDType.get()) + .withInitialDefault(UUID.fromString("00000000-0000-0000-0000-000000000000")) + .build(), + Types.NestedField.optional("added_binary") + .withId(4) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build(), + Types.NestedField.optional("added_fixed") + .withId(5) + .ofType(Types.FixedType.ofLength(4)) + .withInitialDefault(ByteBuffer.wrap(new byte[] {3, 2, 1, 0})) + .build()); + + List columns = IcebergUtils.parseSchema(schema, true, false); + + Assert.assertEquals("7", columns.get(0).getDefaultValue()); + Assert.assertEquals("2024-01-01 00:00:00.123456", columns.get(1).getDefaultValue()); + Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", columns.get(2).getDefaultValue()); + Assert.assertEquals("AAEC/w==", columns.get(3).getDefaultValue()); + + Map base64Defaults = IcebergUtils.getBase64EncodedInitialDefaults(schema); + Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", base64Defaults.get(3)); + Assert.assertEquals("AAEC/w==", base64Defaults.get(4)); + Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( @@ -216,6 +273,84 @@ public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Assert.assertNull(partitionInfoMap); } + @Test + public void testGetIdentityPartitionColumnsIgnoresTransformPartitions() { + Schema schema = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "Dt", Types.StringType.get()), + Types.NestedField.required(3, "ts", Types.TimestampType.withoutZone())); + PartitionSpec specWithTransform = PartitionSpec.builderFor(schema) + .withSpecId(1) + .identity("Dt") + .day("ts") + .build(); + PartitionSpec identityOnlySpec = PartitionSpec.builderFor(schema) + .withSpecId(2) + .identity("id") + .build(); + Map specs = new LinkedHashMap<>(); + specs.put(specWithTransform.specId(), specWithTransform); + specs.put(identityOnlySpec.specId(), identityOnlySpec); + + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.specs()).thenReturn(specs); + + Assert.assertEquals(Arrays.asList("Dt", "id"), IcebergUtils.getIdentityPartitionColumns(table)); + } + + @Test + public void testGetIdentityPartitionInfoMapReturnsIdentityColumnsOnly() { + Schema schema = new Schema( + Types.NestedField.required(1, "Dt", Types.StringType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withoutZone())); + PartitionSpec partitionSpec = PartitionSpec.builderFor(schema) + .identity("Dt") + .day("ts") + .build(); + PartitionData partitionData = new PartitionData(partitionSpec.partitionType()); + partitionData.set(0, "2025-01-01"); + partitionData.set(1, 20000); + + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + + Map partitionInfoMap = IcebergUtils.getIdentityPartitionInfoMap( + partitionData, partitionSpec, table, "UTC"); + Assert.assertEquals(Collections.singletonMap("Dt", "2025-01-01"), partitionInfoMap); + } + + @Test + public void testGetIdentityPartitionInfoMapSupportsFloatingPointPartitions() { + Schema schema = new Schema( + Types.NestedField.required(1, "float_partition", Types.FloatType.get()), + Types.NestedField.required(2, "double_partition", Types.DoubleType.get())); + PartitionSpec partitionSpec = PartitionSpec.builderFor(schema) + .identity("float_partition") + .identity("double_partition") + .build(); + float floatValue = Math.nextUp(0.1F); + double doubleValue = Math.nextUp(0.1D); + PartitionData partitionData = new PartitionData(partitionSpec.partitionType()); + partitionData.set(0, floatValue); + partitionData.set(1, doubleValue); + + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + + Map partitionInfoMap = IcebergUtils.getIdentityPartitionInfoMap( + partitionData, partitionSpec, table, "UTC"); + + String serializedFloat = partitionInfoMap.get("float_partition"); + String serializedDouble = partitionInfoMap.get("double_partition"); + Assert.assertEquals(Float.toString(floatValue), serializedFloat); + Assert.assertEquals(Double.toString(doubleValue), serializedDouble); + Assert.assertEquals(Float.floatToIntBits(floatValue), + Float.floatToIntBits(Float.parseFloat(serializedFloat))); + Assert.assertEquals(Double.doubleToLongBits(doubleValue), + Double.doubleToLongBits(Double.parseDouble(serializedDouble))); + } + @Test public void testGetMatchingManifest() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index f55b4f6f8e8027..46abdf08a587e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -29,6 +29,11 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ScanTaskUtil; import org.junit.Assert; import org.junit.Test; @@ -36,23 +41,64 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; +import java.util.Map; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; private static class TestIcebergScanNode extends IcebergScanNode { + private TableScan tableScan; + TestIcebergScanNode(SessionVariable sv) { super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), sv, ScanContext.EMPTY); } + void setTableScan(TableScan tableScan) { + this.tableScan = tableScan; + } + + @Override + public TableScan createTableScan() { + return tableScan; + } + @Override public boolean isBatchMode() { return false; } } + @Test + public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { + Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Schema currentSchema = new Schema(Types.NestedField.optional("current_string") + .withId(7) + .ofType(Types.StringType.get()) + .withInitialDefault("not-base64") + .build()); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.schemaId()).thenReturn(11); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + TableScan snapshotScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); + Mockito.when(snapshotScan.table()).thenReturn(table); + Mockito.when(snapshotScan.schema()).thenReturn(currentSchema); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + node.setTableScan(snapshotScan); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java index e4146faa690ba9..dda2c3d23447a4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java @@ -53,6 +53,7 @@ import java.util.HashMap; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; public class PaimonMetadataOpsTest { public static String warehouse; @@ -196,6 +197,30 @@ public void testPartition() throws Exception { Assert.assertEquals(1, table.primaryKeys().size()); } + @Test + public void testPartitionPreservesNonLowercaseColumnNames() throws Exception { + String tableName = getTableName(); + Identifier identifier = new Identifier(dbName, tableName); + String sql = "create table " + dbName + "." + tableName + " (" + + "data int, " + + "`PART` int, " + + "`mIxEd_COL` int" + + ") engine = paimon " + + "partition by (`PART`) ()"; + createTable(sql); + Catalog catalog = ops.getCatalog(); + Table table = catalog.getTable(identifier); + + List columnNames = table.rowType().getFields().stream() + .map(DataField::name) + .collect(Collectors.toList()); + + Assert.assertEquals("PART", columnNames.get(1)); + Assert.assertEquals("mIxEd_COL", columnNames.get(2)); + Assert.assertEquals(1, table.partitionKeys().size()); + Assert.assertEquals("PART", table.partitionKeys().get(0)); + } + @Test public void testBucket() throws Exception { String tableName = getTableName(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index ac5ceab80491f0..93cbba81cb7617 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Type; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TFieldPtr; @@ -24,11 +25,13 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryRowWriter; +import org.apache.paimon.data.BinaryString; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.Table; import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; import org.apache.paimon.types.VarCharType; import org.junit.Assert; import org.junit.Test; @@ -83,6 +86,48 @@ public void testGetPartitionInfoMapSupportsFloatingPointPartitions() { Double.doubleToLongBits(Double.parseDouble(serializedDouble))); } + @Test + public void testParseSchemaPreservesNonLowercaseColumnNames() { + RowType rowType = DataTypes.ROW( + DataTypes.FIELD(0, "mIxEd_COL", DataTypes.INT()), + DataTypes.FIELD(1, "PART", DataTypes.STRING())); + + List columns = PaimonUtil.parseSchema(rowType, Collections.singletonList("PART"), false, false); + + Assert.assertEquals("mIxEd_COL", columns.get(0).getName()); + Assert.assertEquals("PART", columns.get(1).getName()); + Assert.assertTrue(columns.get(1).isKey()); + } + + @Test + public void testSystemTableSchemaPreservesNonLowercaseColumnNames() { + RowType rowType = DataTypes.ROW( + DataTypes.FIELD(0, "_ROW_ID", DataTypes.BIGINT()), + DataTypes.FIELD(1, "_SEQUENCE_NUMBER", DataTypes.BIGINT())); + + List columns = PaimonSysExternalTable.buildFullSchema( + rowType.getFields(), false, false); + + Assert.assertEquals("_ROW_ID", columns.get(0).getName()); + Assert.assertEquals("_SEQUENCE_NUMBER", columns.get(1).getName()); + } + + @Test + public void testGetPartitionInfoMapPreservesNonLowercaseKeys() { + DataField mixedCasePartition = DataTypes.FIELD(0, "Dt", DataTypes.STRING()); + Table table = Mockito.mock(Table.class); + Mockito.when(table.name()).thenReturn("mock_table"); + Mockito.when(table.partitionKeys()).thenReturn(Collections.singletonList("Dt")); + Mockito.when(table.rowType()).thenReturn(DataTypes.ROW(mixedCasePartition)); + + BinaryRow partitionValues = BinaryRow.singleColumn(BinaryString.fromString("2026-05-26")); + + Map partitionInfoMap = PaimonUtil.getPartitionInfoMap(table, partitionValues, "UTC"); + + Assert.assertFalse(partitionInfoMap.containsKey("dt")); + Assert.assertEquals("2026-05-26", partitionInfoMap.get("Dt")); + } + @Test public void testBinlogHistorySchemaWithSequenceNumber() { PaimonSysExternalTable binlogTable = Mockito.mock(PaimonSysExternalTable.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index c0da0fadc46d36..d31ecb12ee45f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -42,6 +42,7 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.RawFile; +import org.apache.paimon.types.DataTypes; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -518,6 +519,34 @@ public void testGetBackendPaimonOptionsForJdbcCatalog() throws Exception { Assert.assertEquals(2, backendOptions.size()); } + @Test + public void testGetBackendPaimonOptionsForJniIOManager() { + Map props = new HashMap<>(); + props.put("paimon.doris.enable_jni_io_manager", "true"); + props.put("paimon.doris.jni_io_manager.tmp_dir", "/tmp/doris-paimon"); + props.put("paimon.doris.jni_io_manager.impl_class", "org.example.CustomIOManager"); + + CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); + Mockito.when(catalogProperty.getProperties()).thenReturn(props); + Mockito.when(catalogProperty.getMetastoreProperties()).thenReturn(Mockito.mock(MetastoreProperties.class)); + + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty); + + PaimonSource source = Mockito.mock(PaimonSource.class); + Mockito.when(source.getCatalog()).thenReturn(catalog); + + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + node.setSource(source); + + Map backendOptions = node.getBackendPaimonOptions(); + Assert.assertEquals("true", backendOptions.get("doris.enable_jni_io_manager")); + Assert.assertEquals("/tmp/doris-paimon", backendOptions.get("doris.jni_io_manager.tmp_dir")); + Assert.assertEquals("org.example.CustomIOManager", + backendOptions.get("doris.jni_io_manager.impl_class")); + Assert.assertEquals(3, backendOptions.size()); + } + @Test public void testApplyBackendPaimonOptionsAtScanNodeLevel() throws Exception { PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), @@ -541,9 +570,12 @@ public void testApplyBackendPaimonOptionsAtScanNodeLevel() throws Exception { Assert.assertEquals(backendOptions, node.getFileScanRangeParams().getPaimonOptions()); TFileRangeDesc rangeDesc = new TFileRangeDesc(); + PaimonSplit jniSplit = new PaimonSplit(createDataSplit("scan_level.parquet")); + Assert.assertNotNull(jniSplit.getPartitionValues()); + Assert.assertTrue(jniSplit.getPartitionValues().isEmpty()); invokePrivateMethod(node, "setPaimonParams", new Class[] {TFileRangeDesc.class, PaimonSplit.class}, - rangeDesc, new PaimonSplit(createDataSplit("scan_level.parquet"))); + rangeDesc, jniSplit); Assert.assertFalse(rangeDesc.getTableFormatParams().getPaimonParams().isSetPaimonOptions()); } @@ -568,10 +600,117 @@ public void testSetPartitionValuesBuildsAlignedMetadata() { Assert.assertEquals(Arrays.asList(false, true), rangeDesc.getColumnsFromPathIsNull()); } + @Test + public void testGetPathPartitionKeysReturnsTablePartitionKeys() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + Table table = Mockito.mock(Table.class); + PaimonSysExternalTable sysTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(source.getPaimonTable()).thenReturn(table); + Mockito.when(source.getExternalTable()).thenReturn(sysTable); + Mockito.when(table.partitionKeys()).thenReturn(Arrays.asList("Dt", "Region")); + Mockito.when(sysTable.isDataTable()).thenReturn(true); + node.setSource(source); + + Assert.assertEquals(Arrays.asList("Dt", "Region"), node.getPathPartitionKeys()); + } + + @Test + public void testGetPathPartitionKeysReturnsEmptyForMetadataSystemTable() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonSysExternalTable sysTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(source.getExternalTable()).thenReturn(sysTable); + Mockito.when(sysTable.isDataTable()).thenReturn(false); + node.setSource(source); + + Assert.assertEquals(Collections.emptyList(), node.getPathPartitionKeys()); + } + + @Test + public void testSetPaimonParamsUsesOrderedPartitionKeys() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + Table table = Mockito.mock(Table.class); + PaimonSysExternalTable sysTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(source.getPaimonTable()).thenReturn(table); + Mockito.when(source.getTableLocation()).thenReturn("file:///warehouse"); + Mockito.when(source.getExternalTable()).thenReturn(sysTable); + Mockito.when(sysTable.isDataTable()).thenReturn(true); + Mockito.when(table.partitionKeys()).thenReturn(Arrays.asList("Pt", "Dt")); + node.setSource(source); + + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + rangeDesc.setColumnsFromPathKeys(Collections.singletonList("stale")); + rangeDesc.setColumnsFromPath(Collections.singletonList("old")); + rangeDesc.setColumnsFromPathIsNull(Collections.singletonList(false)); + Map partitionValues = new HashMap<>(); + partitionValues.put("Dt", null); + partitionValues.put("Pt", "p1"); + PaimonSplit split = new PaimonSplit(createDataSplit("ordered.parquet")); + split.setPaimonPartitionValues(partitionValues); + + invokePrivateMethod(node, "setPaimonParams", + new Class[] {TFileRangeDesc.class, PaimonSplit.class}, rangeDesc, split); + + Assert.assertEquals(Arrays.asList("Pt", "Dt"), rangeDesc.getColumnsFromPathKeys()); + Assert.assertEquals(Arrays.asList("p1", ""), rangeDesc.getColumnsFromPath()); + Assert.assertEquals(Arrays.asList(false, true), rangeDesc.getColumnsFromPathIsNull()); + } + + @Test + public void testNativeSplitCarriesPartitionMetadataWithoutRuntimeFilterPruning() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonScanNode spyNode = Mockito.spy(node); + PaimonSource source = Mockito.mock(PaimonSource.class); + Table table = Mockito.mock(Table.class); + PaimonSysExternalTable externalTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(source.getPaimonTable()).thenReturn(table); + Mockito.when(source.getExternalTable()).thenReturn(externalTable); + Mockito.when(table.partitionKeys()).thenReturn(Collections.singletonList("par")); + Mockito.when(table.rowType()).thenReturn(DataTypes.ROW( + DataTypes.FIELD(0, "par", DataTypes.INT()))); + Mockito.when(externalTable.isDataTable()).thenReturn(true); + spyNode.setSource(source); + + Mockito.doReturn(Collections.singletonList(createDataSplit("partitioned.parquet"))) + .when(spyNode).getPaimonSplitFromAPI(); + mockNativeReader(spyNode); + setField(FileQueryScanNode.class, spyNode, "fileSplitter", + new FileSplitter(32L * 1024 * 1024, 64L * 1024 * 1024, 0)); + setField(PaimonScanNode.class, spyNode, "storagePropertiesMap", Collections.emptyMap()); + Mockito.when(sv.isForceJniScanner()).thenReturn(false); + Mockito.when(sv.getIgnoreSplitType()).thenReturn("NONE"); + Mockito.when(sv.getMaxInitialSplitSize()).thenReturn(32L * 1024 * 1024); + Mockito.when(sv.getMaxSplitSize()).thenReturn(64L * 1024 * 1024); + Mockito.when(sv.getTimeZone()).thenReturn("UTC"); + + List splits = spyNode.getSplits(1); + + Assert.assertEquals(1, splits.size()); + PaimonSplit split = (PaimonSplit) splits.get(0); + Assert.assertEquals(Collections.singletonMap("par", "1"), + split.getPaimonPartitionValues()); + Assert.assertEquals(Collections.emptyList(), split.getPartitionValues()); + } + + @Test + public void testGetFieldIndexMatchesMixedCaseColumns() { + List fieldNames = Arrays.asList("data", "mIxEd_COL", "PART"); + + Assert.assertEquals(1, PaimonScanNode.getFieldIndex(fieldNames, "mixed_col")); + Assert.assertEquals(2, PaimonScanNode.getFieldIndex(fieldNames, "part")); + Assert.assertEquals(-1, PaimonScanNode.getFieldIndex(fieldNames, "missing_col")); + } + private void mockJniReader(PaimonScanNode spyNode) { Mockito.doReturn(false).when(spyNode).supportNativeReader(ArgumentMatchers.any(Optional.class)); } + private PaimonScanNode newTestNode(PlanNodeId id, TupleId tupleId, SessionVariable sessionVariable) { + return new PaimonScanNode(id, new TupleDescriptor(tupleId), false, sessionVariable, ScanContext.EMPTY); + } + private void mockNativeReader(PaimonScanNode spyNode) { Mockito.doReturn(true).when(spyNode).supportNativeReader(ArgumentMatchers.any(Optional.class)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java index 370e4965765854..4d140b2ba57037 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java @@ -47,6 +47,7 @@ public void testAnalyzeFileFormatProperties() { Assert.assertEquals(TParquetCompressionType.SNAPPY, parquetFileFormatProperties.getParquetCompressionType()); Assert.assertEquals(false, parquetFileFormatProperties.isParquetDisableDictionary()); + Assert.assertTrue(parquetFileFormatProperties.isEnableInt96Timestamps()); } @Test @@ -139,6 +140,7 @@ public void testFullTResultFileSinkOptions() { parquetFileFormatProperties.fullTResultFileSinkOptions(sinkOptions); Assert.assertEquals(parquetFileFormatProperties.getParquetCompressionType(), sinkOptions.getParquetCompressionType()); Assert.assertEquals(parquetFileFormatProperties.isParquetDisableDictionary(), sinkOptions.isParquetDisableDictionary()); + Assert.assertEquals(parquetFileFormatProperties.isEnableInt96Timestamps(), sinkOptions.isEnableInt96Timestamps()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java index 00588188241897..b3f632b0fee22d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java @@ -182,6 +182,16 @@ public void testRuntimeFilterBroadcastJoinProducerNumDescription() throws Except }, varAttr.description()); } + @Test + public void testExternalTableBatchModeDefaultsAndFuzzyAttribute() throws Exception { + SessionVariable sessionVar = new SessionVariable(); + Assertions.assertTrue(sessionVar.getEnableExternalTableBatchMode()); + + Field field = SessionVariable.class.getDeclaredField("enableExternalTableBatchMode"); + VariableMgr.VarAttr varAttr = field.getAnnotation(VariableMgr.VarAttr.class); + Assertions.assertTrue(varAttr.fuzzy()); + } + @Test public void testSetVarInHint() { String sql = "insert into test_t1 select /*+ set_var(enable_nereids_dml_with_pipeline=false)*/ * from test_t1 where enable_nereids_dml_with_pipeline=true"; diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift index c17199d74edf91..a17cd140c93418 100644 --- a/gensrc/thrift/Exprs.thrift +++ b/gensrc/thrift/Exprs.thrift @@ -88,6 +88,10 @@ enum TExprNodeType { TRY_CAST_EXPR = 41 // for search DSL function SEARCH_EXPR = 42, + // Normal predicate expression + PREDICATE = 43, + // Normal literal + LITERAL = 44, } //enum TAggregationOp { diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 14972f8d434784..46ae54781ae825 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -49,7 +49,16 @@ struct TField { 3: optional string name, // Field name 4: optional Types.TColumnType type, // Corresponding Doris column type 5: optional TNestedField nestedField // Nested field definition (for array, struct, or map types) - 6: optional list name_mapping // iceberg : schema.name-mapping.default, for missing column id. + 6: optional list name_mapping, // iceberg : schema.name-mapping.default, for missing column id. + // Iceberg initial default normalized for transport to BE. Binary-like Iceberg values use + // Base64 because Thrift's Java string carrier cannot preserve arbitrary bytes; other primitive + // values use Doris' FE string representation. An old data file that predates this field + // logically contains this value rather than NULL. + 7: optional string initial_default_value, + // True when initial_default_value is Base64 and must be decoded before constructing the Doris + // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg + // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. + 8: optional bool initial_default_value_is_base64 } diff --git a/gensrc/thrift/Opcodes.thrift b/gensrc/thrift/Opcodes.thrift index 1e4002357e7599..a2d709799482eb 100644 --- a/gensrc/thrift/Opcodes.thrift +++ b/gensrc/thrift/Opcodes.thrift @@ -97,4 +97,6 @@ enum TExprOpcode { MATCH_REGEXP = 76, MATCH_PHRASE_EDGE = 77, TRY_CAST = 78, + // Delete operator from Iceberg/Paimon + DELETE = 79, } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 7723f919c8f853..bffd2b546c978c 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -505,6 +505,7 @@ struct TQueryOptions { // In read path, read from file cache or remote storage when execute query. 1000: optional bool disable_file_cache = false 1001: optional i32 file_cache_query_limit_percent = -1 + 1002: optional bool enable_file_scanner_v2 = false } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index c1d7e5cb0b5e65..6f0a5b96845806 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -327,6 +327,12 @@ struct TPaimonDeletionFileDesc { 3: optional i64 length; } +enum TPaimonReaderType { + PAIMON_NATIVE = 0, + PAIMON_JNI = 1, + PAIMON_CPP = 2, +} + struct TPaimonFileDesc { 1: optional string paimon_split 2: optional string paimon_column_names @@ -344,6 +350,8 @@ struct TPaimonFileDesc { 14: optional string paimon_table // deprecated 15: optional i64 row_count // deprecated 16: optional i64 schema_id; // for schema change. + // Reader implementation for logical paimon split. Native file split uses range format type. + 17: optional TPaimonReaderType reader_type; } struct TTrinoConnectorFileDesc { diff --git a/regression-test/data/export_p0/export/test_show_export.out b/regression-test/data/export_p0/export/test_show_export.out index 90277ca28f2a9f..eb2d2ab154b1b3 100644 --- a/regression-test/data/export_p0/export/test_show_export.out +++ b/regression-test/data/export_p0/export/test_show_export.out @@ -102,156 +102,156 @@ 99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99 99.99 99.99 char99 99 -- !select_load1 -- -1 2017-10-01 2017-10-01T00:00 Beijing 1 1 true 1 1 1.1 1.1 char1 1 1 -10 2017-10-01 2017-10-01T00:00 Beijing 10 10 true 10 10 10.1 10.1 char10 10 10 +1 2017-10-01 2017-10-01T00:00 Beijing 1 1 true 1 1 1.1 1.1 char1 1.000000000 1 +10 2017-10-01 2017-10-01T00:00 Beijing 10 10 true 10 10 10.1 10.1 char10 10.000000000 10 100 2017-10-01 2017-10-01T00:00 \N \N \N \N \N \N \N \N \N \N \N -11 2017-10-01 2017-10-01T00:00 Beijing 11 11 true 11 11 11.11 11.11 char11 11 11 -12 2017-10-01 2017-10-01T00:00 Beijing 12 12 true 12 12 12.12 12.12 char12 12 12 -13 2017-10-01 2017-10-01T00:00 Beijing 13 13 true 13 13 13.13 13.13 char13 13 13 -14 2017-10-01 2017-10-01T00:00 Beijing 14 14 true 14 14 14.14 14.14 char14 14 14 -15 2017-10-01 2017-10-01T00:00 Beijing 15 15 true 15 15 15.15 15.15 char15 15 15 -16 2017-10-01 2017-10-01T00:00 Beijing 16 16 true 16 16 16.16 16.16 char16 16 16 -17 2017-10-01 2017-10-01T00:00 Beijing 17 17 true 17 17 17.17 17.17 char17 17 17 -18 2017-10-01 2017-10-01T00:00 Beijing 18 18 true 18 18 18.18 18.18 char18 18 18 -19 2017-10-01 2017-10-01T00:00 Beijing 19 19 true 19 19 19.19 19.19 char19 19 19 -2 2017-10-01 2017-10-01T00:00 Beijing 2 2 true 2 2 2.2 2.2 char2 2 2 -20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20.2 20.2 char20 20 20 -21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21.21 21.21 char21 21 21 -22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22.22 22.22 char22 22 22 -23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23.23 23.23 char23 23 23 -24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24.24 24.24 char24 24 24 -25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25.25 25.25 char25 25 25 -26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26.26 26.26 char26 26 26 -27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27.27 27.27 char27 27 27 -28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28.28 28.28 char28 28 28 -29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29.29 29.29 char29 29 29 -3 2017-10-01 2017-10-01T00:00 Beijing 3 3 true 3 3 3.3 3.3 char3 3 3 -30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30.3 30.3 char30 30 30 -31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31.31 31.31 char31 31 31 -32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32.32 32.32 char32 32 32 -33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33.33 33.33 char33 33 33 -34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34.34 34.34 char34 34 34 -35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35.35 35.35 char35 35 35 -36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36.36 36.36 char36 36 36 -37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37.37 37.37 char37 37 37 -38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38.38 38.38 char38 38 38 -39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39.39 39.39 char39 39 39 -4 2017-10-01 2017-10-01T00:00 Beijing 4 4 true 4 4 4.4 4.4 char4 4 4 -40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40.4 40.4 char40 40 40 -41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41.41 41.41 char41 41 41 -42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42.42 42.42 char42 42 42 -43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43.43 43.43 char43 43 43 -44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44.44 44.44 char44 44 44 -45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45.45 45.45 char45 45 45 -46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46.46 46.46 char46 46 46 -47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47.47 47.47 char47 47 47 -48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48.48 48.48 char48 48 48 -49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49.49 49.49 char49 49 49 -5 2017-10-01 2017-10-01T00:00 Beijing 5 5 true 5 5 5.5 5.5 char5 5 5 -50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50.5 50.5 char50 50 50 -51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51.51 51.51 char51 51 51 -52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52.52 52.52 char52 52 52 -53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53.53 53.53 char53 53 53 -54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54.54 54.54 char54 54 54 -55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55.55 55.55 char55 55 55 -56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56.56 56.56 char56 56 56 -57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57.57 57.57 char57 57 57 -58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58.58 58.58 char58 58 58 -59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59.59 59.59 char59 59 59 -6 2017-10-01 2017-10-01T00:00 Beijing 6 6 true 6 6 6.6 6.6 char6 6 6 -60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60.6 60.6 char60 60 60 -61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61.61 61.61 char61 61 61 -62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62.62 62.62 char62 62 62 -63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63.63 63.63 char63 63 63 -64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64.64 64.64 char64 64 64 -65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65.65 65.65 char65 65 65 -66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66.66 66.66 char66 66 66 -67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67.67 67.67 char67 67 67 -68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68.68 68.68 char68 68 68 -69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69.69 69.69 char69 69 69 -7 2017-10-01 2017-10-01T00:00 Beijing 7 7 true 7 7 7.7 7.7 char7 7 7 -70 2017-10-01 2017-10-01T00:00 Beijing 70 70 true 70 70 70.7 70.7 char70 70 70 -71 2017-10-01 2017-10-01T00:00 Beijing 71 71 true 71 71 71.71 71.71 char71 71 71 -72 2017-10-01 2017-10-01T00:00 Beijing 72 72 true 72 72 72.72 72.72 char72 72 72 -73 2017-10-01 2017-10-01T00:00 Beijing 73 73 true 73 73 73.73 73.73 char73 73 73 -74 2017-10-01 2017-10-01T00:00 Beijing 74 74 true 74 74 74.74 74.74 char74 74 74 -75 2017-10-01 2017-10-01T00:00 Beijing 75 75 true 75 75 75.75 75.75 char75 75 75 -76 2017-10-01 2017-10-01T00:00 Beijing 76 76 true 76 76 76.76 76.76 char76 76 76 -77 2017-10-01 2017-10-01T00:00 Beijing 77 77 true 77 77 77.77 77.77 char77 77 77 -78 2017-10-01 2017-10-01T00:00 Beijing 78 78 true 78 78 78.78 78.78 char78 78 78 -79 2017-10-01 2017-10-01T00:00 Beijing 79 79 true 79 79 79.79 79.79 char79 79 79 -8 2017-10-01 2017-10-01T00:00 Beijing 8 8 true 8 8 8.8 8.8 char8 8 8 -80 2017-10-01 2017-10-01T00:00 Beijing 80 80 true 80 80 80.8 80.8 char80 80 80 -81 2017-10-01 2017-10-01T00:00 Beijing 81 81 true 81 81 81.81 81.81 char81 81 81 -82 2017-10-01 2017-10-01T00:00 Beijing 82 82 true 82 82 82.82 82.82 char82 82 82 -83 2017-10-01 2017-10-01T00:00 Beijing 83 83 true 83 83 83.83 83.83 char83 83 83 -84 2017-10-01 2017-10-01T00:00 Beijing 84 84 true 84 84 84.84 84.84 char84 84 84 -85 2017-10-01 2017-10-01T00:00 Beijing 85 85 true 85 85 85.85 85.85 char85 85 85 -86 2017-10-01 2017-10-01T00:00 Beijing 86 86 true 86 86 86.86 86.86 char86 86 86 -87 2017-10-01 2017-10-01T00:00 Beijing 87 87 true 87 87 87.87 87.87 char87 87 87 -88 2017-10-01 2017-10-01T00:00 Beijing 88 88 true 88 88 88.88 88.88 char88 88 88 -89 2017-10-01 2017-10-01T00:00 Beijing 89 89 true 89 89 89.89 89.89 char89 89 89 -9 2017-10-01 2017-10-01T00:00 Beijing 9 9 true 9 9 9.9 9.9 char9 9 9 -90 2017-10-01 2017-10-01T00:00 Beijing 90 90 true 90 90 90.9 90.9 char90 90 90 -91 2017-10-01 2017-10-01T00:00 Beijing 91 91 true 91 91 91.91 91.91 char91 91 91 -92 2017-10-01 2017-10-01T00:00 Beijing 92 92 true 92 92 92.92 92.92 char92 92 92 -93 2017-10-01 2017-10-01T00:00 Beijing 93 93 true 93 93 93.93 93.93 char93 93 93 -94 2017-10-01 2017-10-01T00:00 Beijing 94 94 true 94 94 94.94 94.94 char94 94 94 -95 2017-10-01 2017-10-01T00:00 Beijing 95 95 true 95 95 95.95 95.95 char95 95 95 -96 2017-10-01 2017-10-01T00:00 Beijing 96 96 true 96 96 96.96 96.96 char96 96 96 -97 2017-10-01 2017-10-01T00:00 Beijing 97 97 true 97 97 97.97 97.97 char97 97 97 -98 2017-10-01 2017-10-01T00:00 Beijing 98 98 true 98 98 98.98 98.98 char98 98 98 -99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99.99 99.99 char99 99 99 +11 2017-10-01 2017-10-01T00:00 Beijing 11 11 true 11 11 11.11 11.11 char11 11.000000000 11 +12 2017-10-01 2017-10-01T00:00 Beijing 12 12 true 12 12 12.12 12.12 char12 12.000000000 12 +13 2017-10-01 2017-10-01T00:00 Beijing 13 13 true 13 13 13.13 13.13 char13 13.000000000 13 +14 2017-10-01 2017-10-01T00:00 Beijing 14 14 true 14 14 14.14 14.14 char14 14.000000000 14 +15 2017-10-01 2017-10-01T00:00 Beijing 15 15 true 15 15 15.15 15.15 char15 15.000000000 15 +16 2017-10-01 2017-10-01T00:00 Beijing 16 16 true 16 16 16.16 16.16 char16 16.000000000 16 +17 2017-10-01 2017-10-01T00:00 Beijing 17 17 true 17 17 17.17 17.17 char17 17.000000000 17 +18 2017-10-01 2017-10-01T00:00 Beijing 18 18 true 18 18 18.18 18.18 char18 18.000000000 18 +19 2017-10-01 2017-10-01T00:00 Beijing 19 19 true 19 19 19.19 19.19 char19 19.000000000 19 +2 2017-10-01 2017-10-01T00:00 Beijing 2 2 true 2 2 2.2 2.2 char2 2.000000000 2 +20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20.2 20.2 char20 20.000000000 20 +21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21.21 21.21 char21 21.000000000 21 +22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22.22 22.22 char22 22.000000000 22 +23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23.23 23.23 char23 23.000000000 23 +24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24.24 24.24 char24 24.000000000 24 +25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25.25 25.25 char25 25.000000000 25 +26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26.26 26.26 char26 26.000000000 26 +27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27.27 27.27 char27 27.000000000 27 +28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28.28 28.28 char28 28.000000000 28 +29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29.29 29.29 char29 29.000000000 29 +3 2017-10-01 2017-10-01T00:00 Beijing 3 3 true 3 3 3.3 3.3 char3 3.000000000 3 +30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30.3 30.3 char30 30.000000000 30 +31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31.31 31.31 char31 31.000000000 31 +32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32.32 32.32 char32 32.000000000 32 +33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33.33 33.33 char33 33.000000000 33 +34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34.34 34.34 char34 34.000000000 34 +35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35.35 35.35 char35 35.000000000 35 +36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36.36 36.36 char36 36.000000000 36 +37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37.37 37.37 char37 37.000000000 37 +38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38.38 38.38 char38 38.000000000 38 +39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39.39 39.39 char39 39.000000000 39 +4 2017-10-01 2017-10-01T00:00 Beijing 4 4 true 4 4 4.4 4.4 char4 4.000000000 4 +40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40.4 40.4 char40 40.000000000 40 +41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41.41 41.41 char41 41.000000000 41 +42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42.42 42.42 char42 42.000000000 42 +43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43.43 43.43 char43 43.000000000 43 +44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44.44 44.44 char44 44.000000000 44 +45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45.45 45.45 char45 45.000000000 45 +46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46.46 46.46 char46 46.000000000 46 +47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47.47 47.47 char47 47.000000000 47 +48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48.48 48.48 char48 48.000000000 48 +49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49.49 49.49 char49 49.000000000 49 +5 2017-10-01 2017-10-01T00:00 Beijing 5 5 true 5 5 5.5 5.5 char5 5.000000000 5 +50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50.5 50.5 char50 50.000000000 50 +51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51.51 51.51 char51 51.000000000 51 +52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52.52 52.52 char52 52.000000000 52 +53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53.53 53.53 char53 53.000000000 53 +54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54.54 54.54 char54 54.000000000 54 +55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55.55 55.55 char55 55.000000000 55 +56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56.56 56.56 char56 56.000000000 56 +57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57.57 57.57 char57 57.000000000 57 +58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58.58 58.58 char58 58.000000000 58 +59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59.59 59.59 char59 59.000000000 59 +6 2017-10-01 2017-10-01T00:00 Beijing 6 6 true 6 6 6.6 6.6 char6 6.000000000 6 +60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60.6 60.6 char60 60.000000000 60 +61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61.61 61.61 char61 61.000000000 61 +62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62.62 62.62 char62 62.000000000 62 +63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63.63 63.63 char63 63.000000000 63 +64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64.64 64.64 char64 64.000000000 64 +65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65.65 65.65 char65 65.000000000 65 +66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66.66 66.66 char66 66.000000000 66 +67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67.67 67.67 char67 67.000000000 67 +68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68.68 68.68 char68 68.000000000 68 +69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69.69 69.69 char69 69.000000000 69 +7 2017-10-01 2017-10-01T00:00 Beijing 7 7 true 7 7 7.7 7.7 char7 7.000000000 7 +70 2017-10-01 2017-10-01T00:00 Beijing 70 70 true 70 70 70.7 70.7 char70 70.000000000 70 +71 2017-10-01 2017-10-01T00:00 Beijing 71 71 true 71 71 71.71 71.71 char71 71.000000000 71 +72 2017-10-01 2017-10-01T00:00 Beijing 72 72 true 72 72 72.72 72.72 char72 72.000000000 72 +73 2017-10-01 2017-10-01T00:00 Beijing 73 73 true 73 73 73.73 73.73 char73 73.000000000 73 +74 2017-10-01 2017-10-01T00:00 Beijing 74 74 true 74 74 74.74 74.74 char74 74.000000000 74 +75 2017-10-01 2017-10-01T00:00 Beijing 75 75 true 75 75 75.75 75.75 char75 75.000000000 75 +76 2017-10-01 2017-10-01T00:00 Beijing 76 76 true 76 76 76.76 76.76 char76 76.000000000 76 +77 2017-10-01 2017-10-01T00:00 Beijing 77 77 true 77 77 77.77 77.77 char77 77.000000000 77 +78 2017-10-01 2017-10-01T00:00 Beijing 78 78 true 78 78 78.78 78.78 char78 78.000000000 78 +79 2017-10-01 2017-10-01T00:00 Beijing 79 79 true 79 79 79.79 79.79 char79 79.000000000 79 +8 2017-10-01 2017-10-01T00:00 Beijing 8 8 true 8 8 8.8 8.8 char8 8.000000000 8 +80 2017-10-01 2017-10-01T00:00 Beijing 80 80 true 80 80 80.8 80.8 char80 80.000000000 80 +81 2017-10-01 2017-10-01T00:00 Beijing 81 81 true 81 81 81.81 81.81 char81 81.000000000 81 +82 2017-10-01 2017-10-01T00:00 Beijing 82 82 true 82 82 82.82 82.82 char82 82.000000000 82 +83 2017-10-01 2017-10-01T00:00 Beijing 83 83 true 83 83 83.83 83.83 char83 83.000000000 83 +84 2017-10-01 2017-10-01T00:00 Beijing 84 84 true 84 84 84.84 84.84 char84 84.000000000 84 +85 2017-10-01 2017-10-01T00:00 Beijing 85 85 true 85 85 85.85 85.85 char85 85.000000000 85 +86 2017-10-01 2017-10-01T00:00 Beijing 86 86 true 86 86 86.86 86.86 char86 86.000000000 86 +87 2017-10-01 2017-10-01T00:00 Beijing 87 87 true 87 87 87.87 87.87 char87 87.000000000 87 +88 2017-10-01 2017-10-01T00:00 Beijing 88 88 true 88 88 88.88 88.88 char88 88.000000000 88 +89 2017-10-01 2017-10-01T00:00 Beijing 89 89 true 89 89 89.89 89.89 char89 89.000000000 89 +9 2017-10-01 2017-10-01T00:00 Beijing 9 9 true 9 9 9.9 9.9 char9 9.000000000 9 +90 2017-10-01 2017-10-01T00:00 Beijing 90 90 true 90 90 90.9 90.9 char90 90.000000000 90 +91 2017-10-01 2017-10-01T00:00 Beijing 91 91 true 91 91 91.91 91.91 char91 91.000000000 91 +92 2017-10-01 2017-10-01T00:00 Beijing 92 92 true 92 92 92.92 92.92 char92 92.000000000 92 +93 2017-10-01 2017-10-01T00:00 Beijing 93 93 true 93 93 93.93 93.93 char93 93.000000000 93 +94 2017-10-01 2017-10-01T00:00 Beijing 94 94 true 94 94 94.94 94.94 char94 94.000000000 94 +95 2017-10-01 2017-10-01T00:00 Beijing 95 95 true 95 95 95.95 95.95 char95 95.000000000 95 +96 2017-10-01 2017-10-01T00:00 Beijing 96 96 true 96 96 96.96 96.96 char96 96.000000000 96 +97 2017-10-01 2017-10-01T00:00 Beijing 97 97 true 97 97 97.97 97.97 char97 97.000000000 97 +98 2017-10-01 2017-10-01T00:00 Beijing 98 98 true 98 98 98.98 98.98 char98 98.000000000 98 +99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99.99 99.99 char99 99.000000000 99 -- !select_load1 -- -20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20.2 20.2 char20 20 20 -21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21.21 21.21 char21 21 21 -22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22.22 22.22 char22 22 22 -23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23.23 23.23 char23 23 23 -24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24.24 24.24 char24 24 24 -25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25.25 25.25 char25 25 25 -26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26.26 26.26 char26 26 26 -27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27.27 27.27 char27 27 27 -28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28.28 28.28 char28 28 28 -29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29.29 29.29 char29 29 29 -30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30.3 30.3 char30 30 30 -31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31.31 31.31 char31 31 31 -32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32.32 32.32 char32 32 32 -33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33.33 33.33 char33 33 33 -34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34.34 34.34 char34 34 34 -35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35.35 35.35 char35 35 35 -36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36.36 36.36 char36 36 36 -37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37.37 37.37 char37 37 37 -38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38.38 38.38 char38 38 38 -39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39.39 39.39 char39 39 39 -40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40.4 40.4 char40 40 40 -41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41.41 41.41 char41 41 41 -42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42.42 42.42 char42 42 42 -43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43.43 43.43 char43 43 43 -44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44.44 44.44 char44 44 44 -45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45.45 45.45 char45 45 45 -46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46.46 46.46 char46 46 46 -47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47.47 47.47 char47 47 47 -48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48.48 48.48 char48 48 48 -49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49.49 49.49 char49 49 49 -50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50.5 50.5 char50 50 50 -51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51.51 51.51 char51 51 51 -52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52.52 52.52 char52 52 52 -53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53.53 53.53 char53 53 53 -54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54.54 54.54 char54 54 54 -55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55.55 55.55 char55 55 55 -56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56.56 56.56 char56 56 56 -57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57.57 57.57 char57 57 57 -58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58.58 58.58 char58 58 58 -59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59.59 59.59 char59 59 59 -60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60.6 60.6 char60 60 60 -61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61.61 61.61 char61 61 61 -62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62.62 62.62 char62 62 62 -63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63.63 63.63 char63 63 63 -64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64.64 64.64 char64 64 64 -65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65.65 65.65 char65 65 65 -66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66.66 66.66 char66 66 66 -67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67.67 67.67 char67 67 67 -68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68.68 68.68 char68 68 68 -69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69.69 69.69 char69 69 69 +20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20.2 20.2 char20 20.000000000 20 +21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21.21 21.21 char21 21.000000000 21 +22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22.22 22.22 char22 22.000000000 22 +23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23.23 23.23 char23 23.000000000 23 +24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24.24 24.24 char24 24.000000000 24 +25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25.25 25.25 char25 25.000000000 25 +26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26.26 26.26 char26 26.000000000 26 +27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27.27 27.27 char27 27.000000000 27 +28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28.28 28.28 char28 28.000000000 28 +29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29.29 29.29 char29 29.000000000 29 +30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30.3 30.3 char30 30.000000000 30 +31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31.31 31.31 char31 31.000000000 31 +32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32.32 32.32 char32 32.000000000 32 +33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33.33 33.33 char33 33.000000000 33 +34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34.34 34.34 char34 34.000000000 34 +35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35.35 35.35 char35 35.000000000 35 +36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36.36 36.36 char36 36.000000000 36 +37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37.37 37.37 char37 37.000000000 37 +38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38.38 38.38 char38 38.000000000 38 +39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39.39 39.39 char39 39.000000000 39 +40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40.4 40.4 char40 40.000000000 40 +41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41.41 41.41 char41 41.000000000 41 +42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42.42 42.42 char42 42.000000000 42 +43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43.43 43.43 char43 43.000000000 43 +44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44.44 44.44 char44 44.000000000 44 +45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45.45 45.45 char45 45.000000000 45 +46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46.46 46.46 char46 46.000000000 46 +47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47.47 47.47 char47 47.000000000 47 +48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48.48 48.48 char48 48.000000000 48 +49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49.49 49.49 char49 49.000000000 49 +50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50.5 50.5 char50 50.000000000 50 +51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51.51 51.51 char51 51.000000000 51 +52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52.52 52.52 char52 52.000000000 52 +53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53.53 53.53 char53 53.000000000 53 +54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54.54 54.54 char54 54.000000000 54 +55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55.55 55.55 char55 55.000000000 55 +56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56.56 56.56 char56 56.000000000 56 +57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57.57 57.57 char57 57.000000000 57 +58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58.58 58.58 char58 58.000000000 58 +59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59.59 59.59 char59 59.000000000 59 +60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60.6 60.6 char60 60.000000000 60 +61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61.61 61.61 char61 61.000000000 61 +62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62.62 62.62 char62 62.000000000 62 +63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63.63 63.63 char63 63.000000000 63 +64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64.64 64.64 char64 64.000000000 64 +65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65.65 65.65 char65 65.000000000 65 +66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66.66 66.66 char66 66.000000000 66 +67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67.67 67.67 char67 67.000000000 67 +68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68.68 68.68 char68 68.000000000 68 +69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69.69 69.69 char69 69.000000000 69 diff --git a/regression-test/data/export_p0/outfile/parquet/test_outfile_parquet_complex_type.out b/regression-test/data/export_p0/outfile/parquet/test_outfile_parquet_complex_type.out index c8ff8cafdd9854..cd7fe1e40fdb2d 100644 --- a/regression-test/data/export_p0/outfile/parquet/test_outfile_parquet_complex_type.out +++ b/regression-test/data/export_p0/outfile/parquet/test_outfile_parquet_complex_type.out @@ -127,3 +127,15 @@ 9 doris_9 {"user_id":9, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":9, "sex":9, "bool_col":1, "int_col":9, "bigint_col":9, "largeint_col":"9", "float_col":9.9, "double_col":9.9, "char_col":"char9_1234", "decimal_col":9.000000000} 10 doris_10 {"user_id":10, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":null, "age":null, "sex":null, "bool_col":null, "int_col":null, "bigint_col":null, "largeint_col":null, "float_col":null, "double_col":null, "char_col":null, "decimal_col":null} +-- !select_load7 -- +1 doris_1 {"user_id":1, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":1, "sex":1, "bool_col":1, "int_col":1, "bigint_col":1, "largeint_col":"1", "float_col":1.1, "double_col":1.1, "char_col":"char1_1234", "decimal_col":1.000000000} +2 doris_2 {"user_id":2, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":2, "sex":2, "bool_col":1, "int_col":2, "bigint_col":2, "largeint_col":"2", "float_col":2.2, "double_col":2.2, "char_col":"char2_1234", "decimal_col":2.000000000} +3 doris_3 {"user_id":3, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":3, "sex":3, "bool_col":1, "int_col":3, "bigint_col":3, "largeint_col":"3", "float_col":3.3, "double_col":3.3, "char_col":"char3_1234", "decimal_col":3.000000000} +4 doris_4 {"user_id":4, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":4, "sex":4, "bool_col":1, "int_col":4, "bigint_col":4, "largeint_col":"4", "float_col":4.4, "double_col":4.4, "char_col":"char4_1234", "decimal_col":4.000000000} +5 doris_5 {"user_id":5, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":5, "sex":5, "bool_col":1, "int_col":5, "bigint_col":5, "largeint_col":"5", "float_col":5.5, "double_col":5.5, "char_col":"char5_1234", "decimal_col":5.000000000} +6 doris_6 {"user_id":6, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":6, "sex":6, "bool_col":1, "int_col":6, "bigint_col":6, "largeint_col":"6", "float_col":6.6, "double_col":6.6, "char_col":"char6_1234", "decimal_col":6.000000000} +7 doris_7 {"user_id":7, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":7, "sex":7, "bool_col":1, "int_col":7, "bigint_col":7, "largeint_col":"7", "float_col":7.7, "double_col":7.7, "char_col":"char7_1234", "decimal_col":7.000000000} +8 doris_8 {"user_id":8, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":8, "sex":8, "bool_col":1, "int_col":8, "bigint_col":8, "largeint_col":"8", "float_col":8.8, "double_col":8.800000000000001, "char_col":"char8_1234", "decimal_col":8.000000000} +9 doris_9 {"user_id":9, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":"Beijing", "age":9, "sex":9, "bool_col":1, "int_col":9, "bigint_col":9, "largeint_col":"9", "float_col":9.9, "double_col":9.9, "char_col":"char9_1234", "decimal_col":9.000000000} +10 doris_10 {"user_id":10, "date":"2017-10-01", "datetime":"2017-10-01 00:00:00.000000", "city":null, "age":null, "sex":null, "bool_col":null, "int_col":null, "bigint_col":null, "largeint_col":null, "float_col":null, "double_col":null, "char_col":null, "decimal_col":null} + diff --git a/regression-test/data/export_p0/test_export_parquet.out b/regression-test/data/export_p0/test_export_parquet.out index c3358efa4a97af..941dd4469a66c8 100644 --- a/regression-test/data/export_p0/test_export_parquet.out +++ b/regression-test/data/export_p0/test_export_parquet.out @@ -102,104 +102,104 @@ 99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99 99.99 99.99 char99 99 0.0.0.99 ::99 -- !select_load1 -- -1 2017-10-01 2017-10-01T00:00 Beijing 1 1 true 1 1 1.1 1.1 char1 1 1 1 ::1 -10 2017-10-01 2017-10-01T00:00 Beijing 10 10 true 10 10 10.1 10.1 char10 10 10 10 ::10 +1 2017-10-01 2017-10-01T00:00 Beijing 1 1 true 1 1 1.1 1.1 char1 1.000000000 1 1 ::1 +10 2017-10-01 2017-10-01T00:00 Beijing 10 10 true 10 10 10.1 10.1 char10 10.000000000 10 10 ::10 100 2017-10-01 2017-10-01T00:00 \N \N \N \N \N \N \N \N \N \N \N \N \N -11 2017-10-01 2017-10-01T00:00 Beijing 11 11 true 11 11 11.11 11.11 char11 11 11 11 ::11 -12 2017-10-01 2017-10-01T00:00 Beijing 12 12 true 12 12 12.12 12.12 char12 12 12 12 ::12 -13 2017-10-01 2017-10-01T00:00 Beijing 13 13 true 13 13 13.13 13.13 char13 13 13 13 ::13 -14 2017-10-01 2017-10-01T00:00 Beijing 14 14 true 14 14 14.14 14.14 char14 14 14 14 ::14 -15 2017-10-01 2017-10-01T00:00 Beijing 15 15 true 15 15 15.15 15.15 char15 15 15 15 ::15 -16 2017-10-01 2017-10-01T00:00 Beijing 16 16 true 16 16 16.16 16.16 char16 16 16 16 ::16 -17 2017-10-01 2017-10-01T00:00 Beijing 17 17 true 17 17 17.17 17.17 char17 17 17 17 ::17 -18 2017-10-01 2017-10-01T00:00 Beijing 18 18 true 18 18 18.18 18.18 char18 18 18 18 ::18 -19 2017-10-01 2017-10-01T00:00 Beijing 19 19 true 19 19 19.19 19.19 char19 19 19 19 ::19 -2 2017-10-01 2017-10-01T00:00 Beijing 2 2 true 2 2 2.2 2.2 char2 2 2 2 ::2 -20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20.2 20.2 char20 20 20 20 ::20 -21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21.21 21.21 char21 21 21 21 ::21 -22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22.22 22.22 char22 22 22 22 ::22 -23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23.23 23.23 char23 23 23 23 ::23 -24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24.24 24.24 char24 24 24 24 ::24 -25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25.25 25.25 char25 25 25 25 ::25 -26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26.26 26.26 char26 26 26 26 ::26 -27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27.27 27.27 char27 27 27 27 ::27 -28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28.28 28.28 char28 28 28 28 ::28 -29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29.29 29.29 char29 29 29 29 ::29 -3 2017-10-01 2017-10-01T00:00 Beijing 3 3 true 3 3 3.3 3.3 char3 3 3 3 ::3 -30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30.3 30.3 char30 30 30 30 ::30 -31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31.31 31.31 char31 31 31 31 ::31 -32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32.32 32.32 char32 32 32 32 ::32 -33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33.33 33.33 char33 33 33 33 ::33 -34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34.34 34.34 char34 34 34 34 ::34 -35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35.35 35.35 char35 35 35 35 ::35 -36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36.36 36.36 char36 36 36 36 ::36 -37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37.37 37.37 char37 37 37 37 ::37 -38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38.38 38.38 char38 38 38 38 ::38 -39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39.39 39.39 char39 39 39 39 ::39 -4 2017-10-01 2017-10-01T00:00 Beijing 4 4 true 4 4 4.4 4.4 char4 4 4 4 ::4 -40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40.4 40.4 char40 40 40 40 ::40 -41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41.41 41.41 char41 41 41 41 ::41 -42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42.42 42.42 char42 42 42 42 ::42 -43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43.43 43.43 char43 43 43 43 ::43 -44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44.44 44.44 char44 44 44 44 ::44 -45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45.45 45.45 char45 45 45 45 ::45 -46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46.46 46.46 char46 46 46 46 ::46 -47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47.47 47.47 char47 47 47 47 ::47 -48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48.48 48.48 char48 48 48 48 ::48 -49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49.49 49.49 char49 49 49 49 ::49 -5 2017-10-01 2017-10-01T00:00 Beijing 5 5 true 5 5 5.5 5.5 char5 5 5 5 ::5 -50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50.5 50.5 char50 50 50 50 ::50 -51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51.51 51.51 char51 51 51 51 ::51 -52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52.52 52.52 char52 52 52 52 ::52 -53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53.53 53.53 char53 53 53 53 ::53 -54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54.54 54.54 char54 54 54 54 ::54 -55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55.55 55.55 char55 55 55 55 ::55 -56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56.56 56.56 char56 56 56 56 ::56 -57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57.57 57.57 char57 57 57 57 ::57 -58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58.58 58.58 char58 58 58 58 ::58 -59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59.59 59.59 char59 59 59 59 ::59 -6 2017-10-01 2017-10-01T00:00 Beijing 6 6 true 6 6 6.6 6.6 char6 6 6 6 ::6 -60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60.6 60.6 char60 60 60 60 ::60 -61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61.61 61.61 char61 61 61 61 ::61 -62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62.62 62.62 char62 62 62 62 ::62 -63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63.63 63.63 char63 63 63 63 ::63 -64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64.64 64.64 char64 64 64 64 ::64 -65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65.65 65.65 char65 65 65 65 ::65 -66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66.66 66.66 char66 66 66 66 ::66 -67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67.67 67.67 char67 67 67 67 ::67 -68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68.68 68.68 char68 68 68 68 ::68 -69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69.69 69.69 char69 69 69 69 ::69 -7 2017-10-01 2017-10-01T00:00 Beijing 7 7 true 7 7 7.7 7.7 char7 7 7 7 ::7 -70 2017-10-01 2017-10-01T00:00 Beijing 70 70 true 70 70 70.7 70.7 char70 70 70 70 ::70 -71 2017-10-01 2017-10-01T00:00 Beijing 71 71 true 71 71 71.71 71.71 char71 71 71 71 ::71 -72 2017-10-01 2017-10-01T00:00 Beijing 72 72 true 72 72 72.72 72.72 char72 72 72 72 ::72 -73 2017-10-01 2017-10-01T00:00 Beijing 73 73 true 73 73 73.73 73.73 char73 73 73 73 ::73 -74 2017-10-01 2017-10-01T00:00 Beijing 74 74 true 74 74 74.74 74.74 char74 74 74 74 ::74 -75 2017-10-01 2017-10-01T00:00 Beijing 75 75 true 75 75 75.75 75.75 char75 75 75 75 ::75 -76 2017-10-01 2017-10-01T00:00 Beijing 76 76 true 76 76 76.76 76.76 char76 76 76 76 ::76 -77 2017-10-01 2017-10-01T00:00 Beijing 77 77 true 77 77 77.77 77.77 char77 77 77 77 ::77 -78 2017-10-01 2017-10-01T00:00 Beijing 78 78 true 78 78 78.78 78.78 char78 78 78 78 ::78 -79 2017-10-01 2017-10-01T00:00 Beijing 79 79 true 79 79 79.79 79.79 char79 79 79 79 ::79 -8 2017-10-01 2017-10-01T00:00 Beijing 8 8 true 8 8 8.8 8.8 char8 8 8 8 ::8 -80 2017-10-01 2017-10-01T00:00 Beijing 80 80 true 80 80 80.8 80.8 char80 80 80 80 ::80 -81 2017-10-01 2017-10-01T00:00 Beijing 81 81 true 81 81 81.81 81.81 char81 81 81 81 ::81 -82 2017-10-01 2017-10-01T00:00 Beijing 82 82 true 82 82 82.82 82.82 char82 82 82 82 ::82 -83 2017-10-01 2017-10-01T00:00 Beijing 83 83 true 83 83 83.83 83.83 char83 83 83 83 ::83 -84 2017-10-01 2017-10-01T00:00 Beijing 84 84 true 84 84 84.84 84.84 char84 84 84 84 ::84 -85 2017-10-01 2017-10-01T00:00 Beijing 85 85 true 85 85 85.85 85.85 char85 85 85 85 ::85 -86 2017-10-01 2017-10-01T00:00 Beijing 86 86 true 86 86 86.86 86.86 char86 86 86 86 ::86 -87 2017-10-01 2017-10-01T00:00 Beijing 87 87 true 87 87 87.87 87.87 char87 87 87 87 ::87 -88 2017-10-01 2017-10-01T00:00 Beijing 88 88 true 88 88 88.88 88.88 char88 88 88 88 ::88 -89 2017-10-01 2017-10-01T00:00 Beijing 89 89 true 89 89 89.89 89.89 char89 89 89 89 ::89 -9 2017-10-01 2017-10-01T00:00 Beijing 9 9 true 9 9 9.9 9.9 char9 9 9 9 ::9 -90 2017-10-01 2017-10-01T00:00 Beijing 90 90 true 90 90 90.9 90.9 char90 90 90 90 ::90 -91 2017-10-01 2017-10-01T00:00 Beijing 91 91 true 91 91 91.91 91.91 char91 91 91 91 ::91 -92 2017-10-01 2017-10-01T00:00 Beijing 92 92 true 92 92 92.92 92.92 char92 92 92 92 ::92 -93 2017-10-01 2017-10-01T00:00 Beijing 93 93 true 93 93 93.93 93.93 char93 93 93 93 ::93 -94 2017-10-01 2017-10-01T00:00 Beijing 94 94 true 94 94 94.94 94.94 char94 94 94 94 ::94 -95 2017-10-01 2017-10-01T00:00 Beijing 95 95 true 95 95 95.95 95.95 char95 95 95 95 ::95 -96 2017-10-01 2017-10-01T00:00 Beijing 96 96 true 96 96 96.96 96.96 char96 96 96 96 ::96 -97 2017-10-01 2017-10-01T00:00 Beijing 97 97 true 97 97 97.97 97.97 char97 97 97 97 ::97 -98 2017-10-01 2017-10-01T00:00 Beijing 98 98 true 98 98 98.98 98.98 char98 98 98 98 ::98 -99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99.99 99.99 char99 99 99 99 ::99 +11 2017-10-01 2017-10-01T00:00 Beijing 11 11 true 11 11 11.11 11.11 char11 11.000000000 11 11 ::11 +12 2017-10-01 2017-10-01T00:00 Beijing 12 12 true 12 12 12.12 12.12 char12 12.000000000 12 12 ::12 +13 2017-10-01 2017-10-01T00:00 Beijing 13 13 true 13 13 13.13 13.13 char13 13.000000000 13 13 ::13 +14 2017-10-01 2017-10-01T00:00 Beijing 14 14 true 14 14 14.14 14.14 char14 14.000000000 14 14 ::14 +15 2017-10-01 2017-10-01T00:00 Beijing 15 15 true 15 15 15.15 15.15 char15 15.000000000 15 15 ::15 +16 2017-10-01 2017-10-01T00:00 Beijing 16 16 true 16 16 16.16 16.16 char16 16.000000000 16 16 ::16 +17 2017-10-01 2017-10-01T00:00 Beijing 17 17 true 17 17 17.17 17.17 char17 17.000000000 17 17 ::17 +18 2017-10-01 2017-10-01T00:00 Beijing 18 18 true 18 18 18.18 18.18 char18 18.000000000 18 18 ::18 +19 2017-10-01 2017-10-01T00:00 Beijing 19 19 true 19 19 19.19 19.19 char19 19.000000000 19 19 ::19 +2 2017-10-01 2017-10-01T00:00 Beijing 2 2 true 2 2 2.2 2.2 char2 2.000000000 2 2 ::2 +20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20.2 20.2 char20 20.000000000 20 20 ::20 +21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21.21 21.21 char21 21.000000000 21 21 ::21 +22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22.22 22.22 char22 22.000000000 22 22 ::22 +23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23.23 23.23 char23 23.000000000 23 23 ::23 +24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24.24 24.24 char24 24.000000000 24 24 ::24 +25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25.25 25.25 char25 25.000000000 25 25 ::25 +26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26.26 26.26 char26 26.000000000 26 26 ::26 +27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27.27 27.27 char27 27.000000000 27 27 ::27 +28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28.28 28.28 char28 28.000000000 28 28 ::28 +29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29.29 29.29 char29 29.000000000 29 29 ::29 +3 2017-10-01 2017-10-01T00:00 Beijing 3 3 true 3 3 3.3 3.3 char3 3.000000000 3 3 ::3 +30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30.3 30.3 char30 30.000000000 30 30 ::30 +31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31.31 31.31 char31 31.000000000 31 31 ::31 +32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32.32 32.32 char32 32.000000000 32 32 ::32 +33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33.33 33.33 char33 33.000000000 33 33 ::33 +34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34.34 34.34 char34 34.000000000 34 34 ::34 +35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35.35 35.35 char35 35.000000000 35 35 ::35 +36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36.36 36.36 char36 36.000000000 36 36 ::36 +37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37.37 37.37 char37 37.000000000 37 37 ::37 +38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38.38 38.38 char38 38.000000000 38 38 ::38 +39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39.39 39.39 char39 39.000000000 39 39 ::39 +4 2017-10-01 2017-10-01T00:00 Beijing 4 4 true 4 4 4.4 4.4 char4 4.000000000 4 4 ::4 +40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40.4 40.4 char40 40.000000000 40 40 ::40 +41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41.41 41.41 char41 41.000000000 41 41 ::41 +42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42.42 42.42 char42 42.000000000 42 42 ::42 +43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43.43 43.43 char43 43.000000000 43 43 ::43 +44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44.44 44.44 char44 44.000000000 44 44 ::44 +45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45.45 45.45 char45 45.000000000 45 45 ::45 +46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46.46 46.46 char46 46.000000000 46 46 ::46 +47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47.47 47.47 char47 47.000000000 47 47 ::47 +48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48.48 48.48 char48 48.000000000 48 48 ::48 +49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49.49 49.49 char49 49.000000000 49 49 ::49 +5 2017-10-01 2017-10-01T00:00 Beijing 5 5 true 5 5 5.5 5.5 char5 5.000000000 5 5 ::5 +50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50.5 50.5 char50 50.000000000 50 50 ::50 +51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51.51 51.51 char51 51.000000000 51 51 ::51 +52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52.52 52.52 char52 52.000000000 52 52 ::52 +53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53.53 53.53 char53 53.000000000 53 53 ::53 +54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54.54 54.54 char54 54.000000000 54 54 ::54 +55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55.55 55.55 char55 55.000000000 55 55 ::55 +56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56.56 56.56 char56 56.000000000 56 56 ::56 +57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57.57 57.57 char57 57.000000000 57 57 ::57 +58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58.58 58.58 char58 58.000000000 58 58 ::58 +59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59.59 59.59 char59 59.000000000 59 59 ::59 +6 2017-10-01 2017-10-01T00:00 Beijing 6 6 true 6 6 6.6 6.6 char6 6.000000000 6 6 ::6 +60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60.6 60.6 char60 60.000000000 60 60 ::60 +61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61.61 61.61 char61 61.000000000 61 61 ::61 +62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62.62 62.62 char62 62.000000000 62 62 ::62 +63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63.63 63.63 char63 63.000000000 63 63 ::63 +64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64.64 64.64 char64 64.000000000 64 64 ::64 +65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65.65 65.65 char65 65.000000000 65 65 ::65 +66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66.66 66.66 char66 66.000000000 66 66 ::66 +67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67.67 67.67 char67 67.000000000 67 67 ::67 +68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68.68 68.68 char68 68.000000000 68 68 ::68 +69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69.69 69.69 char69 69.000000000 69 69 ::69 +7 2017-10-01 2017-10-01T00:00 Beijing 7 7 true 7 7 7.7 7.7 char7 7.000000000 7 7 ::7 +70 2017-10-01 2017-10-01T00:00 Beijing 70 70 true 70 70 70.7 70.7 char70 70.000000000 70 70 ::70 +71 2017-10-01 2017-10-01T00:00 Beijing 71 71 true 71 71 71.71 71.71 char71 71.000000000 71 71 ::71 +72 2017-10-01 2017-10-01T00:00 Beijing 72 72 true 72 72 72.72 72.72 char72 72.000000000 72 72 ::72 +73 2017-10-01 2017-10-01T00:00 Beijing 73 73 true 73 73 73.73 73.73 char73 73.000000000 73 73 ::73 +74 2017-10-01 2017-10-01T00:00 Beijing 74 74 true 74 74 74.74 74.74 char74 74.000000000 74 74 ::74 +75 2017-10-01 2017-10-01T00:00 Beijing 75 75 true 75 75 75.75 75.75 char75 75.000000000 75 75 ::75 +76 2017-10-01 2017-10-01T00:00 Beijing 76 76 true 76 76 76.76 76.76 char76 76.000000000 76 76 ::76 +77 2017-10-01 2017-10-01T00:00 Beijing 77 77 true 77 77 77.77 77.77 char77 77.000000000 77 77 ::77 +78 2017-10-01 2017-10-01T00:00 Beijing 78 78 true 78 78 78.78 78.78 char78 78.000000000 78 78 ::78 +79 2017-10-01 2017-10-01T00:00 Beijing 79 79 true 79 79 79.79 79.79 char79 79.000000000 79 79 ::79 +8 2017-10-01 2017-10-01T00:00 Beijing 8 8 true 8 8 8.8 8.8 char8 8.000000000 8 8 ::8 +80 2017-10-01 2017-10-01T00:00 Beijing 80 80 true 80 80 80.8 80.8 char80 80.000000000 80 80 ::80 +81 2017-10-01 2017-10-01T00:00 Beijing 81 81 true 81 81 81.81 81.81 char81 81.000000000 81 81 ::81 +82 2017-10-01 2017-10-01T00:00 Beijing 82 82 true 82 82 82.82 82.82 char82 82.000000000 82 82 ::82 +83 2017-10-01 2017-10-01T00:00 Beijing 83 83 true 83 83 83.83 83.83 char83 83.000000000 83 83 ::83 +84 2017-10-01 2017-10-01T00:00 Beijing 84 84 true 84 84 84.84 84.84 char84 84.000000000 84 84 ::84 +85 2017-10-01 2017-10-01T00:00 Beijing 85 85 true 85 85 85.85 85.85 char85 85.000000000 85 85 ::85 +86 2017-10-01 2017-10-01T00:00 Beijing 86 86 true 86 86 86.86 86.86 char86 86.000000000 86 86 ::86 +87 2017-10-01 2017-10-01T00:00 Beijing 87 87 true 87 87 87.87 87.87 char87 87.000000000 87 87 ::87 +88 2017-10-01 2017-10-01T00:00 Beijing 88 88 true 88 88 88.88 88.88 char88 88.000000000 88 88 ::88 +89 2017-10-01 2017-10-01T00:00 Beijing 89 89 true 89 89 89.89 89.89 char89 89.000000000 89 89 ::89 +9 2017-10-01 2017-10-01T00:00 Beijing 9 9 true 9 9 9.9 9.9 char9 9.000000000 9 9 ::9 +90 2017-10-01 2017-10-01T00:00 Beijing 90 90 true 90 90 90.9 90.9 char90 90.000000000 90 90 ::90 +91 2017-10-01 2017-10-01T00:00 Beijing 91 91 true 91 91 91.91 91.91 char91 91.000000000 91 91 ::91 +92 2017-10-01 2017-10-01T00:00 Beijing 92 92 true 92 92 92.92 92.92 char92 92.000000000 92 92 ::92 +93 2017-10-01 2017-10-01T00:00 Beijing 93 93 true 93 93 93.93 93.93 char93 93.000000000 93 93 ::93 +94 2017-10-01 2017-10-01T00:00 Beijing 94 94 true 94 94 94.94 94.94 char94 94.000000000 94 94 ::94 +95 2017-10-01 2017-10-01T00:00 Beijing 95 95 true 95 95 95.95 95.95 char95 95.000000000 95 95 ::95 +96 2017-10-01 2017-10-01T00:00 Beijing 96 96 true 96 96 96.96 96.96 char96 96.000000000 96 96 ::96 +97 2017-10-01 2017-10-01T00:00 Beijing 97 97 true 97 97 97.97 97.97 char97 97.000000000 97 97 ::97 +98 2017-10-01 2017-10-01T00:00 Beijing 98 98 true 98 98 98.98 98.98 char98 98.000000000 98 98 ::98 +99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99.99 99.99 char99 99.000000000 99 99 ::99 diff --git a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out index 59e94ef9429ec9..784ad963ce4a72 100644 --- a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out +++ b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out @@ -30,14 +30,14 @@ 8 nereids \N -- !select_base2 -- -1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 -2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 -3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 +1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1.000000000 1.000000000 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 +2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.940656458412465e-324 char2 100000000.000000000 100000000.000000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 +3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.402823E38 1.797693134862316e+308 char3 999999999.000000000 999999999.000000000 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -- !select_tvf2 -- -1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 -2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 -3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 +1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1.000000000 1.000000000 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 +2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.940656458412465e-324 char2 100000000.000000000 100000000.000000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 +3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.402823E38 1.797693134862316e+308 char3 999999999.000000000 999999999.000000000 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -- !hive_docker_02 -- 1 2023-04-20 2023-04-20 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 @@ -75,14 +75,14 @@ 8 nereids \N -- !select_base2 -- -1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 -2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 -3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 +1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1.000000000 1.000000000 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 +2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.940656458412465e-324 char2 100000000.000000000 100000000.000000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 +3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.402823E38 1.797693134862316e+308 char3 999999999.000000000 999999999.000000000 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -- !select_tvf2 -- -1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 -2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 -3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 +1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1.000000000 1.000000000 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 +2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.940656458412465e-324 char2 100000000.000000000 100000000.000000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 +3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.402823E38 1.797693134862316e+308 char3 999999999.000000000 999999999.000000000 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -- !hive_docker_02 -- 1 2023-04-20 2023-04-20 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 diff --git a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.out b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_complex_type.out similarity index 100% rename from regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.out rename to regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_complex_type.out diff --git a/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out b/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out index 160c99248fe90c..9adea59bbfba3e 100644 --- a/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out +++ b/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out @@ -199,203 +199,3 @@ true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default -- !hive_docker_ctas_types_02 -- true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default --- !ctas_01 -- -2 -3 - --- !hive_docker_ctas_01 -- -2 -3 - --- !ctas_02 -- -2 -3 - --- !hive_docker_ctas_02 -- -2 -3 - --- !ctas_03 -- -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_03 -- -22 value_for_pt11 value_for_pt22 - --- !ctas_04 -- -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_04 -- -22 value_for_pt11 value_for_pt22 - --- !ctas_05 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_05 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !ctas_06 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_06 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !ctas_ex01 -- -2 -3 - --- !hive_docker_ctas_ex01 -- -2 -3 - --- !ctas_ex02 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 \N -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_ex02 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 __HIVE_DEFAULT_PARTITION__ -22 value_for_pt11 value_for_pt22 - --- !ctas_03 -- -\N another string value for col2 -\N string value for col2 -\N yet another string value for col2 - --- !hive_docker_ctas_ex03 -- -\N another string value for col2 -\N string value for col2 -\N yet another string value for col2 - --- !ctas_04 -- -\N 11 value_for_pt1 -\N 22 value_for_pt11 - --- !hive_docker_ctas_ex04 -- -\N 11 value_for_pt1 -\N 22 value_for_pt11 - --- !qualified_table1 -- -11 value_for_pt1 -22 value_for_pt11 - --- !qualified_table2 -- -11 value_for_pt1 -22 value_for_pt11 - --- !ctas_types_01 -- -true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 - --- !hive_docker_ctas_types_01 -- -true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29 23:19:34.0 - --- !ctas_types_02 -- -true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default - --- !hive_docker_ctas_types_02 -- -true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default - --- !ctas_01 -- -2 -3 - --- !hive_docker_ctas_01 -- -2 -3 - --- !ctas_02 -- -2 -3 - --- !hive_docker_ctas_02 -- -2 -3 - --- !ctas_03 -- -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_03 -- -22 value_for_pt11 value_for_pt22 - --- !ctas_04 -- -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_04 -- -22 value_for_pt11 value_for_pt22 - --- !ctas_05 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_05 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !ctas_06 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_06 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !ctas_ex01 -- -2 -3 - --- !hive_docker_ctas_ex01 -- -2 -3 - --- !ctas_ex02 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 \N -22 value_for_pt11 value_for_pt22 - --- !hive_docker_ctas_ex02 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 __HIVE_DEFAULT_PARTITION__ -22 value_for_pt11 value_for_pt22 - --- !ctas_03 -- -\N another string value for col2 -\N string value for col2 -\N yet another string value for col2 - --- !hive_docker_ctas_ex03 -- -\N another string value for col2 -\N string value for col2 -\N yet another string value for col2 - --- !ctas_04 -- -\N 11 value_for_pt1 -\N 22 value_for_pt11 - --- !hive_docker_ctas_ex04 -- -\N 11 value_for_pt1 -\N 22 value_for_pt11 - --- !qualified_table1 -- -11 value_for_pt1 -22 value_for_pt11 - --- !qualified_table2 -- -11 value_for_pt1 -22 value_for_pt11 - --- !ctas_types_01 -- -true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 - --- !hive_docker_ctas_types_01 -- -true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29 23:19:34.0 - --- !ctas_types_02 -- -true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default - --- !hive_docker_ctas_types_02 -- -true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default - diff --git a/regression-test/data/external_table_p0/hive/test_complex_types.out b/regression-test/data/external_table_p0/hive/test_complex_types.out index 6ba19bd908ffdd..7c128fa8c4624c 100644 --- a/regression-test/data/external_table_p0/hive/test_complex_types.out +++ b/regression-test/data/external_table_p0/hive/test_complex_types.out @@ -47,51 +47,3 @@ -- !date_dict -- 2036-12-28 1898-12-28 2539-12-28 --- !null_struct_element -- -0 - --- !map_key_select -- -38111 0.770169659057425 - --- !map_keys -- -["9wXr9n-TBm9Wyt-r8H-SkAq", "CPDH4G-ZXGPkku-3wY-ktaQ", "RvNlMt-HHjHN5M-VjP-xHAI", "qKIhKy-Ws344os-haX-2pmT", "DOJJ5l-UEkwVMs-x9F-HifD", "m871g8-1eFi7jt-oBq-S0yc", "wXugVP-v2fc6IF-DeU-On3T", "B0mXFX-QvgUgo7-Dih-6rDu", "E9zv3F-xMqSbMa-il4-FuDg", "msuFIN-ZkKO8TY-tu4-veH0", "0rSUyl-Un07aIW-KAx-WHnX", "XvbmO8-WA6oAqc-ihc-s8IL", "G6B6RD-AicAlZb-16u-Pn1I", "coDK0Q-tMg1294-JMQ-ZWQu", "4c0aWh-yhL6BOX-rRu-1n0r", "G4iUcG-ZhWw62v-VLt-n6lH", "IIB7qD-WQistwT-Vux-0c9B", "7cTyuR-5ssXm2S-sJR-JTIZ", "3KPhSW-FICEImf-bba-PCiQ", "qQ7Yup-XBeQGFz-3EP-q0vd", "gjRxRo-Af9Oqx5-IzN-3B9d", "1zSj57-nNZpZ0b-ZKn-BeY0", "sTK0mn-wkp1Xp5-PRS-txVM", "sLrM0s-1KnXLb6-1A3-Z1vJ", "UkYdkP-k7YKiKS-Fxp-qAcI", "v8p0YV-R5pAKZ8-UMr-P1bQ", "RJdTav-jk3os9Z-yRk-WhwV", "lB91ic-pNFZkE4-hBx-e104", "gmRV6e-GKJUg0L-ok7-J6Lz", "o3LUyz-7Toh54O-czG-Xep8", "8fzHhM-4otPAss-qTm-phg8", "kZsHhe-vfClpAR-b3H-7aHl", "TdZnlG-BUgMs7Z-iBM-9c3v", "RipJXn-p4gZkyy-1ZY-xkWe", "ke730M-LmMjGdc-EFy-0LUK", "jBSExJ-GXTc5TB-NSa-xBEd", "kI7Cc8-DSg5RdF-qLo-2bhe", "bAn3VI-x6xXWpB-zWe-G5CJ", "jAil30-kbt6K6z-kbr-8foB", "IHIwNs-1QGqy8l-i8i-vu4G", "p0IbZr-tHCtwiV-0hq-NtIt", "iggdij-M3YNBpd-yiD-a8Ro", "BrJEww-C4LpgaS-AeB-So4U", "xnO3Fi-8rXcpgj-zpm-EmuX", "5w57da-phYtDUx-px2-6frG", "31MfFs-1WyUAr6-gQ0-xLxY", "ryBl2p-rSoPhwd-WPv-NCAU", "KN5TEt-gOfJ4Hy-3pp-HiBa", "ytqxb8-utXXjUf-m41-i6ir", "WhGUGz-zzyvEpD-9BM-2bVf", "dE1tFe-zHClt4u-0cY-TQnC", "MveBhC-g29c0dU-tCT-R6nC", "JTpxue-xSqAhGo-AZk-zB1t", "92TVdU-qDJesPN-0lb-JOd3", "0PODnh-IciBdOZ-0CS-oNeL", "KkkW6x-TiemXQw-OiH-dZ9s", "PIs5Aj-g02HRXw-957-GD2z", "yJIzuw-au6460e-0Tl-XYEJ", "KHvMCD-OQDL0eX-nqK-TmEt", "6QJJgV-Z3IZ1Rf-wyv-rIJ6", "qA9ycc-sR2qm6P-PtB-AIax", "uDeuEb-B0t0Ljr-dWk-jkC4", "5vPy52-ygN0MMH-UB4-nZQL", "zbbmrQ-pT3uAuU-Kae-HjM5", "3QShHS-7RwUB10-0W2-H4Qy", "PMc4QI-5lNajXU-f8m-RGIi", "O9t3dl-q8YHozj-saR-A3Jm", "k4eH3O-aHnTKY7-ADp-4Vsi", "RA4epe-lWWnOff-bpM-bSR4", "6ysu2R-gSc5dwU-cv0-LqCJ", "tVl3TY-o42NMVO-k3S-iqOY", "NMgTrr-W1RrCvP-Zaf-paL7", "d1CJmF-CeG5asM-xms-1dwN", "N1D30g-zFjiGzI-eHC-Sof4", "tOhfKu-Gdtf9Ne-KwA-JdHV", "XLzwK0-6ocGDrS-TtU-wlEI", "XDgZfb-Sxc45Zn-mVO-S2QO", "GQD7a0-fnt9BZs-Kvh-dPbJ", "9dJxj9-HFwEQMY-6p9-s8Vt", "1qU9pA-QJGAna9-JoG-H7GS", "rKIkxA-UnGWYSn-0li-ziuB", "tbPazx-IjUrQ8J-NZe-VOPL", "xBpSIv-U6ojkK7-9p5-LviD", "88bnWI-pxrKa7T-n2d-tXk9", "0XviXp-9ksT8s0-fDy-35SW", "e0XauA-GNRALmd-SM2-Y4Gf", "kyvYBk-Bk5M4Xq-gxX-kE1B", "dIiQzS-5sT4ogL-6IV-tLmb", "OlGOyH-dyL1nzj-B2M-z8ir", "zC9Gtn-x8hpfPD-KOu-k31W", "qSq3z2-Lpv0YcB-hBq-Sabd", "LSyNyi-tBZUx1l-hAj-mwsx", "2c9aTP-hXloMK7-ufH-dgq6", "aXksHO-zARQxfo-sgS-8Bf4", "ioOXAL-eVUF0W8-vZx-ZeYX", "DXUkAP-A7SqnHj-V4U-PJfz", "cnzZXk-AOMepfN-hym-qbDH", "CMlAd6-8FF1yXs-fae-Izfv", "qiXnUv-e2PsJWm-tLF-KpjE", "Gfx3k9-JvXa7Wd-rI1-1e1E"] - --- !map_values -- -[0.9805502029231666, 0.5330291595754054, 0.3002474487337981, 0.4856360175030267, 0.7687106425158624, 0.6993506644925102, 0.2849354808825807, 0.3473417455186141, 0.1350012944304507, 0.9708132103700939, 0.1858304263994345, 0.4886337264552073, 0.3635474169515766, 0.5640845268971175, 0.1374134087807577, 0.7766547647451623, 0.5835323296668318, 0.3654459547110349, 0.5479776709993764, 0.8379932542117192, 0.1566504627835081, 0.03371222042250388, 0.1699781825927229, 0.3579630495075078, 0.02809253185597727, 0.7204247029840027, 0.2760499256423206, 0.676890893219096, 0.03529878656700025, 0.02276578351027858, 0.09794991730625469, 0.5278062884613351, 0.1370404181139102, 0.5440352476580856, 0.7205540629419929, 0.1350852984195943, 0.4160946400431862, 0.2972295454562929, 0.9217426503585693, 0.58103998733474, 0.8845427436377473, 0.1017928267299423, 0.9547186973943892, 0.1680102784708342, 0.0008487745421986714, 0.1695241541106989, 0.6783921749433292, 0.7193818386971084, 0.930443435029246, 0.4846665469390518, 0.9924998940864419, 0.7238288481079148, 0.7053563817759009, 0.9735160772776755, 0.7782499787869234, 0.7413304280548174, 0.7550983926033307, 0.8713660446322186, 0.9205209678792637, 0.3419724898972277, 0.3696806985755556, 0.03023259817152302, 0.02477452604862684, 0.9764129157525588, 0.5933057559470283, 0.7612511554831843, 0.378758227033635, 0.9312730459544121, 0.6712083507802412, 0.165080800084368, 0.2292866463959062, 0.3736665350268106, 0.2048064464080658, 0.08394355937496834, 0.8494979696731824, 0.4321556255662622, 0.3534668267198027, 0.8791700434102772, 0.2274527583015258, 0.04886968507359402, 0.7936598110174163, 0.5449717343415919, 0.7635939445968348, 0.08505586183986624, 0.3509115026589145, 0.9633191745238908, 0.3972533910389617, 0.4659759249919267, 0.1579051246328464, 0.7853565578107594, 0.9894919939745654, 0.9395365730655929, 0.202260767382666, 0.1619636856192768, 0.5105569529841616, 0.4531109229280732, 0.2579134268597084, 0.7962109089915747, 0.2772969229539421, 0.9315902037607061] - --- !map_contains_key -- -1077 [0.7805560995873845, 0.9303489002269559, 0.2529522997521877, 0.662270811026298, 0.664725297532439, 0.1019441091764477, 0.9614059300688174, 0.5278126009983843, 0.5287505841216708, 0.426116738236779, 0.4230050239387118, 0.5327026330053651, 0.6025481777942603, 0.2710733647257627, 0.613792118138183, 0.002100302783562991, 0.3200675048728582, 0.5485611014660204, 0.5121510581313707, 0.5145136652805358] {"9wXr9n-TBm9Wyt-r8H-SkAq":0.9338329010480995, "CPDH4G-ZXGPkku-3wY-ktaQ":0.4355256963350881, "RvNlMt-HHjHN5M-VjP-xHAI":0.3263474611804782, "qKIhKy-Ws344os-haX-2pmT":0.565450203625137, "DOJJ5l-UEkwVMs-x9F-HifD":0.09375622010822238, "m871g8-1eFi7jt-oBq-S0yc":0.8819687247951038, "wXugVP-v2fc6IF-DeU-On3T":0.3448233486447311, "B0mXFX-QvgUgo7-Dih-6rDu":0.1914040395475467, "E9zv3F-xMqSbMa-il4-FuDg":0.3857021891084336, "msuFIN-ZkKO8TY-tu4-veH0":0.6646172653074628, "0rSUyl-Un07aIW-KAx-WHnX":0.3558009910430974, "XvbmO8-WA6oAqc-ihc-s8IL":0.4058206434411423, "G6B6RD-AicAlZb-16u-Pn1I":0.7203554946895749, "coDK0Q-tMg1294-JMQ-ZWQu":0.8236328627743186, "4c0aWh-yhL6BOX-rRu-1n0r":0.1398091184230428, "G4iUcG-ZhWw62v-VLt-n6lH":0.1838288978254214, "IIB7qD-WQistwT-Vux-0c9B":0.9174389144309458, "7cTyuR-5ssXm2S-sJR-JTIZ":0.8132237242672837, "3KPhSW-FICEImf-bba-PCiQ":0.6302643579943553, "qQ7Yup-XBeQGFz-3EP-q0vd":0.6109025726752364, "gjRxRo-Af9Oqx5-IzN-3B9d":0.9251468490326916, "1zSj57-nNZpZ0b-ZKn-BeY0":0.5628463109107144, "sTK0mn-wkp1Xp5-PRS-txVM":0.7905808129559996, "sLrM0s-1KnXLb6-1A3-Z1vJ":0.4234598677670157, "UkYdkP-k7YKiKS-Fxp-qAcI":0.7541401266679869, "v8p0YV-R5pAKZ8-UMr-P1bQ":0.2931152565110683, "RJdTav-jk3os9Z-yRk-WhwV":0.5263811309738877, "lB91ic-pNFZkE4-hBx-e104":0.6692292834321788, "gmRV6e-GKJUg0L-ok7-J6Lz":0.05924766959664352, "o3LUyz-7Toh54O-czG-Xep8":0.6284193821127264, "8fzHhM-4otPAss-qTm-phg8":0.8953002441537012, "kZsHhe-vfClpAR-b3H-7aHl":0.1775015612747399, "TdZnlG-BUgMs7Z-iBM-9c3v":0.2749839439504633, "RipJXn-p4gZkyy-1ZY-xkWe":0.05461626895038973, "ke730M-LmMjGdc-EFy-0LUK":0.3078176183644828, "jBSExJ-GXTc5TB-NSa-xBEd":0.6617827850054024, "kI7Cc8-DSg5RdF-qLo-2bhe":0.9835707461323488, "bAn3VI-x6xXWpB-zWe-G5CJ":0.2179821229979456, "jAil30-kbt6K6z-kbr-8foB":0.9788066977245138, "IHIwNs-1QGqy8l-i8i-vu4G":0.4967939741245059, "p0IbZr-tHCtwiV-0hq-NtIt":0.05018379510905702, "iggdij-M3YNBpd-yiD-a8Ro":0.982385582884686, "BrJEww-C4LpgaS-AeB-So4U":0.9024855415553655, "xnO3Fi-8rXcpgj-zpm-EmuX":0.2052911881746857, "5w57da-phYtDUx-px2-6frG":0.2969063879156597, "31MfFs-1WyUAr6-gQ0-xLxY":0.4879555128313509, "ryBl2p-rSoPhwd-WPv-NCAU":0.7954485484495807, "KN5TEt-gOfJ4Hy-3pp-HiBa":0.1533389643648807, "ytqxb8-utXXjUf-m41-i6ir":0.6150208673719357, "WhGUGz-zzyvEpD-9BM-2bVf":0.581040090228354, "dE1tFe-zHClt4u-0cY-TQnC":0.7608999632369456, "MveBhC-g29c0dU-tCT-R6nC":0.3345734028221851, "JTpxue-xSqAhGo-AZk-zB1t":0.3504030277488054, "92TVdU-qDJesPN-0lb-JOd3":0.7387694998319805, "0PODnh-IciBdOZ-0CS-oNeL":0.9515905965769644, "KkkW6x-TiemXQw-OiH-dZ9s":0.4082412331999081, "PIs5Aj-g02HRXw-957-GD2z":0.641526116451016, "yJIzuw-au6460e-0Tl-XYEJ":0.7521928530356236, "KHvMCD-OQDL0eX-nqK-TmEt":0.1309616727896826, "6QJJgV-Z3IZ1Rf-wyv-rIJ6":0.7007110387725962, "qA9ycc-sR2qm6P-PtB-AIax":0.4462977655645909, "uDeuEb-B0t0Ljr-dWk-jkC4":0.6904672767407958, "5vPy52-ygN0MMH-UB4-nZQL":0.6057596542200021, "zbbmrQ-pT3uAuU-Kae-HjM5":0.9812657498686279, "3QShHS-7RwUB10-0W2-H4Qy":0.4155760848860853, "PMc4QI-5lNajXU-f8m-RGIi":0.7046420976800288, "O9t3dl-q8YHozj-saR-A3Jm":0.8543344954196586, "k4eH3O-aHnTKY7-ADp-4Vsi":0.2655832454718557, "RA4epe-lWWnOff-bpM-bSR4":0.7523252210222883, "6ysu2R-gSc5dwU-cv0-LqCJ":0.7830899322716732, "tVl3TY-o42NMVO-k3S-iqOY":0.7923823401215799, "NMgTrr-W1RrCvP-Zaf-paL7":0.4686928654756936, "d1CJmF-CeG5asM-xms-1dwN":0.7622908781076493, "N1D30g-zFjiGzI-eHC-Sof4":0.847542878440137, "tOhfKu-Gdtf9Ne-KwA-JdHV":0.4999285217445154, "XLzwK0-6ocGDrS-TtU-wlEI":0.3985354402705095, "XDgZfb-Sxc45Zn-mVO-S2QO":0.05791580337644187, "GQD7a0-fnt9BZs-Kvh-dPbJ":0.663903859916476, "9dJxj9-HFwEQMY-6p9-s8Vt":0.2194407595305434, "1qU9pA-QJGAna9-JoG-H7GS":0.8877401947295382, "rKIkxA-UnGWYSn-0li-ziuB":0.1607906275036466, "tbPazx-IjUrQ8J-NZe-VOPL":0.6809166916797593, "xBpSIv-U6ojkK7-9p5-LviD":0.1195672647379901, "88bnWI-pxrKa7T-n2d-tXk9":0.1956068951787721, "0XviXp-9ksT8s0-fDy-35SW":0.8690659418822626, "e0XauA-GNRALmd-SM2-Y4Gf":0.6840816888752089, "kyvYBk-Bk5M4Xq-gxX-kE1B":0.7744771682336401, "dIiQzS-5sT4ogL-6IV-tLmb":0.0340772833497166, "OlGOyH-dyL1nzj-B2M-z8ir":0.3765608037933722, "zC9Gtn-x8hpfPD-KOu-k31W":0.864392047887076, "qSq3z2-Lpv0YcB-hBq-Sabd":0.1542847609246678, "LSyNyi-tBZUx1l-hAj-mwsx":0.304034328298701, "2c9aTP-hXloMK7-ufH-dgq6":0.1016852552953107, "aXksHO-zARQxfo-sgS-8Bf4":0.5490533082019959, "ioOXAL-eVUF0W8-vZx-ZeYX":0.4528164038481785, "DXUkAP-A7SqnHj-V4U-PJfz":0.3607407447425939, "cnzZXk-AOMepfN-hym-qbDH":0.4587361500592568, "CMlAd6-8FF1yXs-fae-Izfv":0.07555019720825917, "qiXnUv-e2PsJWm-tLF-KpjE":0.9409681065363688, "Gfx3k9-JvXa7Wd-rI1-1e1E":0.7492793312178226} {"name":"r8HXXQM4XHoI", "age":238221053, "tip":"2023-07-26 15:40:37.694000"} - --- !array_max -- -11028 - --- !null_struct_element_orc -- -0 - --- !map_key_select_orc -- -38111 0.770169659057425 - --- !map_keys_orc -- -["9wXr9n-TBm9Wyt-r8H-SkAq", "CPDH4G-ZXGPkku-3wY-ktaQ", "RvNlMt-HHjHN5M-VjP-xHAI", "qKIhKy-Ws344os-haX-2pmT", "DOJJ5l-UEkwVMs-x9F-HifD", "m871g8-1eFi7jt-oBq-S0yc", "wXugVP-v2fc6IF-DeU-On3T", "B0mXFX-QvgUgo7-Dih-6rDu", "E9zv3F-xMqSbMa-il4-FuDg", "msuFIN-ZkKO8TY-tu4-veH0", "0rSUyl-Un07aIW-KAx-WHnX", "XvbmO8-WA6oAqc-ihc-s8IL", "G6B6RD-AicAlZb-16u-Pn1I", "coDK0Q-tMg1294-JMQ-ZWQu", "4c0aWh-yhL6BOX-rRu-1n0r", "G4iUcG-ZhWw62v-VLt-n6lH", "IIB7qD-WQistwT-Vux-0c9B", "7cTyuR-5ssXm2S-sJR-JTIZ", "3KPhSW-FICEImf-bba-PCiQ", "qQ7Yup-XBeQGFz-3EP-q0vd", "gjRxRo-Af9Oqx5-IzN-3B9d", "1zSj57-nNZpZ0b-ZKn-BeY0", "sTK0mn-wkp1Xp5-PRS-txVM", "sLrM0s-1KnXLb6-1A3-Z1vJ", "UkYdkP-k7YKiKS-Fxp-qAcI", "v8p0YV-R5pAKZ8-UMr-P1bQ", "RJdTav-jk3os9Z-yRk-WhwV", "lB91ic-pNFZkE4-hBx-e104", "gmRV6e-GKJUg0L-ok7-J6Lz", "o3LUyz-7Toh54O-czG-Xep8", "8fzHhM-4otPAss-qTm-phg8", "kZsHhe-vfClpAR-b3H-7aHl", "TdZnlG-BUgMs7Z-iBM-9c3v", "RipJXn-p4gZkyy-1ZY-xkWe", "ke730M-LmMjGdc-EFy-0LUK", "jBSExJ-GXTc5TB-NSa-xBEd", "kI7Cc8-DSg5RdF-qLo-2bhe", "bAn3VI-x6xXWpB-zWe-G5CJ", "jAil30-kbt6K6z-kbr-8foB", "IHIwNs-1QGqy8l-i8i-vu4G", "p0IbZr-tHCtwiV-0hq-NtIt", "iggdij-M3YNBpd-yiD-a8Ro", "BrJEww-C4LpgaS-AeB-So4U", "xnO3Fi-8rXcpgj-zpm-EmuX", "5w57da-phYtDUx-px2-6frG", "31MfFs-1WyUAr6-gQ0-xLxY", "ryBl2p-rSoPhwd-WPv-NCAU", "KN5TEt-gOfJ4Hy-3pp-HiBa", "ytqxb8-utXXjUf-m41-i6ir", "WhGUGz-zzyvEpD-9BM-2bVf", "dE1tFe-zHClt4u-0cY-TQnC", "MveBhC-g29c0dU-tCT-R6nC", "JTpxue-xSqAhGo-AZk-zB1t", "92TVdU-qDJesPN-0lb-JOd3", "0PODnh-IciBdOZ-0CS-oNeL", "KkkW6x-TiemXQw-OiH-dZ9s", "PIs5Aj-g02HRXw-957-GD2z", "yJIzuw-au6460e-0Tl-XYEJ", "KHvMCD-OQDL0eX-nqK-TmEt", "6QJJgV-Z3IZ1Rf-wyv-rIJ6", "qA9ycc-sR2qm6P-PtB-AIax", "uDeuEb-B0t0Ljr-dWk-jkC4", "5vPy52-ygN0MMH-UB4-nZQL", "zbbmrQ-pT3uAuU-Kae-HjM5", "3QShHS-7RwUB10-0W2-H4Qy", "PMc4QI-5lNajXU-f8m-RGIi", "O9t3dl-q8YHozj-saR-A3Jm", "k4eH3O-aHnTKY7-ADp-4Vsi", "RA4epe-lWWnOff-bpM-bSR4", "6ysu2R-gSc5dwU-cv0-LqCJ", "tVl3TY-o42NMVO-k3S-iqOY", "NMgTrr-W1RrCvP-Zaf-paL7", "d1CJmF-CeG5asM-xms-1dwN", "N1D30g-zFjiGzI-eHC-Sof4", "tOhfKu-Gdtf9Ne-KwA-JdHV", "XLzwK0-6ocGDrS-TtU-wlEI", "XDgZfb-Sxc45Zn-mVO-S2QO", "GQD7a0-fnt9BZs-Kvh-dPbJ", "9dJxj9-HFwEQMY-6p9-s8Vt", "1qU9pA-QJGAna9-JoG-H7GS", "rKIkxA-UnGWYSn-0li-ziuB", "tbPazx-IjUrQ8J-NZe-VOPL", "xBpSIv-U6ojkK7-9p5-LviD", "88bnWI-pxrKa7T-n2d-tXk9", "0XviXp-9ksT8s0-fDy-35SW", "e0XauA-GNRALmd-SM2-Y4Gf", "kyvYBk-Bk5M4Xq-gxX-kE1B", "dIiQzS-5sT4ogL-6IV-tLmb", "OlGOyH-dyL1nzj-B2M-z8ir", "zC9Gtn-x8hpfPD-KOu-k31W", "qSq3z2-Lpv0YcB-hBq-Sabd", "LSyNyi-tBZUx1l-hAj-mwsx", "2c9aTP-hXloMK7-ufH-dgq6", "aXksHO-zARQxfo-sgS-8Bf4", "ioOXAL-eVUF0W8-vZx-ZeYX", "DXUkAP-A7SqnHj-V4U-PJfz", "cnzZXk-AOMepfN-hym-qbDH", "CMlAd6-8FF1yXs-fae-Izfv", "qiXnUv-e2PsJWm-tLF-KpjE", "Gfx3k9-JvXa7Wd-rI1-1e1E"] - --- !map_values_orc -- -[0.9805502029231666, 0.5330291595754054, 0.3002474487337981, 0.4856360175030267, 0.7687106425158624, 0.6993506644925102, 0.2849354808825807, 0.3473417455186141, 0.1350012944304507, 0.9708132103700939, 0.1858304263994345, 0.4886337264552073, 0.3635474169515766, 0.5640845268971175, 0.1374134087807577, 0.7766547647451623, 0.5835323296668318, 0.3654459547110349, 0.5479776709993764, 0.8379932542117192, 0.1566504627835081, 0.03371222042250388, 0.1699781825927229, 0.3579630495075078, 0.02809253185597727, 0.7204247029840027, 0.2760499256423206, 0.676890893219096, 0.03529878656700025, 0.02276578351027858, 0.09794991730625469, 0.5278062884613351, 0.1370404181139102, 0.5440352476580856, 0.7205540629419929, 0.1350852984195943, 0.4160946400431862, 0.2972295454562929, 0.9217426503585693, 0.58103998733474, 0.8845427436377473, 0.1017928267299423, 0.9547186973943892, 0.1680102784708342, 0.0008487745421986714, 0.1695241541106989, 0.6783921749433292, 0.7193818386971084, 0.930443435029246, 0.4846665469390518, 0.9924998940864419, 0.7238288481079148, 0.7053563817759009, 0.9735160772776755, 0.7782499787869234, 0.7413304280548174, 0.7550983926033307, 0.8713660446322186, 0.9205209678792637, 0.3419724898972277, 0.3696806985755556, 0.03023259817152302, 0.02477452604862684, 0.9764129157525588, 0.5933057559470283, 0.7612511554831843, 0.378758227033635, 0.9312730459544121, 0.6712083507802412, 0.165080800084368, 0.2292866463959062, 0.3736665350268106, 0.2048064464080658, 0.08394355937496834, 0.8494979696731824, 0.4321556255662622, 0.3534668267198027, 0.8791700434102772, 0.2274527583015258, 0.04886968507359402, 0.7936598110174163, 0.5449717343415919, 0.7635939445968348, 0.08505586183986624, 0.3509115026589145, 0.9633191745238908, 0.3972533910389617, 0.4659759249919267, 0.1579051246328464, 0.7853565578107594, 0.9894919939745654, 0.9395365730655929, 0.202260767382666, 0.1619636856192768, 0.5105569529841616, 0.4531109229280732, 0.2579134268597084, 0.7962109089915747, 0.2772969229539421, 0.9315902037607061] - --- !map_contains_key_orc -- -1077 [0.7805560995873845, 0.9303489002269559, 0.2529522997521877, 0.662270811026298, 0.664725297532439, 0.1019441091764477, 0.9614059300688174, 0.5278126009983843, 0.5287505841216708, 0.426116738236779, 0.4230050239387118, 0.5327026330053651, 0.6025481777942603, 0.2710733647257627, 0.613792118138183, 0.002100302783562991, 0.3200675048728582, 0.5485611014660204, 0.5121510581313707, 0.5145136652805358] {"9wXr9n-TBm9Wyt-r8H-SkAq":0.9338329010480995, "CPDH4G-ZXGPkku-3wY-ktaQ":0.4355256963350881, "RvNlMt-HHjHN5M-VjP-xHAI":0.3263474611804782, "qKIhKy-Ws344os-haX-2pmT":0.565450203625137, "DOJJ5l-UEkwVMs-x9F-HifD":0.09375622010822238, "m871g8-1eFi7jt-oBq-S0yc":0.8819687247951038, "wXugVP-v2fc6IF-DeU-On3T":0.3448233486447311, "B0mXFX-QvgUgo7-Dih-6rDu":0.1914040395475467, "E9zv3F-xMqSbMa-il4-FuDg":0.3857021891084336, "msuFIN-ZkKO8TY-tu4-veH0":0.6646172653074628, "0rSUyl-Un07aIW-KAx-WHnX":0.3558009910430974, "XvbmO8-WA6oAqc-ihc-s8IL":0.4058206434411423, "G6B6RD-AicAlZb-16u-Pn1I":0.7203554946895749, "coDK0Q-tMg1294-JMQ-ZWQu":0.8236328627743186, "4c0aWh-yhL6BOX-rRu-1n0r":0.1398091184230428, "G4iUcG-ZhWw62v-VLt-n6lH":0.1838288978254214, "IIB7qD-WQistwT-Vux-0c9B":0.9174389144309458, "7cTyuR-5ssXm2S-sJR-JTIZ":0.8132237242672837, "3KPhSW-FICEImf-bba-PCiQ":0.6302643579943553, "qQ7Yup-XBeQGFz-3EP-q0vd":0.6109025726752364, "gjRxRo-Af9Oqx5-IzN-3B9d":0.9251468490326916, "1zSj57-nNZpZ0b-ZKn-BeY0":0.5628463109107144, "sTK0mn-wkp1Xp5-PRS-txVM":0.7905808129559996, "sLrM0s-1KnXLb6-1A3-Z1vJ":0.4234598677670157, "UkYdkP-k7YKiKS-Fxp-qAcI":0.7541401266679869, "v8p0YV-R5pAKZ8-UMr-P1bQ":0.2931152565110683, "RJdTav-jk3os9Z-yRk-WhwV":0.5263811309738877, "lB91ic-pNFZkE4-hBx-e104":0.6692292834321788, "gmRV6e-GKJUg0L-ok7-J6Lz":0.05924766959664352, "o3LUyz-7Toh54O-czG-Xep8":0.6284193821127264, "8fzHhM-4otPAss-qTm-phg8":0.8953002441537012, "kZsHhe-vfClpAR-b3H-7aHl":0.1775015612747399, "TdZnlG-BUgMs7Z-iBM-9c3v":0.2749839439504633, "RipJXn-p4gZkyy-1ZY-xkWe":0.05461626895038973, "ke730M-LmMjGdc-EFy-0LUK":0.3078176183644828, "jBSExJ-GXTc5TB-NSa-xBEd":0.6617827850054024, "kI7Cc8-DSg5RdF-qLo-2bhe":0.9835707461323488, "bAn3VI-x6xXWpB-zWe-G5CJ":0.2179821229979456, "jAil30-kbt6K6z-kbr-8foB":0.9788066977245138, "IHIwNs-1QGqy8l-i8i-vu4G":0.4967939741245059, "p0IbZr-tHCtwiV-0hq-NtIt":0.05018379510905702, "iggdij-M3YNBpd-yiD-a8Ro":0.982385582884686, "BrJEww-C4LpgaS-AeB-So4U":0.9024855415553655, "xnO3Fi-8rXcpgj-zpm-EmuX":0.2052911881746857, "5w57da-phYtDUx-px2-6frG":0.2969063879156597, "31MfFs-1WyUAr6-gQ0-xLxY":0.4879555128313509, "ryBl2p-rSoPhwd-WPv-NCAU":0.7954485484495807, "KN5TEt-gOfJ4Hy-3pp-HiBa":0.1533389643648807, "ytqxb8-utXXjUf-m41-i6ir":0.6150208673719357, "WhGUGz-zzyvEpD-9BM-2bVf":0.581040090228354, "dE1tFe-zHClt4u-0cY-TQnC":0.7608999632369456, "MveBhC-g29c0dU-tCT-R6nC":0.3345734028221851, "JTpxue-xSqAhGo-AZk-zB1t":0.3504030277488054, "92TVdU-qDJesPN-0lb-JOd3":0.7387694998319805, "0PODnh-IciBdOZ-0CS-oNeL":0.9515905965769644, "KkkW6x-TiemXQw-OiH-dZ9s":0.4082412331999081, "PIs5Aj-g02HRXw-957-GD2z":0.641526116451016, "yJIzuw-au6460e-0Tl-XYEJ":0.7521928530356236, "KHvMCD-OQDL0eX-nqK-TmEt":0.1309616727896826, "6QJJgV-Z3IZ1Rf-wyv-rIJ6":0.7007110387725962, "qA9ycc-sR2qm6P-PtB-AIax":0.4462977655645909, "uDeuEb-B0t0Ljr-dWk-jkC4":0.6904672767407958, "5vPy52-ygN0MMH-UB4-nZQL":0.6057596542200021, "zbbmrQ-pT3uAuU-Kae-HjM5":0.9812657498686279, "3QShHS-7RwUB10-0W2-H4Qy":0.4155760848860853, "PMc4QI-5lNajXU-f8m-RGIi":0.7046420976800288, "O9t3dl-q8YHozj-saR-A3Jm":0.8543344954196586, "k4eH3O-aHnTKY7-ADp-4Vsi":0.2655832454718557, "RA4epe-lWWnOff-bpM-bSR4":0.7523252210222883, "6ysu2R-gSc5dwU-cv0-LqCJ":0.7830899322716732, "tVl3TY-o42NMVO-k3S-iqOY":0.7923823401215799, "NMgTrr-W1RrCvP-Zaf-paL7":0.4686928654756936, "d1CJmF-CeG5asM-xms-1dwN":0.7622908781076493, "N1D30g-zFjiGzI-eHC-Sof4":0.847542878440137, "tOhfKu-Gdtf9Ne-KwA-JdHV":0.4999285217445154, "XLzwK0-6ocGDrS-TtU-wlEI":0.3985354402705095, "XDgZfb-Sxc45Zn-mVO-S2QO":0.05791580337644187, "GQD7a0-fnt9BZs-Kvh-dPbJ":0.663903859916476, "9dJxj9-HFwEQMY-6p9-s8Vt":0.2194407595305434, "1qU9pA-QJGAna9-JoG-H7GS":0.8877401947295382, "rKIkxA-UnGWYSn-0li-ziuB":0.1607906275036466, "tbPazx-IjUrQ8J-NZe-VOPL":0.6809166916797593, "xBpSIv-U6ojkK7-9p5-LviD":0.1195672647379901, "88bnWI-pxrKa7T-n2d-tXk9":0.1956068951787721, "0XviXp-9ksT8s0-fDy-35SW":0.8690659418822626, "e0XauA-GNRALmd-SM2-Y4Gf":0.6840816888752089, "kyvYBk-Bk5M4Xq-gxX-kE1B":0.7744771682336401, "dIiQzS-5sT4ogL-6IV-tLmb":0.0340772833497166, "OlGOyH-dyL1nzj-B2M-z8ir":0.3765608037933722, "zC9Gtn-x8hpfPD-KOu-k31W":0.864392047887076, "qSq3z2-Lpv0YcB-hBq-Sabd":0.1542847609246678, "LSyNyi-tBZUx1l-hAj-mwsx":0.304034328298701, "2c9aTP-hXloMK7-ufH-dgq6":0.1016852552953107, "aXksHO-zARQxfo-sgS-8Bf4":0.5490533082019959, "ioOXAL-eVUF0W8-vZx-ZeYX":0.4528164038481785, "DXUkAP-A7SqnHj-V4U-PJfz":0.3607407447425939, "cnzZXk-AOMepfN-hym-qbDH":0.4587361500592568, "CMlAd6-8FF1yXs-fae-Izfv":0.07555019720825917, "qiXnUv-e2PsJWm-tLF-KpjE":0.9409681065363688, "Gfx3k9-JvXa7Wd-rI1-1e1E":0.7492793312178226} {"name":"r8HXXQM4XHoI", "age":238221053, "tip":"2023-07-26 15:40:37.694000"} - --- !array_max_orc -- -11028 - --- !offsets_check -- -0 [1, 2] [[], [3], null] {"a":1, "b":2} {"s1":"e", "s2":null} -1 [] [] {} \N -2 \N \N \N {"s1":"h", "s2":10} -3 [5, null] [[6, 7], [8, null], null] {"f":1, "g":null} {"s1":null, "s2":9} - --- !map_with_nullable_key -- -\N \N \N \N \N \N \N \N \N test test aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa test 3 4 5.1 6.2 true false -1.2 12.30 -1234.5678 123456789.12340000 -1234567890.12345678 1234567890123456789012.1234567800000000 test2 {"test":"test"} {"test":"test"} {"test":"test"} {3:3} {4:4} {5:5} {6:6} {1:1} {-1.2:-1.2} {12.30:12.30} {-1234.5678:-1234.5678} {123456789.12340000:123456789.12340000} {-1234567890.12345678:-1234567890.12345678} {1234567890123456789012.1234567800000000:1234567890123456789012.1234567800000000} ["test"] [3] [4] [5] [6] [1] ["test"] ["test"] [-1.2] [12.30] [-1234.5678] [123456789.12340000] [-1234567890.12345678] [1234567890123456789012.1234567800000000] {"s_bigint":1} {"test":[{"s_int":1}]} {"struct_field":["1", "2", "3"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":null, "struct_non_nulls_after_nulls2":"some string"} {"struct_field1":null, "struct_field2":"some string", "strict_field3":{"nested_struct_field1":null, "nested_struct_field2":"nested_string2"}} {"k1":"v1", "k2":null, "k3":"v3"} [null, "test"] ["test-1", null, "test-2"] ["test", null] [null, null, null] - --- !date_dict -- -2036-12-28 1898-12-28 2539-12-28 - diff --git a/regression-test/data/external_table_p0/hive/test_external_catalog_hive.out b/regression-test/data/external_table_p0/hive/test_external_catalog_hive.out index 8a104343fc4e10..57367a3bf0999c 100644 --- a/regression-test/data/external_table_p0/hive/test_external_catalog_hive.out +++ b/regression-test/data/external_table_p0/hive/test_external_catalog_hive.out @@ -123,127 +123,3 @@ a126 15 2017-09-13 2009-09-21T04:23:14.309124 2024-03-23 2024-02-01T21:11:09.170 --- !q01 -- -zhangsan 1 -lisi 1 - --- !q02 -- -1 1 -2 1 -3 1 -4 1 - --- !q03 -- -123 china 4 56 sc -234 america 5 67 ls -345 cana 4 56 fy -567 fre 7 89 pa - --- !q04 -- -p_partkey2 p_name2 p_mfgr2 p_brand2 p_type2 p_size2 p_con2 p_r_price2 p_comment2 -p_partkey1 p_name1 p_mfgr1 p_brand1 p_type1 p_size1 p_con1 p_r_price1 p_comment1 -p_partkey0 p_name0 p_mfgr0 p_brand0 p_type0 p_size0 p_con0 p_r_price0 p_comment0 - --- !q05 -- -batchno appsheet_no filedate t_no tano t_name chged_no mob_no2 home_no off_no -off_no home_no mob_no2 chged_no t_name tano t_no filedate appsheet_no batchno - --- !q06 -- -bill_code dates ord_year ord_month ord_quarter on_time - --- !q07 -- -2 - --- !q08 -- -123 zhangsan 12 123.45 2022-01-01 -124 lisi 12 123.45 2022-01-01 -125 lisan 12 123.45 2022-01-02 - --- !q09 -- -a123 12 -a124 13 -a125 14 -a126 15 - --- !par_fields_in_file_orc1 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_parquet1 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_orc2 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_parquet2 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_orc3 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_parquet3 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_orc4 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_parquet4 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_orc5 -- - --- !par_fields_in_file_parquet5 -- - --- !par_fields_in_file_orc1 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_parquet1 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_orc2 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_parquet2 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_orc3 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_parquet3 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_orc4 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_parquet4 -- -1 Alice 100.0 2023 8 -2 Bob 150.0 2023 8 - --- !par_fields_in_file_orc5 -- - --- !par_fields_in_file_parquet5 -- - --- !parquet_adjusted_utc -- -1997-09-21 1999-01-12T15:12:31.235784 -1998-01-12 1993-06-11T11:33:12.356500 -2002-09-29 2001-01-17T21:23:42.120 -2008-08-07 2023-09-23T11:12:17.458 -2009-11-13 2011-11-12T01:23:06.986 -2012-07-08 2023-11-09T20:21:16.321 -2017-09-13 2009-09-21T04:23:14.309124 -2024-03-23 2024-02-01T21:11:09.170 - diff --git a/regression-test/data/external_table_p0/hive/test_external_catalog_hive_partition.out b/regression-test/data/external_table_p0/hive/test_external_catalog_hive_partition.out index 0402feef40e6b5..deda902d300d43 100644 --- a/regression-test/data/external_table_p0/hive/test_external_catalog_hive_partition.out +++ b/regression-test/data/external_table_p0/hive/test_external_catalog_hive_partition.out @@ -119,123 +119,3 @@ -- !q06 -- 2023-01-03T00:00 100 0.3 test3 --- !q01 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N -0.3 test3 2023-01-03T00:00 100 - --- !q02 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N - --- !q03 -- -0.3 test3 2023-01-03T00:00 100 - --- !q04 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 -2023-01-03T00:00 100 0.3 test3 - --- !q05 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 - --- !q06 -- -2023-01-03T00:00 100 0.3 test3 - --- !q01 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N -0.3 test3 2023-01-03T00:00 100 - --- !q02 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N - --- !q03 -- -0.3 test3 2023-01-03T00:00 100 - --- !q04 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 -2023-01-03T00:00 100 0.3 test3 - --- !q05 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 - --- !q06 -- -2023-01-03T00:00 100 0.3 test3 - --- !q01 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N -0.3 test3 2023-01-03T00:00 100 - --- !q02 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N - --- !q03 -- -0.3 test3 2023-01-03T00:00 100 - --- !q04 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 -2023-01-03T00:00 100 0.3 test3 - --- !q05 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 - --- !q06 -- -2023-01-03T00:00 100 0.3 test3 - --- !q01 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N -0.3 test3 2023-01-03T00:00 100 - --- !q02 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N - --- !q03 -- -0.3 test3 2023-01-03T00:00 100 - --- !q04 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 -2023-01-03T00:00 100 0.3 test3 - --- !q05 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 - --- !q06 -- -2023-01-03T00:00 100 0.3 test3 - --- !q01 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N -0.3 test3 2023-01-03T00:00 100 - --- !q02 -- -0.1 test1 2023-01-01T00:00 \N -0.2 test2 2023-01-02T00:00 \N - --- !q03 -- -0.3 test3 2023-01-03T00:00 100 - --- !q04 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 -2023-01-03T00:00 100 0.3 test3 - --- !q05 -- -2023-01-01T00:00 \N 0.1 test1 -2023-01-02T00:00 \N 0.2 test2 - --- !q06 -- -2023-01-03T00:00 100 0.3 test3 - diff --git a/regression-test/data/external_table_p0/hive/test_hive_compress_type.out b/regression-test/data/external_table_p0/hive/test_hive_compress_type.out index ca9ca885c5b854..ee4c9a8f2731ba 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_compress_type.out +++ b/regression-test/data/external_table_p0/hive/test_hive_compress_type.out @@ -1,486 +1,440 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !q21 -- -600005 - --- !q22 -- -1510010 - --- !q23 -- -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 2023-08-21 -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 bzip2 -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 bzip2 -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 deflate -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 deflate -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 gzip -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 gzip -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 lz4 -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 mix -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 mix -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 mix -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 mix -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 plain -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 plain -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 snappy - --- !q31 -- -600005 - --- !q32 -- -1510010 - --- !q33 -- -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 2023-08-21 -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 bzip2 -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 bzip2 -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 deflate -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 deflate -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 gzip -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 gzip -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 lz4 -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 mix -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 mix -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 mix -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 mix -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 plain -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 plain -4611870011201662970 0 HD Tube 5* 1 2014-03-22T05:11:29 2014-03-22 598875 4243808759 92f6fe1be9b9773206d6b63e50feb470 196 2314158381335918424 0 3 3 http://public_search yandex.ru.livemaster 0 0 [] [4,15,333,3912,14512,12818] [18,348,1010] [] 1846 952 29 10 1 0.77 0 0 24 73d7 1 1 0 0 3238011 0 0 0 0 1119 641 157 2014-03-22T19:51:48 0 0 0 0 utf-8 330 0 0 0 7774109565808082252 11274076 0 0 0 0 0 E 2014-03-22T11:54:54 55 2 3 4 6 [105,11,9,88,45,14,98,72,3,925,2193,6,25,1] 3137666015 cc184643699dccab8d5d4af796c47449 -1 -1 -1 nD Tp 0 -1 0 0 81 0 0 0 -1 -1 -1 -1 -1 -1 -1 -1 0 0 07d21f 0 [] 0 15284527577228392792 14270691585016129648 0 0 [] [] [] [] [] \N c1889e2b9ad1e219ed04c0e9624b5139 1404 0 snappy - -- !q42 -- 215 -- !q43 -- 1 100 5 1000000000 10.5 20.75 true First A Alpha 2023-10-06 2023-10-06T14:30 123.45 -1 578 55 2111222273 56.858597 82.38111658179561 true Random C LYDUG 2023-12-17 2023-12-05T13:04:58 1393.11 -1 979 44 10163954251 28.827957 57.56879940298416 true Random Q DNRGE 2023-12-09 2023-12-10T20:21:58 1581.25 +1 578 55 2111222273 56.8586 82.38111658179561 true Random C LYDUG 2023-12-17 2023-12-05T13:04:58 1393.11 +1 979 44 10163954251 28.82796 57.56879940298416 true Random Q DNRGE 2023-12-09 2023-12-10T20:21:58 1581.25 10 1000 50 10000000000 55.25 65.75 false Tenth J Kappa 2023-10-15 2023-10-15T23:30 1012.34 -10 210 26 8549838179 23.438345 73.36477128189287 true Random N VVXIF 2023-11-24 2023-12-13T18:04:58 226.65 -10 386 51 1214815770 13.959902 36.64197990482059 false Random J ORLGI 2023-12-18 2023-11-27T17:13:58 852.62 -10 966 38 2203748112 45.555325 27.908447208440094 true Random W LFAGO 2023-12-14 2023-11-26T20:00:58 1898.68 +10 210 26 8549838179 23.43834 73.36477128189287 true Random N VVXIF 2023-11-24 2023-12-13T18:04:58 226.65 +10 386 51 1214815770 13.9599 36.64197990482059 false Random J ORLGI 2023-12-18 2023-11-27T17:13:58 852.62 +10 966 38 2203748112 45.55532 27.90844720844009 true Random W LFAGO 2023-12-14 2023-11-26T20:00:58 1898.68 100 281 26 3174393241 51.05278 52.09566669589555 false Random F SLDWB 2023-12-14 2023-12-12T07:03:58 798.30 100 289 71 4919981667 66.56684 69.73132704711037 true Random V QOLAP 2023-12-17 2023-12-23T09:38:58 217.05 -11 1100 55 11000000000 60.5 70.0 true Eleventh K Lambda 2023-10-16 2023-10-16T01:45 1123.45 -11 426 67 8473986652 17.942455 71.80682514420877 true Random X FXDUV 2023-12-04 2023-12-22T07:51:58 129.81 -11 441 19 7370044350 74.261696 62.013817404758086 true Random D UYKZA 2023-12-23 2023-12-15T11:49:58 1805.14 -11 487 27 14556302216 85.33334 62.596750833474495 true Random E QMHJD 2023-12-23 2023-12-24T08:30:58 1491.22 -11 770 17 7962512669 12.508753 83.33847413902296 true Random P LHJRA 2023-12-06 2023-12-04T15:48:58 970.51 +11 1100 55 11000000000 60.5 70 true Eleventh K Lambda 2023-10-16 2023-10-16T01:45 1123.45 +11 426 67 8473986652 17.94246 71.80682514420877 true Random X FXDUV 2023-12-04 2023-12-22T07:51:58 129.81 +11 441 19 7370044350 74.2617 62.01381740475809 true Random D UYKZA 2023-12-23 2023-12-15T11:49:58 1805.14 +11 487 27 14556302216 85.33334 62.59675083347449 true Random E QMHJD 2023-12-23 2023-12-24T08:30:58 1491.22 +11 770 17 7962512669 12.50875 83.33847413902296 true Random P LHJRA 2023-12-06 2023-12-04T15:48:58 970.51 12 1200 60 12000000000 65.75 75.25 false Twelfth L Mu 2023-10-17 2023-10-17T02:15 1234.56 -12 751 8 12205294947 23.468674 64.35048302450815 true Random K FCSBV 2023-12-03 2023-12-17T01:10:58 325.26 +12 751 8 12205294947 23.46867 64.35048302450815 true Random K FCSBV 2023-12-03 2023-12-17T01:10:58 325.26 12 782 48 5080583047 75.55138 49.6324463213595 true Random N WYJDW 2023-12-16 2023-12-18T02:58:58 944.42 -12 987 73 1432735571 40.308147 43.5019559828596 true Random S MZUNG 2023-12-07 2023-12-03T13:42:58 215.12 +12 987 73 1432735571 40.30815 43.5019559828596 true Random S MZUNG 2023-12-07 2023-12-03T13:42:58 215.12 13 1300 65 13000000000 70.0 80.5 true Thirteenth M Nu 2023-10-18 2023-10-18T03:30 1345.67 -13 335 39 13869202091 30.426075 39.02304533093442 true Random L AULCC 2023-12-08 2023-12-13T00:26:58 387.97 +13 335 39 13869202091 30.42607 39.02304533093442 true Random L AULCC 2023-12-08 2023-12-13T00:26:58 387.97 13 402 30 10851194313 74.82481 74.90108005771035 false Random F GEMMK 2023-11-27 2023-12-21T15:03:58 1643.55 -13 503 34 6763884255 23.660393 63.9797872103468 true Random S POEBK 2023-12-22 2023-12-23T23:16:58 486.62 -13 696 74 3370487489 84.544014 88.69976219408227 true Random H RTFJI 2023-11-23 2023-11-25T07:32:58 1761.50 -13 745 48 13047949175 51.168613 85.21972389262197 true Random A AYBWQ 2023-12-22 2023-12-22T16:25:58 1192.48 -13 859 65 7433576046 56.136265 34.87823331022725 false Random L CRFUF 2023-12-23 2023-12-12T15:05:58 1037.15 +13 503 34 6763884255 23.66039 63.9797872103468 true Random S POEBK 2023-12-22 2023-12-23T23:16:58 486.62 +13 696 74 3370487489 84.54401 88.69976219408227 true Random H RTFJI 2023-11-23 2023-11-25T07:32:58 1761.50 +13 745 48 13047949175 51.16861 85.21972389262197 true Random A AYBWQ 2023-12-22 2023-12-22T16:25:58 1192.48 +13 859 65 7433576046 56.13626 34.87823331022725 false Random L CRFUF 2023-12-23 2023-12-12T15:05:58 1037.15 14 1400 70 14000000000 75.25 85.75 false Fourteenth N Xi 2023-10-19 2023-10-19T04:45 1456.78 -14 195 17 2370700139 16.777058 64.81793301410002 false Random P IIGRE 2023-12-12 2023-12-14T22:40:58 1678.44 -14 966 65 7828602539 62.430664 68.85873133439297 true Random I VVOQH 2023-12-01 2023-12-06T00:54:58 1300.43 -14 968 16 11314514196 62.509666 33.1841427251225 false Random T WDEVJ 2023-11-24 2023-12-06T17:54:58 431.61 -15 1500 75 15000000000 80.5 90.0 true Fifteenth O Omicron 2023-10-20 2023-10-20T05:15 1567.89 +14 195 17 2370700139 16.77706 64.81793301410002 false Random P IIGRE 2023-12-12 2023-12-14T22:40:58 1678.44 +14 966 65 7828602539 62.43066 68.85873133439297 true Random I VVOQH 2023-12-01 2023-12-06T00:54:58 1300.43 +14 968 16 11314514196 62.50967 33.1841427251225 false Random T WDEVJ 2023-11-24 2023-12-06T17:54:58 431.61 +15 1500 75 15000000000 80.5 90 true Fifteenth O Omicron 2023-10-20 2023-10-20T05:15 1567.89 15 703 67 4284267079 85.38059 91.41088583496226 true Random T PHZRC 2023-12-04 2023-12-08T15:54:58 185.19 -16 135 22 7901304568 43.944805 85.16901944253635 true Random K NUQEP 2023-11-29 2023-11-25T23:42:58 1440.74 +16 135 22 7901304568 43.94481 85.16901944253635 true Random K NUQEP 2023-11-29 2023-11-25T23:42:58 1440.74 16 615 20 12294128025 77.37379 20.42772029677839 true Random U JHPOB 2023-11-30 2023-12-16T14:29:58 1105.33 -17 289 49 13560709243 39.952793 38.245306832599425 true Random Q QEYVY 2023-12-19 2023-12-07T00:35:58 500.19 -17 499 46 11230409207 51.632103 28.811164197154774 false Random V BVLUH 2023-12-13 2023-12-23T17:59:58 1387.62 +17 289 49 13560709243 39.95279 38.24530683259943 true Random Q QEYVY 2023-12-19 2023-12-07T00:35:58 500.19 +17 499 46 11230409207 51.6321 28.81116419715477 false Random V BVLUH 2023-12-13 2023-12-23T17:59:58 1387.62 17 646 62 11234805830 76.40492 67.46425239009778 true Random N REHZC 2023-12-09 2023-11-28T02:06:58 365.15 17 698 55 1807368797 20.17171 43.84496606184709 true Random P SHSJV 2023-12-01 2023-11-25T11:56:58 810.95 17 794 14 8377523030 28.07663 52.3837762020057 false Random E WPMIN 2023-12-03 2023-11-26T04:59:58 239.42 17 913 32 4647929554 78.91502 70.54487265463735 true Random S WFPNS 2023-11-27 2023-11-26T03:29:58 321.45 -18 690 17 1399456103 63.261967 42.964715823771236 true Random R BWSRS 2023-12-13 2023-12-23T08:33:58 1840.02 -18 835 17 14265814864 18.923101 80.53531451138412 true Random V PIKUZ 2023-12-20 2023-12-21T07:39:58 1167.09 -19 917 66 2340946367 89.035675 22.649362455875274 false Random D HWHMU 2023-11-30 2023-12-10T02:36:58 1960.07 -19 993 13 7039833438 79.769066 69.79049291517285 true Random X OFSUV 2023-12-11 2023-12-08T01:46:58 1958.95 +18 690 17 1399456103 63.26197 42.96471582377124 true Random R BWSRS 2023-12-13 2023-12-23T08:33:58 1840.02 +18 835 17 14265814864 18.9231 80.53531451138412 true Random V PIKUZ 2023-12-20 2023-12-21T07:39:58 1167.09 +19 917 66 2340946367 89.03568 22.64936245587527 false Random D HWHMU 2023-11-30 2023-12-10T02:36:58 1960.07 +19 993 13 7039833438 79.76907 69.79049291517285 true Random X OFSUV 2023-12-11 2023-12-08T01:46:58 1958.95 2 200 10 2000000000 15.75 25.5 false Second B Beta 2023-10-07 2023-10-07T15:45 234.56 2 850 75 7075823565 83.65178 62.56093886118189 false Random F RFHAG 2023-11-24 2023-12-03T01:06:58 495.12 2 921 62 8557914543 78.52379 58.6849882881372 false Random D KBXXS 2023-12-07 2023-12-02T22:24:58 1782.88 -2 925 46 6013180177 41.107002 34.86561026061906 true Random L XLLXY 2023-12-06 2023-12-09T14:04:58 1246.26 -20 248 64 7704906572 35.089928 76.69128821479936 true Random T KQOMS 2023-11-30 2023-12-11T01:35:58 1799.26 -22 200 41 12163439252 64.621254 81.68574929661384 true Random U KGVNU 2023-12-20 2023-11-30T14:56:58 1915.47 +2 925 46 6013180177 41.107 34.86561026061906 true Random L XLLXY 2023-12-06 2023-12-09T14:04:58 1246.26 +20 248 64 7704906572 35.08993 76.69128821479936 true Random T KQOMS 2023-11-30 2023-12-11T01:35:58 1799.26 +22 200 41 12163439252 64.62125 81.68574929661384 true Random U KGVNU 2023-12-20 2023-11-30T14:56:58 1915.47 22 235 19 6963606423 65.68033 54.1995295752517 true Random E ENVRH 2023-12-22 2023-11-29T14:42:58 864.89 -23 192 8 5102667616 54.111057 40.85713971600841 false Random J EBXEB 2023-12-13 2023-12-10T11:32:58 1824.12 -27 452 74 4240215371 50.569168 75.68204627611644 true Random G AZOWU 2023-12-01 2023-11-26T06:24:58 201.31 +23 192 8 5102667616 54.11106 40.85713971600841 false Random J EBXEB 2023-12-13 2023-12-10T11:32:58 1824.12 +27 452 74 4240215371 50.56917 75.68204627611644 true Random G AZOWU 2023-12-01 2023-11-26T06:24:58 201.31 27 866 24 5531365994 72.77447 86.96690821165853 false Random S TZPFJ 2023-11-28 2023-12-13T15:31:58 1274.75 -28 655 21 14580233860 12.503378 48.60220286874443 false Random P DUBQQ 2023-12-12 2023-12-03T20:11:58 922.42 -29 157 34 2302882987 51.924015 20.311140937696468 true Random R MBOXJ 2023-12-02 2023-12-03T14:12:58 1620.80 -29 910 52 5544039917 22.179396 46.32732226806482 true Random C TIZAG 2023-11-28 2023-12-14T16:08:58 900.96 -29 923 57 1591814253 68.57371 33.342802789892986 true Random Q ZONGC 2023-12-20 2023-12-13T09:11:58 1465.38 -3 259 74 7422478791 22.291426 75.38227773520089 true Random S VWAXJ 2023-12-01 2023-12-05T21:23:58 1970.57 -3 300 15 3000000000 20.25 30.0 true Third C Gamma 2023-10-08 2023-10-08T16:15 345.67 -3 422 25 5996825874 89.173584 62.758513798505824 false Random Z CDYAO 2023-12-14 2023-12-08T09:27:58 567.23 +28 655 21 14580233860 12.50338 48.60220286874443 false Random P DUBQQ 2023-12-12 2023-12-03T20:11:58 922.42 +29 157 34 2302882987 51.92402 20.31114093769647 true Random R MBOXJ 2023-12-02 2023-12-03T14:12:58 1620.80 +29 910 52 5544039917 22.1794 46.32732226806482 true Random C TIZAG 2023-11-28 2023-12-14T16:08:58 900.96 +29 923 57 1591814253 68.57371 33.34280278989299 true Random Q ZONGC 2023-12-20 2023-12-13T09:11:58 1465.38 +3 259 74 7422478791 22.29143 75.38227773520089 true Random S VWAXJ 2023-12-01 2023-12-05T21:23:58 1970.57 +3 300 15 3000000000 20.25 30 true Third C Gamma 2023-10-08 2023-10-08T16:15 345.67 +3 422 25 5996825874 89.17358 62.75851379850582 false Random Z CDYAO 2023-12-14 2023-12-08T09:27:58 567.23 3 668 60 1942550969 83.43451 87.15906153619602 true Random F QYSRS 2023-12-22 2023-12-10T22:17:58 320.22 -30 292 71 10308444223 63.039078 76.40649540444898 false Random G DRLHY 2023-12-19 2023-12-14T15:32:58 1165.14 -30 572 6 3022031043 57.813908 72.29244668177799 true Random X EHJDN 2023-12-11 2023-12-12T02:44:58 910.38 -30 830 65 12624057029 38.791172 59.72899174862661 false Random A LFPWP 2023-12-03 2023-12-17T00:10:58 1760.62 +30 292 71 10308444223 63.03908 76.40649540444898 false Random G DRLHY 2023-12-19 2023-12-14T15:32:58 1165.14 +30 572 6 3022031043 57.81391 72.29244668177799 true Random X EHJDN 2023-12-11 2023-12-12T02:44:58 910.38 +30 830 65 12624057029 38.79117 59.72899174862661 false Random A LFPWP 2023-12-03 2023-12-17T00:10:58 1760.62 31 395 22 6141426904 88.37914 52.0655270963123 false Random J DRPJV 2023-12-07 2023-11-29T03:15:58 1076.41 -31 990 5 13678786851 15.762894 85.24173385692956 false Random H THGIM 2023-12-14 2023-12-09T01:24:58 1834.37 +31 990 5 13678786851 15.76289 85.24173385692956 false Random H THGIM 2023-12-14 2023-12-09T01:24:58 1834.37 33 198 20 13225406950 67.7327 58.63863378877107 true Random I ZKXRA 2023-12-07 2023-11-29T02:33:58 566.76 33 321 39 12537851805 38.26871 32.6626492245712 true Random S OICCE 2023-12-02 2023-12-19T16:41:58 306.92 -33 916 53 5666674210 57.998173 61.774881852563475 true Random J WJAXA 2023-11-27 2023-12-05T19:58:58 976.13 +33 916 53 5666674210 57.99817 61.77488185256347 true Random J WJAXA 2023-11-27 2023-12-05T19:58:58 976.13 34 145 44 14060350663 73.02436 68.40544929600975 true Random S UUJFP 2023-11-23 2023-12-12T06:08:58 739.45 -34 585 43 1429300527 61.706585 80.88100239373303 false Random O JKJOH 2023-12-17 2023-12-07T11:00:58 468.11 -35 297 75 2468378214 51.353462 34.18114780065386 false Random C HBYZO 2023-12-05 2023-12-09T21:42:58 534.70 +34 585 43 1429300527 61.70658 80.88100239373303 false Random O JKJOH 2023-12-17 2023-12-07T11:00:58 468.11 +35 297 75 2468378214 51.35346 34.18114780065386 false Random C HBYZO 2023-12-05 2023-12-09T21:42:58 534.70 37 438 39 6809169396 83.56728 40.90894521029911 true Random W GXPAY 2023-12-07 2023-12-18T06:35:58 383.18 38 606 57 14585148556 82.67463 79.18300302689997 false Random E RSFUZ 2023-12-16 2023-11-27T18:55:58 970.25 -39 726 50 3865644066 26.225628 28.534393094364418 false Random F NIUCS 2023-12-05 2023-12-04T19:31:58 1953.82 +39 726 50 3865644066 26.22563 28.53439309436442 false Random F NIUCS 2023-12-05 2023-12-04T19:31:58 1953.82 4 122 24 10738473173 81.15482 60.21481394154484 false Random Y PQJRK 2023-12-20 2023-12-09T02:38:58 1467.35 4 400 20 4000000000 25.5 35.25 false Fourth D Delta 2023-10-09 2023-10-09T17:30 456.78 -4 569 72 10560903405 50.255936 47.535145739285184 false Random O NRIRC 2023-12-05 2023-12-01T09:10:58 1986.99 +4 569 72 10560903405 50.25594 47.53514573928518 false Random O NRIRC 2023-12-05 2023-12-01T09:10:58 1986.99 4 682 22 2040832636 60.33469 67.33499498711046 true Random W QUICJ 2023-11-24 2023-12-14T10:17:58 579.56 -40 230 34 10824964541 16.929768 53.812277279703366 false Random F YDQHF 2023-12-14 2023-12-03T17:42:58 1623.79 +40 230 34 10824964541 16.92977 53.81227727970337 false Random F YDQHF 2023-12-14 2023-12-03T17:42:58 1623.79 40 693 69 13276482882 44.35974 82.57845708670757 true Random B RCCSU 2023-11-29 2023-12-01T20:11:58 183.64 -40 914 7 4902128502 19.442041 33.099787387344406 true Random Q KOCWA 2023-11-28 2023-12-21T09:20:58 1824.80 -41 344 34 14536795918 56.660946 84.15108995619764 false Random Q KYLCH 2023-12-10 2023-12-04T08:25:58 1902.09 +40 914 7 4902128502 19.44204 33.09978738734441 true Random Q KOCWA 2023-11-28 2023-12-21T09:20:58 1824.80 +41 344 34 14536795918 56.66095 84.15108995619764 false Random Q KYLCH 2023-12-10 2023-12-04T08:25:58 1902.09 41 599 54 8095449906 22.58196 37.99742597458578 false Random T GTQXP 2023-12-12 2023-12-22T19:08:58 743.46 -41 697 21 1200243566 12.466168 68.57243624557165 true Random U JZGEG 2023-12-03 2023-12-10T04:51:58 1323.88 +41 697 21 1200243566 12.46617 68.57243624557165 true Random U JZGEG 2023-12-03 2023-12-10T04:51:58 1323.88 41 708 64 11745827370 72.84812 35.31028363777645 true Random O WGSQC 2023-12-02 2023-11-25T17:07:58 1666.71 -41 840 65 8988241658 37.428593 42.25992474748068 false Random E HURYX 2023-12-22 2023-12-19T01:55:58 141.89 +41 840 65 8988241658 37.42859 42.25992474748068 false Random E HURYX 2023-12-22 2023-12-19T01:55:58 141.89 42 143 42 3421815721 65.27691 87.91368867538209 true Random S AXGVL 2023-12-06 2023-11-29T07:36:58 575.01 42 178 38 7559404453 69.69449 64.37154501388798 true Random G QUMUN 2023-12-14 2023-12-17T01:37:58 1190.44 -42 192 28 14454791024 35.465202 46.34876515635648 false Random W NQFGR 2023-12-04 2023-11-24T05:02:58 1428.02 +42 192 28 14454791024 35.4652 46.34876515635648 false Random W NQFGR 2023-12-04 2023-11-24T05:02:58 1428.02 42 355 72 11536856285 74.42886 53.49032479461299 false Random I IQZEI 2023-12-10 2023-12-06T07:17:58 1098.14 -43 178 64 6969956763 40.980415 52.998828731408516 true Random C XQHYB 2023-12-11 2023-12-07T23:00:58 257.08 +43 178 64 6969956763 40.98042 52.99882873140852 true Random C XQHYB 2023-12-11 2023-12-07T23:00:58 257.08 43 828 24 12011396947 45.07647 54.2136449479346 true Random E HIDUO 2023-12-02 2023-12-19T01:14:58 233.10 44 219 38 8596488294 73.52956 94.10797854680568 true Random E HMWBI 2023-12-15 2023-12-06T00:51:58 1907.47 -44 694 55 3626514138 62.504086 72.89799265418553 true Random Z JTDVF 2023-12-01 2023-11-29T12:08:58 1769.92 -44 912 63 8534761366 55.993538 50.235171557550416 false Random N OVQRQ 2023-12-08 2023-11-24T03:39:58 264.92 -44 928 7 1939079012 14.426672 68.86451571230457 false Random I EKVWY 2023-12-15 2023-12-09T10:43:58 846.74 -45 455 25 12639246000 47.011307 26.310712594958694 false Random Z GGEUA 2023-11-27 2023-12-01T20:41:58 1698.21 -45 492 43 3870916386 51.069588 42.652270406300794 true Random H JVZTB 2023-12-04 2023-12-09T21:06:58 1517.83 -47 508 48 1456473942 48.488297 20.377955902326608 false Random B CAOEY 2023-11-29 2023-12-10T14:49:58 1865.52 -47 566 50 1426586688 51.278687 40.47151456873397 true Random F YBOSH 2023-11-26 2023-12-15T03:44:58 1806.35 +44 694 55 3626514138 62.50409 72.89799265418553 true Random Z JTDVF 2023-12-01 2023-11-29T12:08:58 1769.92 +44 912 63 8534761366 55.99354 50.23517155755042 false Random N OVQRQ 2023-12-08 2023-11-24T03:39:58 264.92 +44 928 7 1939079012 14.42667 68.86451571230457 false Random I EKVWY 2023-12-15 2023-12-09T10:43:58 846.74 +45 455 25 12639246000 47.01131 26.31071259495869 false Random Z GGEUA 2023-11-27 2023-12-01T20:41:58 1698.21 +45 492 43 3870916386 51.06959 42.65227040630079 true Random H JVZTB 2023-12-04 2023-12-09T21:06:58 1517.83 +47 508 48 1456473942 48.4883 20.37795590232661 false Random B CAOEY 2023-11-29 2023-12-10T14:49:58 1865.52 +47 566 50 1426586688 51.27869 40.47151456873397 true Random F YBOSH 2023-11-26 2023-12-15T03:44:58 1806.35 47 838 73 14910230294 83.69784 82.28901816600579 true Random L SHXYL 2023-11-24 2023-12-05T22:19:58 1062.15 48 898 59 12871187130 10.13838 70.19705104611333 true Random J WFXNN 2023-12-23 2023-12-17T02:53:58 1050.21 -49 165 38 4482178563 34.706547 69.17129468406594 false Random W CPZNY 2023-12-15 2023-11-23T19:56:58 512.60 -49 412 16 8300982793 56.263252 66.07893608061771 false Random K DWWJI 2023-12-08 2023-12-17T11:32:58 1718.54 +49 165 38 4482178563 34.70655 69.17129468406594 false Random W CPZNY 2023-12-15 2023-11-23T19:56:58 512.60 +49 412 16 8300982793 56.26325 66.07893608061771 false Random K DWWJI 2023-12-08 2023-12-17T11:32:58 1718.54 49 511 51 8602055259 88.1686 88.98712207285577 false Random M ZDKEY 2023-12-10 2023-11-25T02:44:58 241.08 -49 568 70 2916596630 79.16303 56.114316916863025 false Random T ILLIU 2023-11-23 2023-12-07T11:05:58 1039.03 +49 568 70 2916596630 79.16303 56.11431691686303 false Random T ILLIU 2023-11-23 2023-12-07T11:05:58 1039.03 5 500 25 5000000000 30.75 40.5 true Fifth E Epsilon 2023-10-10 2023-10-10T18:45 567.89 -5 768 5 4152322228 41.128906 78.60686390712706 false Random J LXKRA 2023-12-05 2023-11-24T18:13:58 1941.98 -5 823 63 13328808917 77.768196 22.87975226738422 false Random F OIYPV 2023-12-11 2023-12-14T06:43:58 1144.38 -5 887 74 4082758600 22.797577 93.28246034891224 false Random V MPPGX 2023-12-01 2023-11-29T01:53:58 510.50 +5 768 5 4152322228 41.12891 78.60686390712706 false Random J LXKRA 2023-12-05 2023-11-24T18:13:58 1941.98 +5 823 63 13328808917 77.7682 22.87975226738422 false Random F OIYPV 2023-12-11 2023-12-14T06:43:58 1144.38 +5 887 74 4082758600 22.79758 93.28246034891224 false Random V MPPGX 2023-12-01 2023-11-29T01:53:58 510.50 50 126 58 4433111715 75.31828 43.28056186824247 false Random H UTDJF 2023-12-19 2023-12-10T08:24:58 368.42 51 778 59 13914307584 27.48499 91.47665081887983 true Random X FGFHK 2023-12-01 2023-12-10T03:24:58 402.63 -51 898 32 13510411411 18.679659 21.406761033351007 false Random L FECUW 2023-12-10 2023-12-14T02:00:58 700.43 -52 811 31 14085958816 51.067017 65.01991893789116 true Random A CODYQ 2023-12-03 2023-12-07T23:25:58 1797.21 +51 898 32 13510411411 18.67966 21.40676103335101 false Random L FECUW 2023-12-10 2023-12-14T02:00:58 700.43 +52 811 31 14085958816 51.06702 65.01991893789116 true Random A CODYQ 2023-12-03 2023-12-07T23:25:58 1797.21 53 505 52 9862728376 58.40501 57.60544454281924 false Random V WYCTZ 2023-11-24 2023-12-20T05:13:58 210.43 53 667 49 10531976747 50.22229 49.64660893042742 false Random K WNRJE 2023-12-04 2023-12-19T14:57:58 680.97 -53 713 14 1464447148 23.474258 45.35056918414047 false Random Q UHMLT 2023-12-10 2023-11-30T02:07:58 286.70 +53 713 14 1464447148 23.47426 45.35056918414047 false Random Q UHMLT 2023-12-10 2023-11-30T02:07:58 286.70 53 715 29 10917905565 41.83069 93.50885201221966 true Random U TRLSY 2023-12-03 2023-11-26T15:13:58 369.72 -54 467 42 13684826428 38.491455 90.10566649802195 true Random M ERFBG 2023-11-24 2023-12-02T16:23:58 211.00 -54 827 55 7054839267 58.555687 25.891004802115663 false Random O ASMLW 2023-12-13 2023-12-20T16:41:58 1369.32 -54 843 34 9547939940 38.66475 36.370944299232434 true Random P NTVIR 2023-12-12 2023-12-02T06:45:58 1628.37 +54 467 42 13684826428 38.49146 90.10566649802195 true Random M ERFBG 2023-11-24 2023-12-02T16:23:58 211.00 +54 827 55 7054839267 58.55569 25.89100480211566 false Random O ASMLW 2023-12-13 2023-12-20T16:41:58 1369.32 +54 843 34 9547939940 38.66475 36.37094429923243 true Random P NTVIR 2023-12-12 2023-12-02T06:45:58 1628.37 55 908 24 13623721787 40.06427 90.85281792731746 false Random B KFZGI 2023-11-27 2023-12-23T18:06:58 1124.95 -55 964 8 14038541765 70.24135 20.034551391620194 false Random J AYXIT 2023-12-13 2023-12-16T19:38:58 1476.73 -57 936 26 12164628867 56.541275 56.276679149397076 true Random O IPHPZ 2023-12-13 2023-11-30T22:36:58 603.68 -59 144 31 6208909394 67.417076 40.59765633709834 true Random D FLWNA 2023-12-12 2023-12-19T06:17:58 1870.24 +55 964 8 14038541765 70.24135 20.03455139162019 false Random J AYXIT 2023-12-13 2023-12-16T19:38:58 1476.73 +57 936 26 12164628867 56.54128 56.27667914939708 true Random O IPHPZ 2023-12-13 2023-11-30T22:36:58 603.68 +59 144 31 6208909394 67.41708 40.59765633709834 true Random D FLWNA 2023-12-12 2023-12-19T06:17:58 1870.24 59 509 50 5501336408 39.94401 73.35770882761237 true Random I PVZNO 2023-12-04 2023-11-27T04:40:58 1177.33 6 600 30 6000000000 35.25 45.75 false Sixth F Zeta 2023-10-11 2023-10-11T19:15 678.90 -60 711 69 1493870104 22.574188 61.30347648465907 false Random E FHKVR 2023-11-27 2023-12-05T11:26:58 1981.61 -60 875 42 14283877167 48.811504 67.0706975606688 true Random P VJOZH 2023-12-06 2023-12-15T05:20:58 781.71 -61 267 61 11407448558 12.877184 42.144845857251944 true Random B NRWNW 2023-11-30 2023-11-25T09:34:58 859.85 -61 414 63 14506877706 12.540966 58.04557426323987 false Random H NUOAD 2023-12-10 2023-12-06T22:52:58 780.50 -62 451 50 12304139502 51.151623 22.46754141558852 false Random C SRRSV 2023-12-08 2023-12-20T02:48:58 1352.65 -62 793 46 7308804595 39.766644 48.88672198076526 true Random V TPENZ 2023-11-26 2023-12-23T17:51:58 388.46 +60 711 69 1493870104 22.57419 61.30347648465907 false Random E FHKVR 2023-11-27 2023-12-05T11:26:58 1981.61 +60 875 42 14283877167 48.8115 67.0706975606688 true Random P VJOZH 2023-12-06 2023-12-15T05:20:58 781.71 +61 267 61 11407448558 12.87718 42.14484585725194 true Random B NRWNW 2023-11-30 2023-11-25T09:34:58 859.85 +61 414 63 14506877706 12.54097 58.04557426323987 false Random H NUOAD 2023-12-10 2023-12-06T22:52:58 780.50 +62 451 50 12304139502 51.15162 22.46754141558852 false Random C SRRSV 2023-12-08 2023-12-20T02:48:58 1352.65 +62 793 46 7308804595 39.76664 48.88672198076526 true Random V TPENZ 2023-11-26 2023-12-23T17:51:58 388.46 63 112 75 12197306353 85.90137 43.48931389222043 false Random C KKAIT 2023-11-27 2023-12-23T04:23:58 1954.90 -63 383 35 5161212745 39.455276 52.33267523851794 false Random X TMYMC 2023-11-29 2023-12-10T09:09:58 1442.54 -63 410 33 1767102777 72.260124 56.971483381024896 false Random B QXNSM 2023-12-12 2023-12-19T22:57:58 1660.73 -64 479 20 1710421528 53.324104 33.55443503561635 false Random Q ONZRK 2023-12-09 2023-12-01T22:29:58 252.13 +63 383 35 5161212745 39.45528 52.33267523851794 false Random X TMYMC 2023-11-29 2023-12-10T09:09:58 1442.54 +63 410 33 1767102777 72.26012 56.9714833810249 false Random B QXNSM 2023-12-12 2023-12-19T22:57:58 1660.73 +64 479 20 1710421528 53.3241 33.55443503561635 false Random Q ONZRK 2023-12-09 2023-12-01T22:29:58 252.13 64 678 14 13681447851 74.83621 36.94143092647816 true Random J KELFB 2023-12-01 2023-12-07T18:14:58 308.26 -64 719 36 1224510454 64.237434 86.05689694804887 true Random E ZVQPU 2023-11-30 2023-12-03T04:56:58 1879.25 -64 822 26 1154241961 52.165447 26.779469377773403 true Random E YWNAD 2023-12-08 2023-12-19T19:08:58 731.15 -65 571 24 10523050555 45.865078 70.80680527390149 true Random Y DILBW 2023-12-17 2023-11-25T22:41:58 859.30 -66 306 5 14448160602 44.642223 50.24249889525751 false Random X OASEB 2023-12-11 2023-11-27T00:16:58 1345.69 -66 521 30 7757576974 69.440155 92.3562810104632 false Random H SSOCR 2023-12-19 2023-11-30T06:51:58 913.34 -67 484 65 10817432713 62.168163 77.02869166077757 true Random K SAJMG 2023-12-19 2023-12-14T19:47:58 488.01 -68 266 31 8183454755 69.19586 23.139304803938643 false Random S STCBM 2023-11-26 2023-12-22T13:42:58 1722.37 -68 554 33 3525526216 29.078024 29.6567390059356 false Random Y EUGOF 2023-11-23 2023-12-15T10:33:58 395.41 -68 591 60 4813122821 33.210274 54.464145718507616 false Random X EXROI 2023-12-07 2023-12-07T00:39:58 290.11 +64 719 36 1224510454 64.23743 86.05689694804887 true Random E ZVQPU 2023-11-30 2023-12-03T04:56:58 1879.25 +64 822 26 1154241961 52.16545 26.7794693777734 true Random E YWNAD 2023-12-08 2023-12-19T19:08:58 731.15 +65 571 24 10523050555 45.86508 70.80680527390149 true Random Y DILBW 2023-12-17 2023-11-25T22:41:58 859.30 +66 306 5 14448160602 44.64222 50.24249889525751 false Random X OASEB 2023-12-11 2023-11-27T00:16:58 1345.69 +66 521 30 7757576974 69.44016 92.35628101046321 false Random H SSOCR 2023-12-19 2023-11-30T06:51:58 913.34 +67 484 65 10817432713 62.16816 77.02869166077757 true Random K SAJMG 2023-12-19 2023-12-14T19:47:58 488.01 +68 266 31 8183454755 69.19586 23.13930480393864 false Random S STCBM 2023-11-26 2023-12-22T13:42:58 1722.37 +68 554 33 3525526216 29.07802 29.6567390059356 false Random Y EUGOF 2023-11-23 2023-12-15T10:33:58 395.41 +68 591 60 4813122821 33.21027 54.46414571850762 false Random X EXROI 2023-12-07 2023-12-07T00:39:58 290.11 68 756 63 5416393421 66.41538 76.32820339134415 false Random Y CUNAL 2023-12-23 2023-12-14T22:49:58 1109.25 -68 922 13 11664232196 72.683266 37.9910331525765 false Random W PPWBB 2023-11-26 2023-12-10T22:54:58 1968.89 -68 947 60 7257499958 45.661217 77.42577781358565 false Random F ENQGA 2023-11-24 2023-11-29T07:33:58 319.99 -69 416 14 7702410607 31.638903 89.5793904314531 true Random C URQMU 2023-11-25 2023-11-30T15:17:58 1379.22 +68 922 13 11664232196 72.68327 37.9910331525765 false Random W PPWBB 2023-11-26 2023-12-10T22:54:58 1968.89 +68 947 60 7257499958 45.66122 77.42577781358565 false Random F ENQGA 2023-11-24 2023-11-29T07:33:58 319.99 +69 416 14 7702410607 31.6389 89.57939043145311 true Random C URQMU 2023-11-25 2023-11-30T15:17:58 1379.22 7 340 50 8934567449 83.79683 35.39446967734915 false Random L CWYFN 2023-12-05 2023-12-23T02:26:58 806.15 -7 700 35 7000000000 40.5 50.0 true Seventh G Eta 2023-10-12 2023-10-12T20:30 789.01 +7 700 35 7000000000 40.5 50 true Seventh G Eta 2023-10-12 2023-10-12T20:30 789.01 7 969 62 3451343234 57.17074 56.74513811095188 false Random G OWDSC 2023-12-19 2023-12-11T17:17:58 1874.22 -70 231 67 4547989149 35.103123 51.93622592177748 true Random V ZBCVY 2023-11-29 2023-12-22T11:41:58 1749.60 -70 421 23 3153379289 27.412096 79.32006404438445 false Random L VLJWK 2023-12-04 2023-12-12T05:31:58 1163.35 +70 231 67 4547989149 35.10312 51.93622592177748 true Random V ZBCVY 2023-11-29 2023-12-22T11:41:58 1749.60 +70 421 23 3153379289 27.4121 79.32006404438445 false Random L VLJWK 2023-12-04 2023-12-12T05:31:58 1163.35 70 751 56 7828222634 52.8313 55.7263634552559 true Random B TFHMH 2023-11-30 2023-12-24T12:22:58 1166.13 -71 452 25 4464808420 18.155642 61.988641984596185 false Random K YXFVY 2023-12-15 2023-12-08T04:58:58 514.74 -71 594 26 1024634104 62.92234 37.216752731371386 true Random J SPUWU 2023-12-04 2023-12-23T08:50:58 779.97 -72 377 11 3042707243 55.289066 53.72552524152444 true Random Q BAPHV 2023-12-06 2023-11-30T07:14:58 119.39 -73 866 49 4618070115 46.803646 91.41305051885227 true Random H ROYYF 2023-12-07 2023-12-01T10:28:58 1817.67 -74 670 60 4783926122 23.513939 91.24357097091087 true Random Y YFPMC 2023-12-23 2023-12-22T22:29:58 943.62 -75 368 73 6944888766 31.500992 56.88267149430107 false Random H LEXKZ 2023-12-21 2023-12-14T01:12:58 443.91 +71 452 25 4464808420 18.15564 61.98864198459619 false Random K YXFVY 2023-12-15 2023-12-08T04:58:58 514.74 +71 594 26 1024634104 62.92234 37.21675273137139 true Random J SPUWU 2023-12-04 2023-12-23T08:50:58 779.97 +72 377 11 3042707243 55.28907 53.72552524152444 true Random Q BAPHV 2023-12-06 2023-11-30T07:14:58 119.39 +73 866 49 4618070115 46.80365 91.41305051885227 true Random H ROYYF 2023-12-07 2023-12-01T10:28:58 1817.67 +74 670 60 4783926122 23.51394 91.24357097091087 true Random Y YFPMC 2023-12-23 2023-12-22T22:29:58 943.62 +75 368 73 6944888766 31.50099 56.88267149430107 false Random H LEXKZ 2023-12-21 2023-12-14T01:12:58 443.91 76 410 20 10425110604 66.26356 92.68329033006493 false Random L JHFYD 2023-11-23 2023-11-29T10:34:58 867.56 -76 504 70 14161652666 58.071503 67.99111956708262 true Random Y HAVCK 2023-11-27 2023-12-14T16:08:58 1864.98 +76 504 70 14161652666 58.0715 67.99111956708262 true Random Y HAVCK 2023-11-27 2023-12-14T16:08:58 1864.98 77 131 19 2964167114 33.23181 53.35246738882714 false Random G AHGFO 2023-12-19 2023-12-01T10:11:58 1837.90 -77 165 36 12887722637 19.729382 45.61157603163882 true Random S OZOLB 2023-12-02 2023-12-03T05:07:58 1576.79 -79 314 17 6823498005 22.562634 72.70049796639023 true Random K FPSNZ 2023-12-07 2023-12-15T11:52:58 211.50 +77 165 36 12887722637 19.72938 45.61157603163882 true Random S OZOLB 2023-12-02 2023-12-03T05:07:58 1576.79 +79 314 17 6823498005 22.56263 72.70049796639023 true Random K FPSNZ 2023-12-07 2023-12-15T11:52:58 211.50 8 550 48 13655992126 52.90345 51.35114230137935 false Random X JTVSE 2023-12-13 2023-12-15T03:49:58 361.55 8 800 40 8000000000 45.75 55.25 false Eighth H Theta 2023-10-13 2023-10-13T21:45 890.12 8 866 37 13672147880 81.28999 67.66548594336737 false Random H QDJIM 2023-12-14 2023-12-17T18:44:58 1112.05 -80 267 57 8797946135 35.604717 80.51381110359165 false Random K KQTEX 2023-12-09 2023-12-13T06:19:58 1769.15 -80 815 19 14529289205 19.769405 37.37008094684765 true Random Z WLALH 2023-12-11 2023-12-14T03:24:58 479.38 +80 267 57 8797946135 35.60472 80.51381110359165 false Random K KQTEX 2023-12-09 2023-12-13T06:19:58 1769.15 +80 815 19 14529289205 19.76941 37.37008094684765 true Random Z WLALH 2023-12-11 2023-12-14T03:24:58 479.38 81 726 66 9327218218 81.50363 39.9702863173827 true Random X WODRP 2023-11-28 2023-12-23T13:25:58 561.98 82 107 51 1358006007 78.36581 46.09413324325159 true Random C IPNQU 2023-12-01 2023-12-14T05:41:58 417.17 82 133 60 4616538638 88.8813 30.82745983013354 true Random W KPIJE 2023-12-20 2023-12-01T07:57:58 583.41 -82 531 44 10642962933 26.818586 23.851865471979615 false Random F NMQOD 2023-12-13 2023-12-18T19:34:58 861.78 +82 531 44 10642962933 26.81859 23.85186547197961 false Random F NMQOD 2023-12-13 2023-12-18T19:34:58 861.78 82 603 60 9083469993 81.24088 44.46228092092543 true Random Y WTQGU 2023-11-30 2023-11-28T13:18:58 1448.45 82 982 62 8955063933 81.2855 78.30439669511465 true Random J SOCOT 2023-12-02 2023-12-02T21:17:58 814.60 -83 700 46 4569093424 50.063602 47.75811273142146 false Random R TEGAY 2023-12-19 2023-12-07T06:46:58 760.22 -84 427 60 9035762847 81.971306 28.37315065501099 true Random L FETYF 2023-12-01 2023-11-24T15:00:58 1267.12 +83 700 46 4569093424 50.0636 47.75811273142146 false Random R TEGAY 2023-12-19 2023-12-07T06:46:58 760.22 +84 427 60 9035762847 81.97131 28.37315065501099 true Random L FETYF 2023-12-01 2023-11-24T15:00:58 1267.12 85 375 63 6797318130 85.47522 58.16330728665678 true Random E UNZLS 2023-12-01 2023-12-04T05:17:58 1949.48 -85 845 42 2373712244 74.551315 79.15491248184088 false Random B QJRKO 2023-11-29 2023-12-04T09:20:58 317.17 -85 873 18 7233488476 33.83051 31.655950581225508 false Random N RJTIB 2023-11-23 2023-12-11T15:07:58 1249.52 -86 398 27 13222936963 20.387327 44.51255195842424 true Random T ZCRFI 2023-12-21 2023-12-23T12:04:58 1801.53 +85 845 42 2373712244 74.55132 79.15491248184088 false Random B QJRKO 2023-11-29 2023-12-04T09:20:58 317.17 +85 873 18 7233488476 33.83051 31.65595058122551 false Random N RJTIB 2023-11-23 2023-12-11T15:07:58 1249.52 +86 398 27 13222936963 20.38733 44.51255195842424 true Random T ZCRFI 2023-12-21 2023-12-23T12:04:58 1801.53 86 662 53 8875065706 28.64778 30.6775849729486 false Random N YNQAY 2023-12-15 2023-11-24T21:56:58 1108.35 -86 728 18 13390353484 61.060482 87.44751616093882 false Random J BUCVI 2023-12-07 2023-12-14T23:00:58 1611.17 -86 998 74 11080891106 82.568756 32.0122101203062 true Random K VAAMT 2023-12-23 2023-12-01T10:14:58 1708.39 +86 728 18 13390353484 61.06048 87.44751616093882 false Random J BUCVI 2023-12-07 2023-12-14T23:00:58 1611.17 +86 998 74 11080891106 82.56876 32.0122101203062 true Random K VAAMT 2023-12-23 2023-12-01T10:14:58 1708.39 87 145 64 9022533179 37.80205 63.26081178595084 true Random T PEOPK 2023-12-08 2023-12-07T17:41:58 1167.05 -87 641 64 4786767059 14.765089 70.8793353664754 false Random W SQHGN 2023-12-12 2023-12-24T01:19:58 1316.61 -88 274 41 14108849690 73.74919 42.625751442467404 true Random X BVRFA 2023-12-01 2023-11-25T14:32:58 515.18 -88 728 59 8439434199 30.372904 59.410283344764366 false Random F JODWY 2023-12-04 2023-12-01T07:57:58 1753.88 +87 641 64 4786767059 14.76509 70.8793353664754 false Random W SQHGN 2023-12-12 2023-12-24T01:19:58 1316.61 +88 274 41 14108849690 73.74919 42.6257514424674 true Random X BVRFA 2023-12-01 2023-11-25T14:32:58 515.18 +88 728 59 8439434199 30.3729 59.41028334476437 false Random F JODWY 2023-12-04 2023-12-01T07:57:58 1753.88 88 765 69 9753682777 83.42646 25.99260711248508 true Random M MEJAX 2023-11-25 2023-12-20T09:21:58 1647.22 -89 129 64 6400162051 67.910965 80.48074661432221 true Random Y ZXJWQ 2023-12-16 2023-12-19T10:23:58 1882.65 -89 377 22 14340881803 32.61157 82.5503801214006 false Random K ACYZU 2023-12-01 2023-11-27T02:05:58 672.13 -89 964 41 12706120446 69.484116 32.39048200771184 true Random J IIRNY 2023-12-16 2023-11-29T01:54:58 1298.71 -9 113 7 6162580854 11.346889 46.82839094332704 false Random A SJTAF 2023-12-14 2023-11-23T18:27:58 1610.49 +89 129 64 6400162051 67.91096 80.48074661432221 true Random Y ZXJWQ 2023-12-16 2023-12-19T10:23:58 1882.65 +89 377 22 14340881803 32.61157 82.55038012140059 false Random K ACYZU 2023-12-01 2023-11-27T02:05:58 672.13 +89 964 41 12706120446 69.48412 32.39048200771184 true Random J IIRNY 2023-12-16 2023-11-29T01:54:58 1298.71 +9 113 7 6162580854 11.34689 46.82839094332704 false Random A SJTAF 2023-12-14 2023-11-23T18:27:58 1610.49 9 268 59 8149280252 86.66627 70.91298799618343 false Random E PVKYK 2023-12-21 2023-11-25T00:28:58 263.17 9 900 45 9000000000 50.0 60.5 true Ninth I Iota 2023-10-14 2023-10-14T22:15 901.23 9 907 24 6113036809 66.06377 50.26485838775805 true Random X XLPOL 2023-11-23 2023-12-02T09:03:58 256.61 90 391 26 12874761259 21.49042 53.46850617467312 true Random Q QTJPE 2023-12-17 2023-12-03T17:40:58 748.05 -91 389 11 14784237986 11.174142 27.692284427565397 true Random P DYILB 2023-12-14 2023-12-21T11:07:58 1175.73 +91 389 11 14784237986 11.17414 27.6922844275654 true Random P DYILB 2023-12-14 2023-12-21T11:07:58 1175.73 91 528 68 14588592231 77.4651 88.92064181463138 false Random U JXZUA 2023-12-16 2023-12-21T02:28:58 1834.07 -92 344 29 5182139341 31.653255 44.26814517218887 true Random F NGHOS 2023-12-06 2023-12-09T21:25:58 1291.06 -93 887 20 13555948969 70.57364 32.621532934876804 false Random D SPMEK 2023-11-26 2023-12-20T18:11:58 258.86 -94 216 49 8773264156 81.617195 43.03983700523827 true Random D VHWYT 2023-12-13 2023-11-30T07:03:58 1178.27 -94 693 60 4818659234 26.04229 83.2975107272106 true Random B ENSQO 2023-12-22 2023-12-12T06:08:58 1283.81 -95 560 62 1389447643 19.202044 85.46518830161321 true Random S LQRRB 2023-12-16 2023-12-12T06:12:58 445.65 -96 595 72 11506136303 21.917727 74.74561804277158 true Random T SPLKA 2023-12-02 2023-11-30T00:39:58 1693.61 -96 637 39 5516035994 55.90832 60.522041012562816 true Random O YPETL 2023-12-02 2023-11-28T02:47:58 1175.16 -97 415 74 10346322649 21.667427 46.58901867647463 false Random R KWFOF 2023-12-21 2023-11-27T12:18:58 1157.72 +92 344 29 5182139341 31.65326 44.26814517218887 true Random F NGHOS 2023-12-06 2023-12-09T21:25:58 1291.06 +93 887 20 13555948969 70.57364 32.6215329348768 false Random D SPMEK 2023-11-26 2023-12-20T18:11:58 258.86 +94 216 49 8773264156 81.6172 43.03983700523827 true Random D VHWYT 2023-12-13 2023-11-30T07:03:58 1178.27 +94 693 60 4818659234 26.04229 83.29751072721059 true Random B ENSQO 2023-12-22 2023-12-12T06:08:58 1283.81 +95 560 62 1389447643 19.20204 85.46518830161321 true Random S LQRRB 2023-12-16 2023-12-12T06:12:58 445.65 +96 595 72 11506136303 21.91773 74.74561804277158 true Random T SPLKA 2023-12-02 2023-11-30T00:39:58 1693.61 +96 637 39 5516035994 55.90832 60.52204101256282 true Random O YPETL 2023-12-02 2023-11-28T02:47:58 1175.16 +97 415 74 10346322649 21.66743 46.58901867647463 false Random R KWFOF 2023-12-21 2023-11-27T12:18:58 1157.72 97 839 60 14818779777 46.17389 68.98285340004992 false Random W HMFPU 2023-12-01 2023-12-04T08:41:58 1683.48 -98 228 65 4782017237 55.10206 31.414570993700565 true Random P EOIFT 2023-12-07 2023-12-15T08:12:58 137.49 -99 632 39 8911195323 74.581276 78.2764804276292 false Random Q WTQCL 2023-12-02 2023-12-05T09:18:58 200.21 +98 228 65 4782017237 55.10206 31.41457099370056 true Random P EOIFT 2023-12-07 2023-12-15T08:12:58 137.49 +99 632 39 8911195323 74.58128 78.2764804276292 false Random Q WTQCL 2023-12-02 2023-12-05T09:18:58 200.21 -- !q44 -- -17 289 49 13560709243 39.952793 38.245306832599425 true Random Q QEYVY 2023-12-19 2023-12-07T00:35:58 500.19 -17 499 46 11230409207 51.632103 28.811164197154774 false Random V BVLUH 2023-12-13 2023-12-23T17:59:58 1387.62 +17 289 49 13560709243 39.95279 38.24530683259943 true Random Q QEYVY 2023-12-19 2023-12-07T00:35:58 500.19 +17 499 46 11230409207 51.6321 28.81116419715477 false Random V BVLUH 2023-12-13 2023-12-23T17:59:58 1387.62 17 646 62 11234805830 76.40492 67.46425239009778 true Random N REHZC 2023-12-09 2023-11-28T02:06:58 365.15 17 698 55 1807368797 20.17171 43.84496606184709 true Random P SHSJV 2023-12-01 2023-11-25T11:56:58 810.95 17 794 14 8377523030 28.07663 52.3837762020057 false Random E WPMIN 2023-12-03 2023-11-26T04:59:58 239.42 17 913 32 4647929554 78.91502 70.54487265463735 true Random S WFPNS 2023-11-27 2023-11-26T03:29:58 321.45 -- !q45 -- -11 1100 55 11000000000 60.5 70.0 true Eleventh K Lambda 2023-10-16 2023-10-16T01:45 1123.45 -11 487 27 14556302216 85.33334 62.596750833474495 true Random E QMHJD 2023-12-23 2023-12-24T08:30:58 1491.22 +11 1100 55 11000000000 60.5 70 true Eleventh K Lambda 2023-10-16 2023-10-16T01:45 1123.45 +11 487 27 14556302216 85.33334 62.59675083347449 true Random E QMHJD 2023-12-23 2023-12-24T08:30:58 1491.22 12 1200 60 12000000000 65.75 75.25 false Twelfth L Mu 2023-10-17 2023-10-17T02:15 1234.56 -12 751 8 12205294947 23.468674 64.35048302450815 true Random K FCSBV 2023-12-03 2023-12-17T01:10:58 325.26 +12 751 8 12205294947 23.46867 64.35048302450815 true Random K FCSBV 2023-12-03 2023-12-17T01:10:58 325.26 13 1300 65 13000000000 70.0 80.5 true Thirteenth M Nu 2023-10-18 2023-10-18T03:30 1345.67 -13 335 39 13869202091 30.426075 39.02304533093442 true Random L AULCC 2023-12-08 2023-12-13T00:26:58 387.97 +13 335 39 13869202091 30.42607 39.02304533093442 true Random L AULCC 2023-12-08 2023-12-13T00:26:58 387.97 13 402 30 10851194313 74.82481 74.90108005771035 false Random F GEMMK 2023-11-27 2023-12-21T15:03:58 1643.55 -13 745 48 13047949175 51.168613 85.21972389262197 true Random A AYBWQ 2023-12-22 2023-12-22T16:25:58 1192.48 +13 745 48 13047949175 51.16861 85.21972389262197 true Random A AYBWQ 2023-12-22 2023-12-22T16:25:58 1192.48 14 1400 70 14000000000 75.25 85.75 false Fourteenth N Xi 2023-10-19 2023-10-19T04:45 1456.78 -14 968 16 11314514196 62.509666 33.1841427251225 false Random T WDEVJ 2023-11-24 2023-12-06T17:54:58 431.61 -15 1500 75 15000000000 80.5 90.0 true Fifteenth O Omicron 2023-10-20 2023-10-20T05:15 1567.89 +14 968 16 11314514196 62.50967 33.1841427251225 false Random T WDEVJ 2023-11-24 2023-12-06T17:54:58 431.61 +15 1500 75 15000000000 80.5 90 true Fifteenth O Omicron 2023-10-20 2023-10-20T05:15 1567.89 16 615 20 12294128025 77.37379 20.42772029677839 true Random U JHPOB 2023-11-30 2023-12-16T14:29:58 1105.33 -17 289 49 13560709243 39.952793 38.245306832599425 true Random Q QEYVY 2023-12-19 2023-12-07T00:35:58 500.19 -17 499 46 11230409207 51.632103 28.811164197154774 false Random V BVLUH 2023-12-13 2023-12-23T17:59:58 1387.62 +17 289 49 13560709243 39.95279 38.24530683259943 true Random Q QEYVY 2023-12-19 2023-12-07T00:35:58 500.19 +17 499 46 11230409207 51.6321 28.81116419715477 false Random V BVLUH 2023-12-13 2023-12-23T17:59:58 1387.62 17 646 62 11234805830 76.40492 67.46425239009778 true Random N REHZC 2023-12-09 2023-11-28T02:06:58 365.15 -18 835 17 14265814864 18.923101 80.53531451138412 true Random V PIKUZ 2023-12-20 2023-12-21T07:39:58 1167.09 -22 200 41 12163439252 64.621254 81.68574929661384 true Random U KGVNU 2023-12-20 2023-11-30T14:56:58 1915.47 -28 655 21 14580233860 12.503378 48.60220286874443 false Random P DUBQQ 2023-12-12 2023-12-03T20:11:58 922.42 -30 830 65 12624057029 38.791172 59.72899174862661 false Random A LFPWP 2023-12-03 2023-12-17T00:10:58 1760.62 -31 990 5 13678786851 15.762894 85.24173385692956 false Random H THGIM 2023-12-14 2023-12-09T01:24:58 1834.37 +18 835 17 14265814864 18.9231 80.53531451138412 true Random V PIKUZ 2023-12-20 2023-12-21T07:39:58 1167.09 +22 200 41 12163439252 64.62125 81.68574929661384 true Random U KGVNU 2023-12-20 2023-11-30T14:56:58 1915.47 +28 655 21 14580233860 12.50338 48.60220286874443 false Random P DUBQQ 2023-12-12 2023-12-03T20:11:58 922.42 +30 830 65 12624057029 38.79117 59.72899174862661 false Random A LFPWP 2023-12-03 2023-12-17T00:10:58 1760.62 +31 990 5 13678786851 15.76289 85.24173385692956 false Random H THGIM 2023-12-14 2023-12-09T01:24:58 1834.37 33 198 20 13225406950 67.7327 58.63863378877107 true Random I ZKXRA 2023-12-07 2023-11-29T02:33:58 566.76 33 321 39 12537851805 38.26871 32.6626492245712 true Random S OICCE 2023-12-02 2023-12-19T16:41:58 306.92 34 145 44 14060350663 73.02436 68.40544929600975 true Random S UUJFP 2023-11-23 2023-12-12T06:08:58 739.45 38 606 57 14585148556 82.67463 79.18300302689997 false Random E RSFUZ 2023-12-16 2023-11-27T18:55:58 970.25 4 122 24 10738473173 81.15482 60.21481394154484 false Random Y PQJRK 2023-12-20 2023-12-09T02:38:58 1467.35 -40 230 34 10824964541 16.929768 53.812277279703366 false Random F YDQHF 2023-12-14 2023-12-03T17:42:58 1623.79 +40 230 34 10824964541 16.92977 53.81227727970337 false Random F YDQHF 2023-12-14 2023-12-03T17:42:58 1623.79 40 693 69 13276482882 44.35974 82.57845708670757 true Random B RCCSU 2023-11-29 2023-12-01T20:11:58 183.64 -41 344 34 14536795918 56.660946 84.15108995619764 false Random Q KYLCH 2023-12-10 2023-12-04T08:25:58 1902.09 +41 344 34 14536795918 56.66095 84.15108995619764 false Random Q KYLCH 2023-12-10 2023-12-04T08:25:58 1902.09 41 708 64 11745827370 72.84812 35.31028363777645 true Random O WGSQC 2023-12-02 2023-11-25T17:07:58 1666.71 -42 192 28 14454791024 35.465202 46.34876515635648 false Random W NQFGR 2023-12-04 2023-11-24T05:02:58 1428.02 +42 192 28 14454791024 35.4652 46.34876515635648 false Random W NQFGR 2023-12-04 2023-11-24T05:02:58 1428.02 42 355 72 11536856285 74.42886 53.49032479461299 false Random I IQZEI 2023-12-10 2023-12-06T07:17:58 1098.14 43 828 24 12011396947 45.07647 54.2136449479346 true Random E HIDUO 2023-12-02 2023-12-19T01:14:58 233.10 -45 455 25 12639246000 47.011307 26.310712594958694 false Random Z GGEUA 2023-11-27 2023-12-01T20:41:58 1698.21 +45 455 25 12639246000 47.01131 26.31071259495869 false Random Z GGEUA 2023-11-27 2023-12-01T20:41:58 1698.21 47 838 73 14910230294 83.69784 82.28901816600579 true Random L SHXYL 2023-11-24 2023-12-05T22:19:58 1062.15 48 898 59 12871187130 10.13838 70.19705104611333 true Random J WFXNN 2023-12-23 2023-12-17T02:53:58 1050.21 -5 823 63 13328808917 77.768196 22.87975226738422 false Random F OIYPV 2023-12-11 2023-12-14T06:43:58 1144.38 +5 823 63 13328808917 77.7682 22.87975226738422 false Random F OIYPV 2023-12-11 2023-12-14T06:43:58 1144.38 51 778 59 13914307584 27.48499 91.47665081887983 true Random X FGFHK 2023-12-01 2023-12-10T03:24:58 402.63 -51 898 32 13510411411 18.679659 21.406761033351007 false Random L FECUW 2023-12-10 2023-12-14T02:00:58 700.43 -52 811 31 14085958816 51.067017 65.01991893789116 true Random A CODYQ 2023-12-03 2023-12-07T23:25:58 1797.21 +51 898 32 13510411411 18.67966 21.40676103335101 false Random L FECUW 2023-12-10 2023-12-14T02:00:58 700.43 +52 811 31 14085958816 51.06702 65.01991893789116 true Random A CODYQ 2023-12-03 2023-12-07T23:25:58 1797.21 53 715 29 10917905565 41.83069 93.50885201221966 true Random U TRLSY 2023-12-03 2023-11-26T15:13:58 369.72 -54 467 42 13684826428 38.491455 90.10566649802195 true Random M ERFBG 2023-11-24 2023-12-02T16:23:58 211.00 +54 467 42 13684826428 38.49146 90.10566649802195 true Random M ERFBG 2023-11-24 2023-12-02T16:23:58 211.00 55 908 24 13623721787 40.06427 90.85281792731746 false Random B KFZGI 2023-11-27 2023-12-23T18:06:58 1124.95 -55 964 8 14038541765 70.24135 20.034551391620194 false Random J AYXIT 2023-12-13 2023-12-16T19:38:58 1476.73 -57 936 26 12164628867 56.541275 56.276679149397076 true Random O IPHPZ 2023-12-13 2023-11-30T22:36:58 603.68 -60 875 42 14283877167 48.811504 67.0706975606688 true Random P VJOZH 2023-12-06 2023-12-15T05:20:58 781.71 -61 267 61 11407448558 12.877184 42.144845857251944 true Random B NRWNW 2023-11-30 2023-11-25T09:34:58 859.85 -61 414 63 14506877706 12.540966 58.04557426323987 false Random H NUOAD 2023-12-10 2023-12-06T22:52:58 780.50 -62 451 50 12304139502 51.151623 22.46754141558852 false Random C SRRSV 2023-12-08 2023-12-20T02:48:58 1352.65 +55 964 8 14038541765 70.24135 20.03455139162019 false Random J AYXIT 2023-12-13 2023-12-16T19:38:58 1476.73 +57 936 26 12164628867 56.54128 56.27667914939708 true Random O IPHPZ 2023-12-13 2023-11-30T22:36:58 603.68 +60 875 42 14283877167 48.8115 67.0706975606688 true Random P VJOZH 2023-12-06 2023-12-15T05:20:58 781.71 +61 267 61 11407448558 12.87718 42.14484585725194 true Random B NRWNW 2023-11-30 2023-11-25T09:34:58 859.85 +61 414 63 14506877706 12.54097 58.04557426323987 false Random H NUOAD 2023-12-10 2023-12-06T22:52:58 780.50 +62 451 50 12304139502 51.15162 22.46754141558852 false Random C SRRSV 2023-12-08 2023-12-20T02:48:58 1352.65 63 112 75 12197306353 85.90137 43.48931389222043 false Random C KKAIT 2023-11-27 2023-12-23T04:23:58 1954.90 64 678 14 13681447851 74.83621 36.94143092647816 true Random J KELFB 2023-12-01 2023-12-07T18:14:58 308.26 -66 306 5 14448160602 44.642223 50.24249889525751 false Random X OASEB 2023-12-11 2023-11-27T00:16:58 1345.69 -67 484 65 10817432713 62.168163 77.02869166077757 true Random K SAJMG 2023-12-19 2023-12-14T19:47:58 488.01 -68 922 13 11664232196 72.683266 37.9910331525765 false Random W PPWBB 2023-11-26 2023-12-10T22:54:58 1968.89 -76 504 70 14161652666 58.071503 67.99111956708262 true Random Y HAVCK 2023-11-27 2023-12-14T16:08:58 1864.98 -77 165 36 12887722637 19.729382 45.61157603163882 true Random S OZOLB 2023-12-02 2023-12-03T05:07:58 1576.79 +66 306 5 14448160602 44.64222 50.24249889525751 false Random X OASEB 2023-12-11 2023-11-27T00:16:58 1345.69 +67 484 65 10817432713 62.16816 77.02869166077757 true Random K SAJMG 2023-12-19 2023-12-14T19:47:58 488.01 +68 922 13 11664232196 72.68327 37.9910331525765 false Random W PPWBB 2023-11-26 2023-12-10T22:54:58 1968.89 +76 504 70 14161652666 58.0715 67.99111956708262 true Random Y HAVCK 2023-11-27 2023-12-14T16:08:58 1864.98 +77 165 36 12887722637 19.72938 45.61157603163882 true Random S OZOLB 2023-12-02 2023-12-03T05:07:58 1576.79 8 550 48 13655992126 52.90345 51.35114230137935 false Random X JTVSE 2023-12-13 2023-12-15T03:49:58 361.55 8 866 37 13672147880 81.28999 67.66548594336737 false Random H QDJIM 2023-12-14 2023-12-17T18:44:58 1112.05 -80 815 19 14529289205 19.769405 37.37008094684765 true Random Z WLALH 2023-12-11 2023-12-14T03:24:58 479.38 -86 398 27 13222936963 20.387327 44.51255195842424 true Random T ZCRFI 2023-12-21 2023-12-23T12:04:58 1801.53 -86 728 18 13390353484 61.060482 87.44751616093882 false Random J BUCVI 2023-12-07 2023-12-14T23:00:58 1611.17 -86 998 74 11080891106 82.568756 32.0122101203062 true Random K VAAMT 2023-12-23 2023-12-01T10:14:58 1708.39 -88 274 41 14108849690 73.74919 42.625751442467404 true Random X BVRFA 2023-12-01 2023-11-25T14:32:58 515.18 -89 377 22 14340881803 32.61157 82.5503801214006 false Random K ACYZU 2023-12-01 2023-11-27T02:05:58 672.13 -89 964 41 12706120446 69.484116 32.39048200771184 true Random J IIRNY 2023-12-16 2023-11-29T01:54:58 1298.71 +80 815 19 14529289205 19.76941 37.37008094684765 true Random Z WLALH 2023-12-11 2023-12-14T03:24:58 479.38 +86 398 27 13222936963 20.38733 44.51255195842424 true Random T ZCRFI 2023-12-21 2023-12-23T12:04:58 1801.53 +86 728 18 13390353484 61.06048 87.44751616093882 false Random J BUCVI 2023-12-07 2023-12-14T23:00:58 1611.17 +86 998 74 11080891106 82.56876 32.0122101203062 true Random K VAAMT 2023-12-23 2023-12-01T10:14:58 1708.39 +88 274 41 14108849690 73.74919 42.6257514424674 true Random X BVRFA 2023-12-01 2023-11-25T14:32:58 515.18 +89 377 22 14340881803 32.61157 82.55038012140059 false Random K ACYZU 2023-12-01 2023-11-27T02:05:58 672.13 +89 964 41 12706120446 69.48412 32.39048200771184 true Random J IIRNY 2023-12-16 2023-11-29T01:54:58 1298.71 90 391 26 12874761259 21.49042 53.46850617467312 true Random Q QTJPE 2023-12-17 2023-12-03T17:40:58 748.05 -91 389 11 14784237986 11.174142 27.692284427565397 true Random P DYILB 2023-12-14 2023-12-21T11:07:58 1175.73 +91 389 11 14784237986 11.17414 27.6922844275654 true Random P DYILB 2023-12-14 2023-12-21T11:07:58 1175.73 91 528 68 14588592231 77.4651 88.92064181463138 false Random U JXZUA 2023-12-16 2023-12-21T02:28:58 1834.07 -93 887 20 13555948969 70.57364 32.621532934876804 false Random D SPMEK 2023-11-26 2023-12-20T18:11:58 258.86 -96 595 72 11506136303 21.917727 74.74561804277158 true Random T SPLKA 2023-12-02 2023-11-30T00:39:58 1693.61 +93 887 20 13555948969 70.57364 32.6215329348768 false Random D SPMEK 2023-11-26 2023-12-20T18:11:58 258.86 +96 595 72 11506136303 21.91773 74.74561804277158 true Random T SPLKA 2023-12-02 2023-11-30T00:39:58 1693.61 97 839 60 14818779777 46.17389 68.98285340004992 false Random W HMFPU 2023-12-01 2023-12-04T08:41:58 1683.48 -- !q46 -- -1 578 55 2111222273 56.858597 82.38111658179561 true Random C LYDUG 2023-12-17 2023-12-05T13:04:58 1393.11 -29 910 52 5544039917 22.179396 46.32732226806482 true Random C TIZAG 2023-11-28 2023-12-14T16:08:58 900.96 -3 300 15 3000000000 20.25 30.0 true Third C Gamma 2023-10-08 2023-10-08T16:15 345.67 -43 178 64 6969956763 40.980415 52.998828731408516 true Random C XQHYB 2023-12-11 2023-12-07T23:00:58 257.08 -69 416 14 7702410607 31.638903 89.5793904314531 true Random C URQMU 2023-11-25 2023-11-30T15:17:58 1379.22 +1 578 55 2111222273 56.8586 82.38111658179561 true Random C LYDUG 2023-12-17 2023-12-05T13:04:58 1393.11 +29 910 52 5544039917 22.1794 46.32732226806482 true Random C TIZAG 2023-11-28 2023-12-14T16:08:58 900.96 +3 300 15 3000000000 20.25 30 true Third C Gamma 2023-10-08 2023-10-08T16:15 345.67 +43 178 64 6969956763 40.98042 52.99882873140852 true Random C XQHYB 2023-12-11 2023-12-07T23:00:58 257.08 +69 416 14 7702410607 31.6389 89.57939043145311 true Random C URQMU 2023-11-25 2023-11-30T15:17:58 1379.22 82 107 51 1358006007 78.36581 46.09413324325159 true Random C IPNQU 2023-12-01 2023-12-14T05:41:58 417.17 -- !q47 -- -1 578 55 2111222273 56.858597 82.38111658179561 true Random C LYDUG 2023-12-17 2023-12-05T13:04:58 1393.11 -1 979 44 10163954251 28.827957 57.56879940298416 true Random Q DNRGE 2023-12-09 2023-12-10T20:21:58 1581.25 +1 578 55 2111222273 56.8586 82.38111658179561 true Random C LYDUG 2023-12-17 2023-12-05T13:04:58 1393.11 +1 979 44 10163954251 28.82796 57.56879940298416 true Random Q DNRGE 2023-12-09 2023-12-10T20:21:58 1581.25 10 1000 50 10000000000 55.25 65.75 false Tenth J Kappa 2023-10-15 2023-10-15T23:30 1012.34 -10 966 38 2203748112 45.555325 27.908447208440094 true Random W LFAGO 2023-12-14 2023-11-26T20:00:58 1898.68 -11 1100 55 11000000000 60.5 70.0 true Eleventh K Lambda 2023-10-16 2023-10-16T01:45 1123.45 -11 441 19 7370044350 74.261696 62.013817404758086 true Random D UYKZA 2023-12-23 2023-12-15T11:49:58 1805.14 -11 487 27 14556302216 85.33334 62.596750833474495 true Random E QMHJD 2023-12-23 2023-12-24T08:30:58 1491.22 +10 966 38 2203748112 45.55532 27.90844720844009 true Random W LFAGO 2023-12-14 2023-11-26T20:00:58 1898.68 +11 1100 55 11000000000 60.5 70 true Eleventh K Lambda 2023-10-16 2023-10-16T01:45 1123.45 +11 441 19 7370044350 74.2617 62.01381740475809 true Random D UYKZA 2023-12-23 2023-12-15T11:49:58 1805.14 +11 487 27 14556302216 85.33334 62.59675083347449 true Random E QMHJD 2023-12-23 2023-12-24T08:30:58 1491.22 12 1200 60 12000000000 65.75 75.25 false Twelfth L Mu 2023-10-17 2023-10-17T02:15 1234.56 13 1300 65 13000000000 70.0 80.5 true Thirteenth M Nu 2023-10-18 2023-10-18T03:30 1345.67 13 402 30 10851194313 74.82481 74.90108005771035 false Random F GEMMK 2023-11-27 2023-12-21T15:03:58 1643.55 -13 696 74 3370487489 84.544014 88.69976219408227 true Random H RTFJI 2023-11-23 2023-11-25T07:32:58 1761.50 -13 745 48 13047949175 51.168613 85.21972389262197 true Random A AYBWQ 2023-12-22 2023-12-22T16:25:58 1192.48 -13 859 65 7433576046 56.136265 34.87823331022725 false Random L CRFUF 2023-12-23 2023-12-12T15:05:58 1037.15 +13 696 74 3370487489 84.54401 88.69976219408227 true Random H RTFJI 2023-11-23 2023-11-25T07:32:58 1761.50 +13 745 48 13047949175 51.16861 85.21972389262197 true Random A AYBWQ 2023-12-22 2023-12-22T16:25:58 1192.48 +13 859 65 7433576046 56.13626 34.87823331022725 false Random L CRFUF 2023-12-23 2023-12-12T15:05:58 1037.15 14 1400 70 14000000000 75.25 85.75 false Fourteenth N Xi 2023-10-19 2023-10-19T04:45 1456.78 -14 195 17 2370700139 16.777058 64.81793301410002 false Random P IIGRE 2023-12-12 2023-12-14T22:40:58 1678.44 -14 966 65 7828602539 62.430664 68.85873133439297 true Random I VVOQH 2023-12-01 2023-12-06T00:54:58 1300.43 -15 1500 75 15000000000 80.5 90.0 true Fifteenth O Omicron 2023-10-20 2023-10-20T05:15 1567.89 -16 135 22 7901304568 43.944805 85.16901944253635 true Random K NUQEP 2023-11-29 2023-11-25T23:42:58 1440.74 +14 195 17 2370700139 16.77706 64.81793301410002 false Random P IIGRE 2023-12-12 2023-12-14T22:40:58 1678.44 +14 966 65 7828602539 62.43066 68.85873133439297 true Random I VVOQH 2023-12-01 2023-12-06T00:54:58 1300.43 +15 1500 75 15000000000 80.5 90 true Fifteenth O Omicron 2023-10-20 2023-10-20T05:15 1567.89 +16 135 22 7901304568 43.94481 85.16901944253635 true Random K NUQEP 2023-11-29 2023-11-25T23:42:58 1440.74 16 615 20 12294128025 77.37379 20.42772029677839 true Random U JHPOB 2023-11-30 2023-12-16T14:29:58 1105.33 -17 499 46 11230409207 51.632103 28.811164197154774 false Random V BVLUH 2023-12-13 2023-12-23T17:59:58 1387.62 -18 690 17 1399456103 63.261967 42.964715823771236 true Random R BWSRS 2023-12-13 2023-12-23T08:33:58 1840.02 -18 835 17 14265814864 18.923101 80.53531451138412 true Random V PIKUZ 2023-12-20 2023-12-21T07:39:58 1167.09 -19 917 66 2340946367 89.035675 22.649362455875274 false Random D HWHMU 2023-11-30 2023-12-10T02:36:58 1960.07 -19 993 13 7039833438 79.769066 69.79049291517285 true Random X OFSUV 2023-12-11 2023-12-08T01:46:58 1958.95 +17 499 46 11230409207 51.6321 28.81116419715477 false Random V BVLUH 2023-12-13 2023-12-23T17:59:58 1387.62 +18 690 17 1399456103 63.26197 42.96471582377124 true Random R BWSRS 2023-12-13 2023-12-23T08:33:58 1840.02 +18 835 17 14265814864 18.9231 80.53531451138412 true Random V PIKUZ 2023-12-20 2023-12-21T07:39:58 1167.09 +19 917 66 2340946367 89.03568 22.64936245587527 false Random D HWHMU 2023-11-30 2023-12-10T02:36:58 1960.07 +19 993 13 7039833438 79.76907 69.79049291517285 true Random X OFSUV 2023-12-11 2023-12-08T01:46:58 1958.95 2 921 62 8557914543 78.52379 58.6849882881372 false Random D KBXXS 2023-12-07 2023-12-02T22:24:58 1782.88 -2 925 46 6013180177 41.107002 34.86561026061906 true Random L XLLXY 2023-12-06 2023-12-09T14:04:58 1246.26 -20 248 64 7704906572 35.089928 76.69128821479936 true Random T KQOMS 2023-11-30 2023-12-11T01:35:58 1799.26 -22 200 41 12163439252 64.621254 81.68574929661384 true Random U KGVNU 2023-12-20 2023-11-30T14:56:58 1915.47 -23 192 8 5102667616 54.111057 40.85713971600841 false Random J EBXEB 2023-12-13 2023-12-10T11:32:58 1824.12 +2 925 46 6013180177 41.107 34.86561026061906 true Random L XLLXY 2023-12-06 2023-12-09T14:04:58 1246.26 +20 248 64 7704906572 35.08993 76.69128821479936 true Random T KQOMS 2023-11-30 2023-12-11T01:35:58 1799.26 +22 200 41 12163439252 64.62125 81.68574929661384 true Random U KGVNU 2023-12-20 2023-11-30T14:56:58 1915.47 +23 192 8 5102667616 54.11106 40.85713971600841 false Random J EBXEB 2023-12-13 2023-12-10T11:32:58 1824.12 27 866 24 5531365994 72.77447 86.96690821165853 false Random S TZPFJ 2023-11-28 2023-12-13T15:31:58 1274.75 -29 157 34 2302882987 51.924015 20.311140937696468 true Random R MBOXJ 2023-12-02 2023-12-03T14:12:58 1620.80 -29 923 57 1591814253 68.57371 33.342802789892986 true Random Q ZONGC 2023-12-20 2023-12-13T09:11:58 1465.38 -3 259 74 7422478791 22.291426 75.38227773520089 true Random S VWAXJ 2023-12-01 2023-12-05T21:23:58 1970.57 -30 292 71 10308444223 63.039078 76.40649540444898 false Random G DRLHY 2023-12-19 2023-12-14T15:32:58 1165.14 -30 830 65 12624057029 38.791172 59.72899174862661 false Random A LFPWP 2023-12-03 2023-12-17T00:10:58 1760.62 +29 157 34 2302882987 51.92402 20.31114093769647 true Random R MBOXJ 2023-12-02 2023-12-03T14:12:58 1620.80 +29 923 57 1591814253 68.57371 33.34280278989299 true Random Q ZONGC 2023-12-20 2023-12-13T09:11:58 1465.38 +3 259 74 7422478791 22.29143 75.38227773520089 true Random S VWAXJ 2023-12-01 2023-12-05T21:23:58 1970.57 +30 292 71 10308444223 63.03908 76.40649540444898 false Random G DRLHY 2023-12-19 2023-12-14T15:32:58 1165.14 +30 830 65 12624057029 38.79117 59.72899174862661 false Random A LFPWP 2023-12-03 2023-12-17T00:10:58 1760.62 31 395 22 6141426904 88.37914 52.0655270963123 false Random J DRPJV 2023-12-07 2023-11-29T03:15:58 1076.41 -31 990 5 13678786851 15.762894 85.24173385692956 false Random H THGIM 2023-12-14 2023-12-09T01:24:58 1834.37 -39 726 50 3865644066 26.225628 28.534393094364418 false Random F NIUCS 2023-12-05 2023-12-04T19:31:58 1953.82 +31 990 5 13678786851 15.76289 85.24173385692956 false Random H THGIM 2023-12-14 2023-12-09T01:24:58 1834.37 +39 726 50 3865644066 26.22563 28.53439309436442 false Random F NIUCS 2023-12-05 2023-12-04T19:31:58 1953.82 4 122 24 10738473173 81.15482 60.21481394154484 false Random Y PQJRK 2023-12-20 2023-12-09T02:38:58 1467.35 -4 569 72 10560903405 50.255936 47.535145739285184 false Random O NRIRC 2023-12-05 2023-12-01T09:10:58 1986.99 -40 230 34 10824964541 16.929768 53.812277279703366 false Random F YDQHF 2023-12-14 2023-12-03T17:42:58 1623.79 -40 914 7 4902128502 19.442041 33.099787387344406 true Random Q KOCWA 2023-11-28 2023-12-21T09:20:58 1824.80 -41 344 34 14536795918 56.660946 84.15108995619764 false Random Q KYLCH 2023-12-10 2023-12-04T08:25:58 1902.09 -41 697 21 1200243566 12.466168 68.57243624557165 true Random U JZGEG 2023-12-03 2023-12-10T04:51:58 1323.88 +4 569 72 10560903405 50.25594 47.53514573928518 false Random O NRIRC 2023-12-05 2023-12-01T09:10:58 1986.99 +40 230 34 10824964541 16.92977 53.81227727970337 false Random F YDQHF 2023-12-14 2023-12-03T17:42:58 1623.79 +40 914 7 4902128502 19.44204 33.09978738734441 true Random Q KOCWA 2023-11-28 2023-12-21T09:20:58 1824.80 +41 344 34 14536795918 56.66095 84.15108995619764 false Random Q KYLCH 2023-12-10 2023-12-04T08:25:58 1902.09 +41 697 21 1200243566 12.46617 68.57243624557165 true Random U JZGEG 2023-12-03 2023-12-10T04:51:58 1323.88 41 708 64 11745827370 72.84812 35.31028363777645 true Random O WGSQC 2023-12-02 2023-11-25T17:07:58 1666.71 42 178 38 7559404453 69.69449 64.37154501388798 true Random G QUMUN 2023-12-14 2023-12-17T01:37:58 1190.44 -42 192 28 14454791024 35.465202 46.34876515635648 false Random W NQFGR 2023-12-04 2023-11-24T05:02:58 1428.02 +42 192 28 14454791024 35.4652 46.34876515635648 false Random W NQFGR 2023-12-04 2023-11-24T05:02:58 1428.02 42 355 72 11536856285 74.42886 53.49032479461299 false Random I IQZEI 2023-12-10 2023-12-06T07:17:58 1098.14 44 219 38 8596488294 73.52956 94.10797854680568 true Random E HMWBI 2023-12-15 2023-12-06T00:51:58 1907.47 -44 694 55 3626514138 62.504086 72.89799265418553 true Random Z JTDVF 2023-12-01 2023-11-29T12:08:58 1769.92 -45 455 25 12639246000 47.011307 26.310712594958694 false Random Z GGEUA 2023-11-27 2023-12-01T20:41:58 1698.21 -45 492 43 3870916386 51.069588 42.652270406300794 true Random H JVZTB 2023-12-04 2023-12-09T21:06:58 1517.83 -47 508 48 1456473942 48.488297 20.377955902326608 false Random B CAOEY 2023-11-29 2023-12-10T14:49:58 1865.52 -47 566 50 1426586688 51.278687 40.47151456873397 true Random F YBOSH 2023-11-26 2023-12-15T03:44:58 1806.35 +44 694 55 3626514138 62.50409 72.89799265418553 true Random Z JTDVF 2023-12-01 2023-11-29T12:08:58 1769.92 +45 455 25 12639246000 47.01131 26.31071259495869 false Random Z GGEUA 2023-11-27 2023-12-01T20:41:58 1698.21 +45 492 43 3870916386 51.06959 42.65227040630079 true Random H JVZTB 2023-12-04 2023-12-09T21:06:58 1517.83 +47 508 48 1456473942 48.4883 20.37795590232661 false Random B CAOEY 2023-11-29 2023-12-10T14:49:58 1865.52 +47 566 50 1426586688 51.27869 40.47151456873397 true Random F YBOSH 2023-11-26 2023-12-15T03:44:58 1806.35 47 838 73 14910230294 83.69784 82.28901816600579 true Random L SHXYL 2023-11-24 2023-12-05T22:19:58 1062.15 48 898 59 12871187130 10.13838 70.19705104611333 true Random J WFXNN 2023-12-23 2023-12-17T02:53:58 1050.21 -49 412 16 8300982793 56.263252 66.07893608061771 false Random K DWWJI 2023-12-08 2023-12-17T11:32:58 1718.54 -49 568 70 2916596630 79.16303 56.114316916863025 false Random T ILLIU 2023-11-23 2023-12-07T11:05:58 1039.03 -5 768 5 4152322228 41.128906 78.60686390712706 false Random J LXKRA 2023-12-05 2023-11-24T18:13:58 1941.98 -5 823 63 13328808917 77.768196 22.87975226738422 false Random F OIYPV 2023-12-11 2023-12-14T06:43:58 1144.38 -52 811 31 14085958816 51.067017 65.01991893789116 true Random A CODYQ 2023-12-03 2023-12-07T23:25:58 1797.21 -54 827 55 7054839267 58.555687 25.891004802115663 false Random O ASMLW 2023-12-13 2023-12-20T16:41:58 1369.32 -54 843 34 9547939940 38.66475 36.370944299232434 true Random P NTVIR 2023-12-12 2023-12-02T06:45:58 1628.37 +49 412 16 8300982793 56.26325 66.07893608061771 false Random K DWWJI 2023-12-08 2023-12-17T11:32:58 1718.54 +49 568 70 2916596630 79.16303 56.11431691686303 false Random T ILLIU 2023-11-23 2023-12-07T11:05:58 1039.03 +5 768 5 4152322228 41.12891 78.60686390712706 false Random J LXKRA 2023-12-05 2023-11-24T18:13:58 1941.98 +5 823 63 13328808917 77.7682 22.87975226738422 false Random F OIYPV 2023-12-11 2023-12-14T06:43:58 1144.38 +52 811 31 14085958816 51.06702 65.01991893789116 true Random A CODYQ 2023-12-03 2023-12-07T23:25:58 1797.21 +54 827 55 7054839267 58.55569 25.89100480211566 false Random O ASMLW 2023-12-13 2023-12-20T16:41:58 1369.32 +54 843 34 9547939940 38.66475 36.37094429923243 true Random P NTVIR 2023-12-12 2023-12-02T06:45:58 1628.37 55 908 24 13623721787 40.06427 90.85281792731746 false Random B KFZGI 2023-11-27 2023-12-23T18:06:58 1124.95 -55 964 8 14038541765 70.24135 20.034551391620194 false Random J AYXIT 2023-12-13 2023-12-16T19:38:58 1476.73 -59 144 31 6208909394 67.417076 40.59765633709834 true Random D FLWNA 2023-12-12 2023-12-19T06:17:58 1870.24 +55 964 8 14038541765 70.24135 20.03455139162019 false Random J AYXIT 2023-12-13 2023-12-16T19:38:58 1476.73 +59 144 31 6208909394 67.41708 40.59765633709834 true Random D FLWNA 2023-12-12 2023-12-19T06:17:58 1870.24 59 509 50 5501336408 39.94401 73.35770882761237 true Random I PVZNO 2023-12-04 2023-11-27T04:40:58 1177.33 -60 711 69 1493870104 22.574188 61.30347648465907 false Random E FHKVR 2023-11-27 2023-12-05T11:26:58 1981.61 -62 451 50 12304139502 51.151623 22.46754141558852 false Random C SRRSV 2023-12-08 2023-12-20T02:48:58 1352.65 +60 711 69 1493870104 22.57419 61.30347648465907 false Random E FHKVR 2023-11-27 2023-12-05T11:26:58 1981.61 +62 451 50 12304139502 51.15162 22.46754141558852 false Random C SRRSV 2023-12-08 2023-12-20T02:48:58 1352.65 63 112 75 12197306353 85.90137 43.48931389222043 false Random C KKAIT 2023-11-27 2023-12-23T04:23:58 1954.90 -63 383 35 5161212745 39.455276 52.33267523851794 false Random X TMYMC 2023-11-29 2023-12-10T09:09:58 1442.54 -63 410 33 1767102777 72.260124 56.971483381024896 false Random B QXNSM 2023-12-12 2023-12-19T22:57:58 1660.73 -64 719 36 1224510454 64.237434 86.05689694804887 true Random E ZVQPU 2023-11-30 2023-12-03T04:56:58 1879.25 -66 306 5 14448160602 44.642223 50.24249889525751 false Random X OASEB 2023-12-11 2023-11-27T00:16:58 1345.69 -68 266 31 8183454755 69.19586 23.139304803938643 false Random S STCBM 2023-11-26 2023-12-22T13:42:58 1722.37 +63 383 35 5161212745 39.45528 52.33267523851794 false Random X TMYMC 2023-11-29 2023-12-10T09:09:58 1442.54 +63 410 33 1767102777 72.26012 56.9714833810249 false Random B QXNSM 2023-12-12 2023-12-19T22:57:58 1660.73 +64 719 36 1224510454 64.23743 86.05689694804887 true Random E ZVQPU 2023-11-30 2023-12-03T04:56:58 1879.25 +66 306 5 14448160602 44.64222 50.24249889525751 false Random X OASEB 2023-12-11 2023-11-27T00:16:58 1345.69 +68 266 31 8183454755 69.19586 23.13930480393864 false Random S STCBM 2023-11-26 2023-12-22T13:42:58 1722.37 68 756 63 5416393421 66.41538 76.32820339134415 false Random Y CUNAL 2023-12-23 2023-12-14T22:49:58 1109.25 -68 922 13 11664232196 72.683266 37.9910331525765 false Random W PPWBB 2023-11-26 2023-12-10T22:54:58 1968.89 -69 416 14 7702410607 31.638903 89.5793904314531 true Random C URQMU 2023-11-25 2023-11-30T15:17:58 1379.22 +68 922 13 11664232196 72.68327 37.9910331525765 false Random W PPWBB 2023-11-26 2023-12-10T22:54:58 1968.89 +69 416 14 7702410607 31.6389 89.57939043145311 true Random C URQMU 2023-11-25 2023-11-30T15:17:58 1379.22 7 969 62 3451343234 57.17074 56.74513811095188 false Random G OWDSC 2023-12-19 2023-12-11T17:17:58 1874.22 -70 231 67 4547989149 35.103123 51.93622592177748 true Random V ZBCVY 2023-11-29 2023-12-22T11:41:58 1749.60 -70 421 23 3153379289 27.412096 79.32006404438445 false Random L VLJWK 2023-12-04 2023-12-12T05:31:58 1163.35 +70 231 67 4547989149 35.10312 51.93622592177748 true Random V ZBCVY 2023-11-29 2023-12-22T11:41:58 1749.60 +70 421 23 3153379289 27.4121 79.32006404438445 false Random L VLJWK 2023-12-04 2023-12-12T05:31:58 1163.35 70 751 56 7828222634 52.8313 55.7263634552559 true Random B TFHMH 2023-11-30 2023-12-24T12:22:58 1166.13 -73 866 49 4618070115 46.803646 91.41305051885227 true Random H ROYYF 2023-12-07 2023-12-01T10:28:58 1817.67 -76 504 70 14161652666 58.071503 67.99111956708262 true Random Y HAVCK 2023-11-27 2023-12-14T16:08:58 1864.98 +73 866 49 4618070115 46.80365 91.41305051885227 true Random H ROYYF 2023-12-07 2023-12-01T10:28:58 1817.67 +76 504 70 14161652666 58.0715 67.99111956708262 true Random Y HAVCK 2023-11-27 2023-12-14T16:08:58 1864.98 77 131 19 2964167114 33.23181 53.35246738882714 false Random G AHGFO 2023-12-19 2023-12-01T10:11:58 1837.90 -77 165 36 12887722637 19.729382 45.61157603163882 true Random S OZOLB 2023-12-02 2023-12-03T05:07:58 1576.79 +77 165 36 12887722637 19.72938 45.61157603163882 true Random S OZOLB 2023-12-02 2023-12-03T05:07:58 1576.79 8 866 37 13672147880 81.28999 67.66548594336737 false Random H QDJIM 2023-12-14 2023-12-17T18:44:58 1112.05 -80 267 57 8797946135 35.604717 80.51381110359165 false Random K KQTEX 2023-12-09 2023-12-13T06:19:58 1769.15 +80 267 57 8797946135 35.60472 80.51381110359165 false Random K KQTEX 2023-12-09 2023-12-13T06:19:58 1769.15 82 603 60 9083469993 81.24088 44.46228092092543 true Random Y WTQGU 2023-11-30 2023-11-28T13:18:58 1448.45 -84 427 60 9035762847 81.971306 28.37315065501099 true Random L FETYF 2023-12-01 2023-11-24T15:00:58 1267.12 +84 427 60 9035762847 81.97131 28.37315065501099 true Random L FETYF 2023-12-01 2023-11-24T15:00:58 1267.12 85 375 63 6797318130 85.47522 58.16330728665678 true Random E UNZLS 2023-12-01 2023-12-04T05:17:58 1949.48 -85 873 18 7233488476 33.83051 31.655950581225508 false Random N RJTIB 2023-11-23 2023-12-11T15:07:58 1249.52 -86 398 27 13222936963 20.387327 44.51255195842424 true Random T ZCRFI 2023-12-21 2023-12-23T12:04:58 1801.53 +85 873 18 7233488476 33.83051 31.65595058122551 false Random N RJTIB 2023-11-23 2023-12-11T15:07:58 1249.52 +86 398 27 13222936963 20.38733 44.51255195842424 true Random T ZCRFI 2023-12-21 2023-12-23T12:04:58 1801.53 86 662 53 8875065706 28.64778 30.6775849729486 false Random N YNQAY 2023-12-15 2023-11-24T21:56:58 1108.35 -86 728 18 13390353484 61.060482 87.44751616093882 false Random J BUCVI 2023-12-07 2023-12-14T23:00:58 1611.17 -86 998 74 11080891106 82.568756 32.0122101203062 true Random K VAAMT 2023-12-23 2023-12-01T10:14:58 1708.39 +86 728 18 13390353484 61.06048 87.44751616093882 false Random J BUCVI 2023-12-07 2023-12-14T23:00:58 1611.17 +86 998 74 11080891106 82.56876 32.0122101203062 true Random K VAAMT 2023-12-23 2023-12-01T10:14:58 1708.39 87 145 64 9022533179 37.80205 63.26081178595084 true Random T PEOPK 2023-12-08 2023-12-07T17:41:58 1167.05 -87 641 64 4786767059 14.765089 70.8793353664754 false Random W SQHGN 2023-12-12 2023-12-24T01:19:58 1316.61 -88 728 59 8439434199 30.372904 59.410283344764366 false Random F JODWY 2023-12-04 2023-12-01T07:57:58 1753.88 +87 641 64 4786767059 14.76509 70.8793353664754 false Random W SQHGN 2023-12-12 2023-12-24T01:19:58 1316.61 +88 728 59 8439434199 30.3729 59.41028334476437 false Random F JODWY 2023-12-04 2023-12-01T07:57:58 1753.88 88 765 69 9753682777 83.42646 25.99260711248508 true Random M MEJAX 2023-11-25 2023-12-20T09:21:58 1647.22 -89 129 64 6400162051 67.910965 80.48074661432221 true Random Y ZXJWQ 2023-12-16 2023-12-19T10:23:58 1882.65 -89 964 41 12706120446 69.484116 32.39048200771184 true Random J IIRNY 2023-12-16 2023-11-29T01:54:58 1298.71 -9 113 7 6162580854 11.346889 46.82839094332704 false Random A SJTAF 2023-12-14 2023-11-23T18:27:58 1610.49 -91 389 11 14784237986 11.174142 27.692284427565397 true Random P DYILB 2023-12-14 2023-12-21T11:07:58 1175.73 +89 129 64 6400162051 67.91096 80.48074661432221 true Random Y ZXJWQ 2023-12-16 2023-12-19T10:23:58 1882.65 +89 964 41 12706120446 69.48412 32.39048200771184 true Random J IIRNY 2023-12-16 2023-11-29T01:54:58 1298.71 +9 113 7 6162580854 11.34689 46.82839094332704 false Random A SJTAF 2023-12-14 2023-11-23T18:27:58 1610.49 +91 389 11 14784237986 11.17414 27.6922844275654 true Random P DYILB 2023-12-14 2023-12-21T11:07:58 1175.73 91 528 68 14588592231 77.4651 88.92064181463138 false Random U JXZUA 2023-12-16 2023-12-21T02:28:58 1834.07 -92 344 29 5182139341 31.653255 44.26814517218887 true Random F NGHOS 2023-12-06 2023-12-09T21:25:58 1291.06 -94 216 49 8773264156 81.617195 43.03983700523827 true Random D VHWYT 2023-12-13 2023-11-30T07:03:58 1178.27 -94 693 60 4818659234 26.04229 83.2975107272106 true Random B ENSQO 2023-12-22 2023-12-12T06:08:58 1283.81 -96 595 72 11506136303 21.917727 74.74561804277158 true Random T SPLKA 2023-12-02 2023-11-30T00:39:58 1693.61 -96 637 39 5516035994 55.90832 60.522041012562816 true Random O YPETL 2023-12-02 2023-11-28T02:47:58 1175.16 -97 415 74 10346322649 21.667427 46.58901867647463 false Random R KWFOF 2023-12-21 2023-11-27T12:18:58 1157.72 +92 344 29 5182139341 31.65326 44.26814517218887 true Random F NGHOS 2023-12-06 2023-12-09T21:25:58 1291.06 +94 216 49 8773264156 81.6172 43.03983700523827 true Random D VHWYT 2023-12-13 2023-11-30T07:03:58 1178.27 +94 693 60 4818659234 26.04229 83.29751072721059 true Random B ENSQO 2023-12-22 2023-12-12T06:08:58 1283.81 +96 595 72 11506136303 21.91773 74.74561804277158 true Random T SPLKA 2023-12-02 2023-11-30T00:39:58 1693.61 +96 637 39 5516035994 55.90832 60.52204101256282 true Random O YPETL 2023-12-02 2023-11-28T02:47:58 1175.16 +97 415 74 10346322649 21.66743 46.58901867647463 false Random R KWFOF 2023-12-21 2023-11-27T12:18:58 1157.72 97 839 60 14818779777 46.17389 68.98285340004992 false Random W HMFPU 2023-12-01 2023-12-04T08:41:58 1683.48 -- !q48 -- 1 100 5 1000000000 10.5 20.75 true First A Alpha 2023-10-06 2023-10-06T14:30 123.45 10 1000 50 10000000000 55.25 65.75 false Tenth J Kappa 2023-10-15 2023-10-15T23:30 1012.34 -11 1100 55 11000000000 60.5 70.0 true Eleventh K Lambda 2023-10-16 2023-10-16T01:45 1123.45 +11 1100 55 11000000000 60.5 70 true Eleventh K Lambda 2023-10-16 2023-10-16T01:45 1123.45 12 1200 60 12000000000 65.75 75.25 false Twelfth L Mu 2023-10-17 2023-10-17T02:15 1234.56 13 1300 65 13000000000 70.0 80.5 true Thirteenth M Nu 2023-10-18 2023-10-18T03:30 1345.67 14 1400 70 14000000000 75.25 85.75 false Fourteenth N Xi 2023-10-19 2023-10-19T04:45 1456.78 -15 1500 75 15000000000 80.5 90.0 true Fifteenth O Omicron 2023-10-20 2023-10-20T05:15 1567.89 +15 1500 75 15000000000 80.5 90 true Fifteenth O Omicron 2023-10-20 2023-10-20T05:15 1567.89 2 200 10 2000000000 15.75 25.5 false Second B Beta 2023-10-07 2023-10-07T15:45 234.56 -3 300 15 3000000000 20.25 30.0 true Third C Gamma 2023-10-08 2023-10-08T16:15 345.67 +3 300 15 3000000000 20.25 30 true Third C Gamma 2023-10-08 2023-10-08T16:15 345.67 4 400 20 4000000000 25.5 35.25 false Fourth D Delta 2023-10-09 2023-10-09T17:30 456.78 5 500 25 5000000000 30.75 40.5 true Fifth E Epsilon 2023-10-10 2023-10-10T18:45 567.89 6 600 30 6000000000 35.25 45.75 false Sixth F Zeta 2023-10-11 2023-10-11T19:15 678.90 -7 700 35 7000000000 40.5 50.0 true Seventh G Eta 2023-10-12 2023-10-12T20:30 789.01 +7 700 35 7000000000 40.5 50 true Seventh G Eta 2023-10-12 2023-10-12T20:30 789.01 8 800 40 8000000000 45.75 55.25 false Eighth H Theta 2023-10-13 2023-10-13T21:45 890.12 9 900 45 9000000000 50.0 60.5 true Ninth I Iota 2023-10-14 2023-10-14T22:15 901.23 @@ -570,4 +524,3 @@ 438 491 21 66065079309 6.6624016E7 1.5542114222539822E10 false CEbvKZRdvMHxzVOIejq wJ eoTkUlht 2023-12-08 2023-12-17T19:49:48 86666.80 -- !lzo_8 -- - diff --git a/regression-test/data/external_table_p0/hive/test_hive_get_schema_from_table.out b/regression-test/data/external_table_p0/hive/test_hive_get_schema_from_table.out index fe8243f91e0a95..62fabbe7d08be7 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_get_schema_from_table.out +++ b/regression-test/data/external_table_p0/hive/test_hive_get_schema_from_table.out @@ -650,654 +650,3 @@ true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 -- !schema_7 -- \N \N \N \N \N \N \N \N \N test test test 1 2 3 4 5.1 6.2 true false 2011-05-06 2011-05-06T07:08:09.123 -1.2 12.30 -1234.5678 123456789.12340000 -1234567890.12345678 1234567890123456789012.1234567800000000 dGVzdDI= --- !all_types_bool_col_topn_asc -- -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 -false 1 1 1 10 1.1 10.1 11 01/02/09 1 2009-01-02T07:11:00.450 2009 1 -false 3 3 3 30 3.3 30.3 13 01/02/09 3 2009-01-02T07:13:00.480 2009 1 -false 5 5 5 50 5.5 50.5 15 01/02/09 5 2009-01-02T07:15:00.550 2009 1 -false 7 7 7 70 7.7 70.7 17 01/02/09 7 2009-01-02T07:17:00.660 2009 1 -false 9 9 9 90 9.9 90.89999999999999 19 01/02/09 9 2009-01-02T07:19:00.810 2009 1 - --- !all_types_bool_col_topn_desc -- -true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 -true 4 4 4 40 4.4 40.4 7294 12/31/10 4 2010-12-31T12:04:13.560 2010 12 -true 2 2 2 20 2.2 20.2 7292 12/31/10 2 2010-12-31T12:02:13.510 2010 12 -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 -true 8 8 8 80 8.8 80.8 7288 12/30/10 8 2010-12-30T11:58:13.330 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7286 12/30/10 6 2010-12-30T11:56:13.200 2010 12 -true 4 4 4 40 4.4 40.4 7284 12/30/10 4 2010-12-30T11:54:13.110 2010 12 -true 2 2 2 20 2.2 20.2 7282 12/30/10 2 2010-12-30T11:52:13.600 2010 12 -true 0 0 0 0 0.0 0 7280 12/30/10 0 2010-12-30T11:50:13.500 2010 12 - --- !all_types_tinyint_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_tinyint_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7289 12/30/10 9 2010-12-30T11:59:13.410 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7279 12/29/10 9 2010-12-29T11:49:12.960 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7269 12/28/10 9 2010-12-28T11:39:12.510 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7259 12/27/10 9 2010-12-27T11:29:12.600 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7249 12/26/10 9 2010-12-26T11:19:11.610 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7239 12/25/10 9 2010-12-25T11:09:11.160 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7229 12/24/10 9 2010-12-24T10:59:10.710 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7219 12/23/10 9 2010-12-23T10:49:10.260 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7209 12/22/10 9 2010-12-22T10:39:09.810 2010 12 - --- !all_types_smallint_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_smallint_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7289 12/30/10 9 2010-12-30T11:59:13.410 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7279 12/29/10 9 2010-12-29T11:49:12.960 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7269 12/28/10 9 2010-12-28T11:39:12.510 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7259 12/27/10 9 2010-12-27T11:29:12.600 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7249 12/26/10 9 2010-12-26T11:19:11.610 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7239 12/25/10 9 2010-12-25T11:09:11.160 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7229 12/24/10 9 2010-12-24T10:59:10.710 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7219 12/23/10 9 2010-12-23T10:49:10.260 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7209 12/22/10 9 2010-12-22T10:39:09.810 2010 12 - --- !all_types_int_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_int_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7289 12/30/10 9 2010-12-30T11:59:13.410 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7279 12/29/10 9 2010-12-29T11:49:12.960 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7269 12/28/10 9 2010-12-28T11:39:12.510 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7259 12/27/10 9 2010-12-27T11:29:12.600 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7249 12/26/10 9 2010-12-26T11:19:11.610 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7239 12/25/10 9 2010-12-25T11:09:11.160 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7229 12/24/10 9 2010-12-24T10:59:10.710 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7219 12/23/10 9 2010-12-23T10:49:10.260 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7209 12/22/10 9 2010-12-22T10:39:09.810 2010 12 - --- !all_types_bigint_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_bigint_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7289 12/30/10 9 2010-12-30T11:59:13.410 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7279 12/29/10 9 2010-12-29T11:49:12.960 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7269 12/28/10 9 2010-12-28T11:39:12.510 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7259 12/27/10 9 2010-12-27T11:29:12.600 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7249 12/26/10 9 2010-12-26T11:19:11.610 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7239 12/25/10 9 2010-12-25T11:09:11.160 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7229 12/24/10 9 2010-12-24T10:59:10.710 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7219 12/23/10 9 2010-12-23T10:49:10.260 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7209 12/22/10 9 2010-12-22T10:39:09.810 2010 12 - --- !all_types_float_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_float_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7289 12/30/10 9 2010-12-30T11:59:13.410 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7279 12/29/10 9 2010-12-29T11:49:12.960 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7269 12/28/10 9 2010-12-28T11:39:12.510 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7259 12/27/10 9 2010-12-27T11:29:12.600 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7249 12/26/10 9 2010-12-26T11:19:11.610 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7239 12/25/10 9 2010-12-25T11:09:11.160 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7229 12/24/10 9 2010-12-24T10:59:10.710 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7219 12/23/10 9 2010-12-23T10:49:10.260 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7209 12/22/10 9 2010-12-22T10:39:09.810 2010 12 - --- !all_types_double_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_double_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7289 12/30/10 9 2010-12-30T11:59:13.410 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7279 12/29/10 9 2010-12-29T11:49:12.960 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7269 12/28/10 9 2010-12-28T11:39:12.510 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7259 12/27/10 9 2010-12-27T11:29:12.600 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7249 12/26/10 9 2010-12-26T11:19:11.610 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7239 12/25/10 9 2010-12-25T11:09:11.160 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7229 12/24/10 9 2010-12-24T10:59:10.710 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7219 12/23/10 9 2010-12-23T10:49:10.260 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7209 12/22/10 9 2010-12-22T10:39:09.810 2010 12 - --- !all_types_id_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_id_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 -false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 -false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 -true 4 4 4 40 4.4 40.4 7294 12/31/10 4 2010-12-31T12:04:13.560 2010 12 -false 3 3 3 30 3.3 30.3 7293 12/31/10 3 2010-12-31T12:03:13.530 2010 12 -true 2 2 2 20 2.2 20.2 7292 12/31/10 2 2010-12-31T12:02:13.510 2010 12 -false 1 1 1 10 1.1 10.1 7291 12/31/10 1 2010-12-31T12:01:13.500 2010 12 -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 - --- !all_types_date_string_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_date_string_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 -false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 -false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 -true 4 4 4 40 4.4 40.4 7294 12/31/10 4 2010-12-31T12:04:13.560 2010 12 -false 3 3 3 30 3.3 30.3 7293 12/31/10 3 2010-12-31T12:03:13.530 2010 12 -true 2 2 2 20 2.2 20.2 7292 12/31/10 2 2010-12-31T12:02:13.510 2010 12 -false 1 1 1 10 1.1 10.1 7291 12/31/10 1 2010-12-31T12:01:13.500 2010 12 -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 - --- !all_types_string_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_string_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7289 12/30/10 9 2010-12-30T11:59:13.410 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7279 12/29/10 9 2010-12-29T11:49:12.960 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7269 12/28/10 9 2010-12-28T11:39:12.510 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7259 12/27/10 9 2010-12-27T11:29:12.600 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7249 12/26/10 9 2010-12-26T11:19:11.610 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7239 12/25/10 9 2010-12-25T11:09:11.160 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7229 12/24/10 9 2010-12-24T10:59:10.710 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7219 12/23/10 9 2010-12-23T10:49:10.260 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7209 12/22/10 9 2010-12-22T10:39:09.810 2010 12 - --- !all_types_timestamp_col_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_timestamp_col_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 -false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 -false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 -true 4 4 4 40 4.4 40.4 7294 12/31/10 4 2010-12-31T12:04:13.560 2010 12 -false 3 3 3 30 3.3 30.3 7293 12/31/10 3 2010-12-31T12:03:13.530 2010 12 -true 2 2 2 20 2.2 20.2 7292 12/31/10 2 2010-12-31T12:02:13.510 2010 12 -false 1 1 1 10 1.1 10.1 7291 12/31/10 1 2010-12-31T12:01:13.500 2010 12 -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 - --- !all_types_year_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_year_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 -false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 -false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 -true 4 4 4 40 4.4 40.4 7294 12/31/10 4 2010-12-31T12:04:13.560 2010 12 -false 3 3 3 30 3.3 30.3 7293 12/31/10 3 2010-12-31T12:03:13.530 2010 12 -true 2 2 2 20 2.2 20.2 7292 12/31/10 2 2010-12-31T12:02:13.510 2010 12 -false 1 1 1 10 1.1 10.1 7291 12/31/10 1 2010-12-31T12:01:13.500 2010 12 -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 - --- !all_types_month_topn_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_month_topn_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 -false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 -false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 -true 4 4 4 40 4.4 40.4 7294 12/31/10 4 2010-12-31T12:04:13.560 2010 12 -false 3 3 3 30 3.3 30.3 7293 12/31/10 3 2010-12-31T12:03:13.530 2010 12 -true 2 2 2 20 2.2 20.2 7292 12/31/10 2 2010-12-31T12:02:13.510 2010 12 -false 1 1 1 10 1.1 10.1 7291 12/31/10 1 2010-12-31T12:01:13.500 2010 12 -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 - --- !all_types_bool_col_topn_abs_asc -- -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 -false 1 1 1 10 1.1 10.1 11 01/02/09 1 2009-01-02T07:11:00.450 2009 1 -false 3 3 3 30 3.3 30.3 13 01/02/09 3 2009-01-02T07:13:00.480 2009 1 -false 5 5 5 50 5.5 50.5 15 01/02/09 5 2009-01-02T07:15:00.550 2009 1 -false 7 7 7 70 7.7 70.7 17 01/02/09 7 2009-01-02T07:17:00.660 2009 1 -false 9 9 9 90 9.9 90.89999999999999 19 01/02/09 9 2009-01-02T07:19:00.810 2009 1 - --- !all_types_bool_col_topn_abs_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 -false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 -false 3 3 3 30 3.3 30.3 7293 12/31/10 3 2010-12-31T12:03:13.530 2010 12 -false 1 1 1 10 1.1 10.1 7291 12/31/10 1 2010-12-31T12:01:13.500 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7289 12/30/10 9 2010-12-30T11:59:13.410 2010 12 -false 7 7 7 70 7.7 70.7 7287 12/30/10 7 2010-12-30T11:57:13.260 2010 12 -false 5 5 5 50 5.5 50.5 7285 12/30/10 5 2010-12-30T11:55:13.150 2010 12 -false 3 3 3 30 3.3 30.3 7283 12/30/10 3 2010-12-30T11:53:13.800 2010 12 -false 1 1 1 10 1.1 10.1 7281 12/30/10 1 2010-12-30T11:51:13.500 2010 12 - --- !all_types_tinyint_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_tinyint_col_topn_abs_desc -- -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 -true 0 0 0 0 0.0 0 7280 12/30/10 0 2010-12-30T11:50:13.500 2010 12 -true 0 0 0 0 0.0 0 7270 12/29/10 0 2010-12-29T11:40:12.600 2010 12 -true 0 0 0 0 0.0 0 7260 12/28/10 0 2010-12-28T11:30:12.150 2010 12 -true 0 0 0 0 0.0 0 7250 12/27/10 0 2010-12-27T11:20:11.700 2010 12 -true 0 0 0 0 0.0 0 7240 12/26/10 0 2010-12-26T11:10:11.250 2010 12 -true 0 0 0 0 0.0 0 7230 12/25/10 0 2010-12-25T11:00:10.800 2010 12 -true 0 0 0 0 0.0 0 7220 12/24/10 0 2010-12-24T10:50:10.350 2010 12 -true 0 0 0 0 0.0 0 7210 12/23/10 0 2010-12-23T10:40:09.900 2010 12 -true 0 0 0 0 0.0 0 7200 12/22/10 0 2010-12-22T10:30:09.450 2010 12 - --- !all_types_smallint_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_smallint_col_topn_abs_desc -- -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 -true 0 0 0 0 0.0 0 7280 12/30/10 0 2010-12-30T11:50:13.500 2010 12 -true 0 0 0 0 0.0 0 7270 12/29/10 0 2010-12-29T11:40:12.600 2010 12 -true 0 0 0 0 0.0 0 7260 12/28/10 0 2010-12-28T11:30:12.150 2010 12 -true 0 0 0 0 0.0 0 7250 12/27/10 0 2010-12-27T11:20:11.700 2010 12 -true 0 0 0 0 0.0 0 7240 12/26/10 0 2010-12-26T11:10:11.250 2010 12 -true 0 0 0 0 0.0 0 7230 12/25/10 0 2010-12-25T11:00:10.800 2010 12 -true 0 0 0 0 0.0 0 7220 12/24/10 0 2010-12-24T10:50:10.350 2010 12 -true 0 0 0 0 0.0 0 7210 12/23/10 0 2010-12-23T10:40:09.900 2010 12 -true 0 0 0 0 0.0 0 7200 12/22/10 0 2010-12-22T10:30:09.450 2010 12 - --- !all_types_int_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_int_col_topn_abs_desc -- -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 -true 0 0 0 0 0.0 0 7280 12/30/10 0 2010-12-30T11:50:13.500 2010 12 -true 0 0 0 0 0.0 0 7270 12/29/10 0 2010-12-29T11:40:12.600 2010 12 -true 0 0 0 0 0.0 0 7260 12/28/10 0 2010-12-28T11:30:12.150 2010 12 -true 0 0 0 0 0.0 0 7250 12/27/10 0 2010-12-27T11:20:11.700 2010 12 -true 0 0 0 0 0.0 0 7240 12/26/10 0 2010-12-26T11:10:11.250 2010 12 -true 0 0 0 0 0.0 0 7230 12/25/10 0 2010-12-25T11:00:10.800 2010 12 -true 0 0 0 0 0.0 0 7220 12/24/10 0 2010-12-24T10:50:10.350 2010 12 -true 0 0 0 0 0.0 0 7210 12/23/10 0 2010-12-23T10:40:09.900 2010 12 -true 0 0 0 0 0.0 0 7200 12/22/10 0 2010-12-22T10:30:09.450 2010 12 - --- !all_types_bigint_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_bigint_col_topn_abs_desc -- -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 -true 0 0 0 0 0.0 0 7280 12/30/10 0 2010-12-30T11:50:13.500 2010 12 -true 0 0 0 0 0.0 0 7270 12/29/10 0 2010-12-29T11:40:12.600 2010 12 -true 0 0 0 0 0.0 0 7260 12/28/10 0 2010-12-28T11:30:12.150 2010 12 -true 0 0 0 0 0.0 0 7250 12/27/10 0 2010-12-27T11:20:11.700 2010 12 -true 0 0 0 0 0.0 0 7240 12/26/10 0 2010-12-26T11:10:11.250 2010 12 -true 0 0 0 0 0.0 0 7230 12/25/10 0 2010-12-25T11:00:10.800 2010 12 -true 0 0 0 0 0.0 0 7220 12/24/10 0 2010-12-24T10:50:10.350 2010 12 -true 0 0 0 0 0.0 0 7210 12/23/10 0 2010-12-23T10:40:09.900 2010 12 -true 0 0 0 0 0.0 0 7200 12/22/10 0 2010-12-22T10:30:09.450 2010 12 - --- !all_types_float_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_float_col_topn_abs_desc -- -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 -true 0 0 0 0 0.0 0 7280 12/30/10 0 2010-12-30T11:50:13.500 2010 12 -true 0 0 0 0 0.0 0 7270 12/29/10 0 2010-12-29T11:40:12.600 2010 12 -true 0 0 0 0 0.0 0 7260 12/28/10 0 2010-12-28T11:30:12.150 2010 12 -true 0 0 0 0 0.0 0 7250 12/27/10 0 2010-12-27T11:20:11.700 2010 12 -true 0 0 0 0 0.0 0 7240 12/26/10 0 2010-12-26T11:10:11.250 2010 12 -true 0 0 0 0 0.0 0 7230 12/25/10 0 2010-12-25T11:00:10.800 2010 12 -true 0 0 0 0 0.0 0 7220 12/24/10 0 2010-12-24T10:50:10.350 2010 12 -true 0 0 0 0 0.0 0 7210 12/23/10 0 2010-12-23T10:40:09.900 2010 12 -true 0 0 0 0 0.0 0 7200 12/22/10 0 2010-12-22T10:30:09.450 2010 12 - --- !all_types_double_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_double_col_topn_abs_desc -- -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 -true 0 0 0 0 0.0 0 7280 12/30/10 0 2010-12-30T11:50:13.500 2010 12 -true 0 0 0 0 0.0 0 7270 12/29/10 0 2010-12-29T11:40:12.600 2010 12 -true 0 0 0 0 0.0 0 7260 12/28/10 0 2010-12-28T11:30:12.150 2010 12 -true 0 0 0 0 0.0 0 7250 12/27/10 0 2010-12-27T11:20:11.700 2010 12 -true 0 0 0 0 0.0 0 7240 12/26/10 0 2010-12-26T11:10:11.250 2010 12 -true 0 0 0 0 0.0 0 7230 12/25/10 0 2010-12-25T11:00:10.800 2010 12 -true 0 0 0 0 0.0 0 7220 12/24/10 0 2010-12-24T10:50:10.350 2010 12 -true 0 0 0 0 0.0 0 7210 12/23/10 0 2010-12-23T10:40:09.900 2010 12 -true 0 0 0 0 0.0 0 7200 12/22/10 0 2010-12-22T10:30:09.450 2010 12 - --- !all_types_id_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_id_topn_abs_desc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_date_string_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_date_string_col_topn_abs_desc -- -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 -false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 -false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 -true 4 4 4 40 4.4 40.4 7294 12/31/10 4 2010-12-31T12:04:13.560 2010 12 -false 3 3 3 30 3.3 30.3 7293 12/31/10 3 2010-12-31T12:03:13.530 2010 12 -true 2 2 2 20 2.2 20.2 7292 12/31/10 2 2010-12-31T12:02:13.510 2010 12 -false 1 1 1 10 1.1 10.1 7291 12/31/10 1 2010-12-31T12:01:13.500 2010 12 -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 - --- !all_types_string_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -true 0 0 0 0 0.0 0 10 01/02/09 0 2009-01-02T07:10:00.450 2009 1 -true 0 0 0 0 0.0 0 20 01/03/09 0 2009-01-03T07:20:00.900 2009 1 -true 0 0 0 0 0.0 0 30 01/04/09 0 2009-01-04T07:30:01.350 2009 1 -true 0 0 0 0 0.0 0 40 01/05/09 0 2009-01-05T07:40:01.800 2009 1 -true 0 0 0 0 0.0 0 50 01/06/09 0 2009-01-06T07:50:02.250 2009 1 -true 0 0 0 0 0.0 0 60 01/07/09 0 2009-01-07T08:00:02.700 2009 1 -true 0 0 0 0 0.0 0 70 01/08/09 0 2009-01-08T08:10:03.150 2009 1 -true 0 0 0 0 0.0 0 80 01/09/09 0 2009-01-09T08:20:03.600 2009 1 -true 0 0 0 0 0.0 0 90 01/10/09 0 2009-01-10T08:30:04.500 2009 1 - --- !all_types_string_col_topn_abs_desc -- -true 0 0 0 0 0.0 0 7290 12/31/10 0 2010-12-31T12:00:13.500 2010 12 -true 0 0 0 0 0.0 0 7280 12/30/10 0 2010-12-30T11:50:13.500 2010 12 -true 0 0 0 0 0.0 0 7270 12/29/10 0 2010-12-29T11:40:12.600 2010 12 -true 0 0 0 0 0.0 0 7260 12/28/10 0 2010-12-28T11:30:12.150 2010 12 -true 0 0 0 0 0.0 0 7250 12/27/10 0 2010-12-27T11:20:11.700 2010 12 -true 0 0 0 0 0.0 0 7240 12/26/10 0 2010-12-26T11:10:11.250 2010 12 -true 0 0 0 0 0.0 0 7230 12/25/10 0 2010-12-25T11:00:10.800 2010 12 -true 0 0 0 0 0.0 0 7220 12/24/10 0 2010-12-24T10:50:10.350 2010 12 -true 0 0 0 0 0.0 0 7210 12/23/10 0 2010-12-23T10:40:09.900 2010 12 -true 0 0 0 0 0.0 0 7200 12/22/10 0 2010-12-22T10:30:09.450 2010 12 - --- !all_types_timestamp_col_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_timestamp_col_topn_abs_desc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_year_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_year_topn_abs_desc -- -false 9 9 9 90 9.9 90.89999999999999 3649 12/31/09 9 2009-12-31T12:09:13.860 2009 12 -true 8 8 8 80 8.8 80.8 3648 12/31/09 8 2009-12-31T12:08:13.780 2009 12 -false 7 7 7 70 7.7 70.7 3647 12/31/09 7 2009-12-31T12:07:13.710 2009 12 -true 6 6 6 60 6.6 60.59999999999999 3646 12/31/09 6 2009-12-31T12:06:13.650 2009 12 -false 5 5 5 50 5.5 50.5 3645 12/31/09 5 2009-12-31T12:05:13.600 2009 12 -true 4 4 4 40 4.4 40.4 3644 12/31/09 4 2009-12-31T12:04:13.560 2009 12 -false 3 3 3 30 3.3 30.3 3643 12/31/09 3 2009-12-31T12:03:13.530 2009 12 -true 2 2 2 20 2.2 20.2 3642 12/31/09 2 2009-12-31T12:02:13.510 2009 12 -false 1 1 1 10 1.1 10.1 3641 12/31/09 1 2009-12-31T12:01:13.500 2009 12 -true 0 0 0 0 0.0 0 3640 12/31/09 0 2009-12-31T12:00:13.500 2009 12 - --- !all_types_month_topn_abs_asc -- -true 0 0 0 0 0.0 0 0 01/01/09 0 2009-01-01T07:00 2009 1 -false 1 1 1 10 1.1 10.1 1 01/01/09 1 2009-01-01T07:01 2009 1 -true 2 2 2 20 2.2 20.2 2 01/01/09 2 2009-01-01T07:02:00.100 2009 1 -false 3 3 3 30 3.3 30.3 3 01/01/09 3 2009-01-01T07:03:00.300 2009 1 -true 4 4 4 40 4.4 40.4 4 01/01/09 4 2009-01-01T07:04:00.600 2009 1 -false 5 5 5 50 5.5 50.5 5 01/01/09 5 2009-01-01T07:05:00.100 2009 1 -true 6 6 6 60 6.6 60.59999999999999 6 01/01/09 6 2009-01-01T07:06:00.150 2009 1 -false 7 7 7 70 7.7 70.7 7 01/01/09 7 2009-01-01T07:07:00.210 2009 1 -true 8 8 8 80 8.8 80.8 8 01/01/09 8 2009-01-01T07:08:00.280 2009 1 -false 9 9 9 90 9.9 90.89999999999999 9 01/01/09 9 2009-01-01T07:09:00.360 2009 1 - --- !all_types_month_topn_abs_desc -- -false 9 9 9 90 9.9 90.89999999999999 3959 01/31/10 9 2010-01-31T12:09:13.860 2010 1 -true 8 8 8 80 8.8 80.8 3958 01/31/10 8 2010-01-31T12:08:13.780 2010 1 -false 7 7 7 70 7.7 70.7 3957 01/31/10 7 2010-01-31T12:07:13.710 2010 1 -true 6 6 6 60 6.6 60.59999999999999 3956 01/31/10 6 2010-01-31T12:06:13.650 2010 1 -false 5 5 5 50 5.5 50.5 3955 01/31/10 5 2010-01-31T12:05:13.600 2010 1 -true 4 4 4 40 4.4 40.4 3954 01/31/10 4 2010-01-31T12:04:13.560 2010 1 -false 3 3 3 30 3.3 30.3 3953 01/31/10 3 2010-01-31T12:03:13.530 2010 1 -true 2 2 2 20 2.2 20.2 3952 01/31/10 2 2010-01-31T12:02:13.510 2010 1 -false 1 1 1 10 1.1 10.1 3951 01/31/10 1 2010-01-31T12:01:13.500 2010 1 -true 0 0 0 0 0.0 0 3950 01/31/10 0 2010-01-31T12:00:13.500 2010 1 - --- !schema_1 -- -1 638 6 15635 32.00 49620.16 0.07 0.02 N O 1996-01-30 1996-02-07 1996-02-03 DELIVER IN PERSON MAIL arefully slyly ex cn beijing - --- !schema_2 -- -6374628540732951412 -77 -65 -70 -107 -215 65 0 -526 -1309 3750 8827 -19795 34647 57042 -1662 -138248 -890685 -228568 1633079 -2725524 6163040 -10491702 697237 74565050 127767368 93532213 -209675435 -32116110 -3624917040 -2927805617 15581947241 21893441661 24075494509 -116822110531 -59683724667 -146210393388 114424524398 1341560771667 -1638742564263 520137948334 -2927347587131 7415137351179 -7963937754617 52157548982266 140803519083304 -294675355729619 -868076759504942 181128508165910 -91753231238823 -3511241416682881 -11545256318348796 -1952917510863468 -5161099825338866 -59726090170689781 287170105829528178 607326725526282735 1253194074103207461 -162443950414676064 -2964036188567341159 2602201580810990248 5581917084094110764 111739292249520611 -315687754593838642 -2804420462762366976 -2078683524 - --- !schema_3 -- -false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 -false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 -false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 -true 6 6 6 60 6.6 60.59999999999999 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 -true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 - --- !schema_4 -- -2 24 15314771 999319712124142303 true 6.009337E8 4.817722807977021e+16 \N northern rural 2022-08-30T23:21:08 407186.2849 phones int_col 2019-01-01 [2.595433907849411e+17, 5.88165568758352e+17, 4.780259987226574e+17, 6.926622881251557e+17, 9.86405645575228e+17] \N phones int_col -5 59 317349992 998913039814974432 false 5.6584858E8 9.900861328269033e+17 Handling man satisfy firework descent top. Racing closed county set-up crown cave. Correctly front duration pure. \N 2022-09-02T19:52:57 372765.2493 desktops tinyint_col 2021-10-03 [9.983261252571983e+17, 3.612076153030643e+17, 9.969131496509435e+17, 8.991290717923475e+17, 1.195589374709888e+17] ["CrySxz", "FMXGRcaGbahSVqhp", "oRKqPmhM", "VdODasEdDWFSRIQf"] desktops tinyint_col -6 62 915699741 999653836472045196 true 4.51937504E8 8.796150544502191e+17 Tale get speed platform august curved. Ease grass neighbour landlord. Baby genetic youth. \N 2022-08-07T09:30:56 875620.2176 phones smallint_col \N [9.423540715161855e+17, 4.833249992029562e+17, 9.167007747789834e+17] ["zNfbLeFx", "GNTJOmWJyRmOK", "hwvfhSQGsaaMEqUrWCK", "cQrQsROKLARA", "nONj", "oepXBFB", "IPtUql"] phones smallint_col - --- !schema_5 -- -00cwjIryUv EXHwpeK2Nl hv2PYEMYMM eo69nyw4Yv K6797tgjFg LlFNd8Kyy5 wkpLCO3uo1 AIXCj1MfeD ni0HxZbiUO 6IjRdM8Gqi qsTMK6A2eC 1wu7v9OPwW qavArd9tDc sU88hZADLj lyzWlwLOCx 2022-11-25 - --- !schema_6 -- -"" "test" - --- !schema_7 -- -\N \N \N \N \N \N \N \N \N test test test 1 2 3 4 5.1 6.2 true false 2011-05-06 2011-05-06T07:08:09.123 -1.2 12.30 -1234.5678 123456789.12340000 -1234567890.12345678 1234567890123456789012.1234567800000000 dGVzdDI= - diff --git a/regression-test/data/external_table_p0/hive/test_hive_openx_json.out b/regression-test/data/external_table_p0/hive/test_hive_openx_json.out index 6eadea56694c85..f4fd28e4d05725 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_openx_json.out +++ b/regression-test/data/external_table_p0/hive/test_hive_openx_json.out @@ -10,6 +10,7 @@ \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N +\N \N \N \N \N 1 Alice [1, 2, 3] {"math":90, "english":85} {"a":100, "b":"test1", "c":1234567890} 2 Bob [4, 5] {"math":80, "science":95} {"a":200, "b":"test2", "c":9876543210} diff --git a/regression-test/data/external_table_p0/hive/test_hive_schema_evolution.out b/regression-test/data/external_table_p0/hive/test_hive_schema_evolution.out index dbea5056998664..1cb5cde15144e4 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_schema_evolution.out +++ b/regression-test/data/external_table_p0/hive/test_hive_schema_evolution.out @@ -35,39 +35,3 @@ \N 2023-01-01T13:01:03 --- !q01 -- -1 kaka \N -2 messi 2023-01-01T13:01:03 - --- !q02 -- -1 kaka \N -2 messi 2023-01-01T13:01:03 - --- !q03 -- -\N -2023-01-01T13:01:03 - --- !q01 -- -1 kaka \N -2 messi 2023-01-01T21:01:03 - --- !q02 -- -1 kaka \N -2 messi 2023-01-01T21:01:03 - --- !q03 -- -\N -2023-01-01T21:01:03 - --- !q01 -- -1 kaka \N -2 messi 2023-01-01T13:01:03 - --- !q02 -- -1 kaka \N -2 messi 2023-01-01T13:01:03 - --- !q03 -- -\N -2023-01-01T13:01:03 - diff --git a/regression-test/data/external_table_p0/hive/write/test_hive_write_insert.out b/regression-test/data/external_table_p0/hive/write/test_hive_write_insert.out index 932b62b5034b94..d3df453f105971 100644 --- a/regression-test/data/external_table_p0/hive/write/test_hive_write_insert.out +++ b/regression-test/data/external_table_p0/hive/write/test_hive_write_insert.out @@ -21,232 +21,6 @@ false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -1 -- !q05 -- \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 --- !q06 -- - --- !q01 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q02 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q03 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q04 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N - --- !q05 -- - --- !q01 -- -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q02 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q03 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q04 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -123456.789012 -123456789.012345678901 str binary_value 2024-03-25 2024-03-25T12:00 2024-03-25T12:00:00.123457 2024-03-25T12:00:00.123457 char_value11111 char_value22222 char_value33333 varchar_value11111 varchar_value22222 varchar_value33333 {"key7":"value1"} {"key7":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {5.3456:2.3456} {5.34567890:2.34567890} {2.34567890:2.34567890} {7.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [9.4567, 4.5678] [6.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240321 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q05 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q01 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q02 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q03 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240322 -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240320 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q04 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240322 -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240320 - --- !q01 -- -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q02 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q03 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q04 -- -false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -123456.789012 -123456789.012345678901 str binary_value 2024-03-25 2024-03-25T12:00 2024-03-25T12:00:00.123457 2024-03-25T12:00:00.123457 char_value11111 char_value22222 char_value33333 varchar_value11111 varchar_value22222 varchar_value33333 {"key7":"value1"} {"key7":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {5.3456:2.3456} {5.34567890:2.34567890} {2.34567890:2.34567890} {7.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [9.4567, 4.5678] [6.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240325 - --- !q05 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 - --- !q06 -- - --- !q01 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q02 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q03 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q04 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N - --- !q05 -- - --- !q01 -- -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q02 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q03 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q04 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -123456.789012 -123456789.012345678901 str binary_value 2024-03-25 2024-03-25T12:00 2024-03-25T12:00:00.123457 2024-03-25T12:00:00.123457 char_value11111 char_value22222 char_value33333 varchar_value11111 varchar_value22222 varchar_value33333 {"key7":"value1"} {"key7":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {5.3456:2.3456} {5.34567890:2.34567890} {2.34567890:2.34567890} {7.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [9.4567, 4.5678] [6.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240321 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q05 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q01 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q02 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q03 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240322 -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240320 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123456 2024-03-20T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q04 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240322 -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240320 - --- !q01 -- -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q02 -- -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q03 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 -false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 -true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 - --- !q04 -- -false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -123456.789012 -123456789.012345678901 str binary_value 2024-03-25 2024-03-25T12:00 2024-03-25T12:00:00.123457 2024-03-25T12:00:00.123457 char_value11111 char_value22222 char_value33333 varchar_value11111 varchar_value22222 varchar_value33333 {"key7":"value1"} {"key7":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {5.3456:2.3456} {5.34567890:2.34567890} {2.34567890:2.34567890} {7.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [9.4567, 4.5678] [6.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240325 - --- !q05 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 - --- !q06 -- - -- !q01 -- false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 @@ -276,8 +50,6 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 \N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N --- !q05 -- - -- !q01 -- true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 @@ -357,8 +129,6 @@ false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -1 -- !q05 -- \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 --- !q06 -- - -- !q01 -- false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123456 2024-03-22T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] ["varchar1", "varchar2"] ["char1", "char2"] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 @@ -388,8 +158,6 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 \N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N --- !q05 -- - -- !q01 -- true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] ["varchar1", "varchar2"] ["char1", "char2"] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_deletion_vector.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_deletion_vector.out index 41819b90bd731a..6d3531f345ea8f 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_deletion_vector.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_deletion_vector.out @@ -11,40 +11,97 @@ 6 1 f 8 1 h +-- !q3 -- +2 1 b +6 1 f +8 1 h +10 2 j +12 2 l + +-- !no_delete -- +1 1 a +2 1 b +3 1 c +4 1 d +5 1 e +6 1 f +7 1 g +8 1 h + +-- !equality_only -- +1 a +2 b + +-- !position_only_count -- +5632 + +-- !position_and_dv_count -- +5120 + +-- !equality_and_dv -- +2 1 b +4 1 d +6 1 f +8 1 h +12 2 l + +-- !split_cache_single_file_count -- +2048 4192256 + -- !q4 -- 2 1 b -- !q5 -- 2 1 b +-- !q5_orc -- +2 1 b + -- !q6 -- -- !q7 -- +-- !q7_orc -- + -- !q8 -- 2 1 b -- !q9 -- 2 1 b +-- !q9_orc -- +2 1 b + -- !q10 -- 5 -- !q11 -- 3 +-- !q11_orc -- +5 + +-- !no_delete_count -- +8 + -- !q12 -- 5 -- !q13 -- 3 +-- !q13_orc -- +5 + -- !q14 -- 5 -- !q15 -- 3 +-- !q15_orc -- +5 + -- !q16 -- 33334 @@ -63,3 +120,65 @@ -- !q21 -- 33334 +-- !q22 -- +12 2 l +10 2 j +8 1 h + +-- !q23 -- +8 1 h +6 1 f +2 1 b + +-- !q24 -- +1 3 16 +2 2 22 + +-- !q25 -- +1 3 16 + +-- !q26 -- +1 3 16 +2 2 22 + +-- !q27 -- +2 b +6 f + +-- !q28 -- +2 b +6 f + +-- !q29 -- +2 b +6 f + +-- !q30 -- +100 + +-- !q31 -- +100 + +-- !delete_type_matrix -- +dv_equality 0 PARQUET 12 12 +dv_equality 1 PUFFIN 6 6 +dv_equality 2 PARQUET 1 1 +dv_only 0 PARQUET 10 10 +dv_only 1 PUFFIN 5 5 +dv_position 0 PARQUET 1 7680 +dv_position 1 PUFFIN 1 2560 +equality_only 0 PARQUET 2 2 +equality_only 2 PARQUET 2 2 +no_delete 0 PARQUET 8 8 +position_only 0 PARQUET 1 7680 +position_only 1 PARQUET 4 2048 + +-- !position_delete_ignored_with_dv -- + +-- !equality_delete_active_with_dv -- +2 1 b +4 1 d +6 1 f +8 1 h +12 2 l + diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.out index 529e37390f05bc..bfc73649139041 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.out @@ -8,62 +8,62 @@ id int Yes true \N ts_tz timestamptz(6) Yes true \N WITH_TIMEZONE -- !select_tvf0 -- -1 2025-01-01 00:00:00+08:00 -2 2025-06-01 12:34:56+08:00 -3 2025-12-31 23:59:59+08:00 +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 4 \N -- !select_tvf0_desc -- id int Yes false \N NONE -ts_tz timestamptz Yes false \N NONE +ts_tz timestamptz(6) Yes false \N NONE -- !select_tvf0_false -- -1 2025-01-01 00:00:00+08:00 -2 2025-06-01 12:34:56+08:00 -3 2025-12-31 23:59:59+08:00 +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 4 \N -- !select_tvf0_desc_false -- id int Yes false \N NONE -ts_tz timestamptz Yes false \N NONE +ts_tz timestamptz(6) Yes false \N NONE -- !select_tvf1 -- -1 2025-01-01 00:00:00+08:00 -2 2025-06-01 12:34:56+08:00 -3 2025-12-31 23:59:59+08:00 +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 4 \N -- !select_tvf1_desc -- id int Yes false \N NONE -ts_tz timestamptz Yes false \N NONE +ts_tz timestamptz(6) Yes false \N NONE -- !select_tvf1_false -- -1 2025-01-01 00:00:00+08:00 -2 2025-06-01 12:34:56+08:00 -3 2025-12-31 23:59:59+08:00 +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 4 \N -- !select_tvf1_desc_false -- id int Yes false \N NONE -ts_tz timestamptz Yes false \N NONE +ts_tz timestamptz(6) Yes false \N NONE -- !select_tvf2 -- -1 2025-01-01 00:00:00+08:00 -2 2025-06-01 12:34:56+08:00 -3 2025-12-31 23:59:59+08:00 +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 4 \N -- !select_tvf2_desc -- id int Yes false \N NONE -ts_tz timestamptz Yes false \N NONE +ts_tz timestamptz(6) Yes false \N NONE -- !select_tvf3 -- -1 2025-01-01 00:00:00+08:00 -2 2025-06-01 12:34:56+08:00 -3 2025-12-31 23:59:59+08:00 +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 4 \N -- !select_tvf3_desc -- id int Yes false \N NONE -ts_tz timestamptz Yes false \N NONE +ts_tz timestamptz(6) Yes false \N NONE diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out index 1b8af743c6a0f2..be0c0e2236450e 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc -- id int Yes true \N -test:a1b2.raw.abc-gg-1-a text Yes true \N +TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N -- !q_1 -- 1 row1 @@ -29,7 +29,7 @@ test:a1b2.raw.abc-gg-1-a text Yes true \N -- !desc -- id int Yes true \N -test:a1b2.raw.abc-gg-1-a text Yes true \N +TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N -- !q_1 -- 1 row1 @@ -54,4 +54,3 @@ test:a1b2.raw.abc-gg-1-a text Yes true \N 3 row3 2 row2 1 row1 - diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.out b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.out index 66207238741815..6a6ebab9001e43 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.out @@ -28,11 +28,11 @@ ts_ltz timestamptz(3) Yes true \N WITH_TIMEZONE 3 2024-11-11 11:11:11.123+08:00 -- !mapping_tz -- -1 2024-01-01 10:00:00+08:00 -2 2026-01-06 16:13:12+08:00 -3 2024-11-11 11:11:11+08:00 +1 2024-01-01 10:00:00.000+08:00 +2 2026-01-06 16:13:12.000+08:00 +3 2024-11-11 11:11:11.123+08:00 -- !mapping_tz_desc -- id int Yes false \N NONE -ts_ltz timestamptz Yes false \N NONE +ts_ltz timestamptz(3) Yes false \N NONE diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out index f0b1e92a088538..d8b9e54710109b 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out @@ -35,6 +35,18 @@ -- !9 -- 7 +-- !10 -- +3 3_1 + +-- !11 -- +3 3_1 + +-- !12 -- +4 + +-- !13 -- +4 + -- !1 -- 3 @@ -71,3 +83,15 @@ -- !9 -- 7 +-- !10 -- +3 3_1 + +-- !11 -- +3 3_1 + +-- !12 -- +4 + +-- !13 -- +4 + diff --git a/regression-test/data/external_table_p0/tvf/csv_enclose_state.csv b/regression-test/data/external_table_p0/tvf/csv_enclose_state.csv new file mode 100644 index 00000000000000..f6259106d54027 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/csv_enclose_state.csv @@ -0,0 +1,3 @@ +id,name,score,extra +1,ab"cd,20,x +2,C:\dir\,30,40,y diff --git a/regression-test/data/external_table_p0/tvf/csv_matching_escape_enclose.csv b/regression-test/data/external_table_p0/tvf/csv_matching_escape_enclose.csv new file mode 100644 index 00000000000000..adbafedf68ee96 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/csv_matching_escape_enclose.csv @@ -0,0 +1,2 @@ +id,name,score +"1","alice","10" diff --git a/regression-test/data/external_table_p0/tvf/csv_quoted_null.csv b/regression-test/data/external_table_p0/tvf/csv_quoted_null.csv new file mode 100644 index 00000000000000..83c42bb7104fb5 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/csv_quoted_null.csv @@ -0,0 +1,3 @@ +id,name,score +1,"\N",10 +2,\N,20 diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out index 0e21a8fad6f690..015d9391317356 100644 Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out index 16b89ac45d63ca..79b63e41cc1b4d 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out @@ -24,14 +24,14 @@ apple_banana_mango81 apple_banana_mango9 -- !test_2 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 -- !test_3 -- [{"one":"0 - 0 - 1", "two":"0 - 0 - 2", "three":"0 - 0 - 3"}, {"one":"0 - 1 - 1", "two":"0 - 1 - 2", "three":"0 - 1 - 3"}] @@ -39,14 +39,14 @@ apple_banana_mango9 [{"one":"2 - 0 - 1", "two":"2 - 0 - 2", "three":"2 - 0 - 3"}, {"one":"2 - 1 - 1", "two":"2 - 1 - 2", "three":"2 - 1 - 3"}] -- !test_4 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 -- !test_5 -- ["good", "bye"] @@ -89,17 +89,17 @@ apple_banana_mango9 1981-01-07T00:00 15.8 1981-01-08T00:00 17.4 1981-01-09T00:00 21.8 -1981-01-10T00:00 20.0 +1981-01-10T00:00 20 -- !test_13 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 -- !test_14 -- [{"one":"First inner", "two":null, "three":null}, {"one":null, "two":"Second inner", "three":null}, {"one":null, "two":null, "three":"Third inner"}] @@ -119,17 +119,17 @@ apple_banana_mango9 -- !test_16 -- 1 Alice 2022-11-16T02:32:09 2 Bob 2022-11-16T02:32:09 -3 Cecilia 2022-11-16T02:32:09 +3 Cecilia 2022-11-16T02:32:09.123534 -- !test_17 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 -- !test_18 -- 0.00 @@ -151,14 +151,14 @@ apple_banana_mango9 2 -- !test_20 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 -- !test_21 -- 1001-01-07 1001-01-07 @@ -171,49 +171,49 @@ apple_banana_mango9 1001-01-07 1001-01-14 -- !test_22 -- -1001-01-07T17:07:47.171 1001-01-07T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-08T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-09T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-10T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-11T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-12T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-13T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-14T17:07:47.171 +1001-01-07T17:07:46.123 1001-01-07T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-08T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-09T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-10T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-11T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-12T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-13T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-14T17:07:46.123 -- !test_23 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 -- !test_24 -- false 1 2 3 10 1.2 val_1 val_1 HEARTS false 1 2 3 10 1.2 val_1 val_1 HEARTS ["arr_1", "arr_2", "arr_3"] [1] {1:"val_1", 2:"val_2", 3:"val_3"} {1:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 2:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 3:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}]} false 3 4 5 30 3.2 val_3 val_3 CLUBS \N \N \N \N \N \N \N \N \N ["arr_3", "arr_4", "arr_5"] [3] {3:"val_3", 4:"val_4", 5:"val_5"} {3:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 4:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 5:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}]} false 5 6 7 50 5.2 val_5 val_5 HEARTS false 5 6 7 50 5.2 val_5 val_5 HEARTS ["arr_5", "arr_6", "arr_7"] [5] {5:"val_5", 6:"val_6", 7:"val_7"} {5:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 6:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 7:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}]} false 7 8 9 70 7.2 val_7 val_7 CLUBS false 7 8 9 70 7.2 val_7 val_7 CLUBS ["arr_7", "arr_8", "arr_9"] [7] {7:"val_7", 8:"val_8", 9:"val_9"} {7:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 8:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 9:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}]} -false 9 10 11 90 9.2 val_9 val_9 HEARTS \N \N \N \N \N \N \N \N \N ["arr_9", "arr_10", "arr_11"] [9] {9:"val_9", 10:"val_10", 11:"val_11"} {9:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 10:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 11:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}]} +false 9 10 11 90 9.199999999999999 val_9 val_9 HEARTS \N \N \N \N \N \N \N \N \N ["arr_9", "arr_10", "arr_11"] [9] {9:"val_9", 10:"val_10", 11:"val_11"} {9:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 10:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 11:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}]} true 0 1 2 0 0.2 val_0 val_0 SPADES \N \N \N \N \N \N \N \N \N ["arr_0", "arr_1", "arr_2"] [0] {0:"val_0", 1:"val_1", 2:"val_2"} {0:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 1:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 2:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}]} true 2 3 4 20 2.2 val_2 val_2 DIAMONDS true 2 3 4 20 2.2 val_2 val_2 DIAMONDS ["arr_2", "arr_3", "arr_4"] [2] {2:"val_2", 3:"val_3", 4:"val_4"} {2:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 3:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 4:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}]} true 4 5 6 40 4.2 val_4 val_4 SPADES true 4 5 6 40 4.2 val_4 val_4 SPADES ["arr_4", "arr_5", "arr_6"] [4] {4:"val_4", 5:"val_5", 6:"val_6"} {4:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 5:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 6:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}]} true 6 7 8 60 6.2 val_6 val_6 DIAMONDS \N \N \N \N \N \N \N \N \N ["arr_6", "arr_7", "arr_8"] [6] {6:"val_6", 7:"val_7", 8:"val_8"} {6:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 7:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 8:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}]} -true 8 9 10 80 8.2 val_8 val_8 SPADES true 8 9 10 80 8.2 val_8 val_8 SPADES ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} +true 8 9 10 80 8.199999999999999 val_8 val_8 SPADES true 8 9 10 80 8.199999999999999 val_8 val_8 SPADES ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} -- !test_25 -- {"duration":"111222333444"} -- !test_26 -- -1001-01-07T17:07:47.171 1001-01-07T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-08T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-09T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-10T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-11T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-12T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-13T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-14T17:07:47.171 +1001-01-07T17:07:46.123 1001-01-07T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-08T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-09T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-10T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-11T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-12T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-13T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-14T17:07:46.123 -- !test_27 -- 1001-01-07 1001-01-07 @@ -238,14 +238,14 @@ true 8 9 10 80 8.2 val_8 val_8 SPADES true 8 9 10 80 8.2 val_8 val_8 SPADES ["ar 9.00 -- !test_29 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 -- !test_30 -- \N @@ -259,23 +259,20 @@ true 8 9 10 80 8.2 val_8 val_8 SPADES true 8 9 10 80 8.2 val_8 val_8 SPADES ["ar 8.4 93.7 --- !test_31 -- -{"list":[{"element":"hello"}]} - -- !test_32 -- 1970-01-01T08:00:00.010 1970-01-01T08:00:00.010 1970-01-01T08:00:00.010 -- !test_33 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 -- !test_34 -- 1001-01-07 1001-01-07 @@ -288,22 +285,22 @@ true 8 9 10 80 8.2 val_8 val_8 SPADES true 8 9 10 80 8.2 val_8 val_8 SPADES ["ar 1001-01-07 1001-01-14 -- !test_35 -- -1001-01-07T17:07:47.171 1001-01-07T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-08T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-09T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-10T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-11T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-12T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-13T17:07:47.171 -1001-01-07T17:07:47.171 1001-01-14T17:07:47.171 +1001-01-07T17:07:46.123 1001-01-07T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-08T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-09T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-10T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-11T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-12T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-13T17:07:46.123 +1001-01-07T17:07:46.123 1001-01-14T17:07:46.123 -- !test_36 -- -1001-01-07T17:07:47.172032 1001-01-07T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-08T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-09T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-10T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-11T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-12T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-13T17:07:47.172032 -1001-01-07T17:07:47.172032 1001-01-14T17:07:47.172032 +1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 +1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out index 368a1728c941e1..93c2fd8c672e39 100644 Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out index 816aefbc495efc..d694f2db141554 100644 Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out index 38d457d1069867..c7b9542f5c7662 100644 Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index a797eca8601867..4fe42d7fcdcc77 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -736,12 +736,25 @@ true -- !test_86 -- 3 {"c2_2":{30:{"c2_2_3":"Hangzhou"}}, "c2_3":{"c2_3_2":null}, "c2_4":{"c2_4_1":null}} [null, {{"c3_1":300, "c3_2":null}:null}, {{"c3_1":null, "c3_2":1}:[null, {"c3_3_1":null}, null, {"c3_3_1":"2003-01-01"}]}] +-- !test_87 -- +1 01:02:03 +2 02:03:04 +3 03:04:05 +4 \N + -- !test_88 -- 1 ["a", "b"] 2 ["c", "d"] -- !test_89 -- +-- !test_90 -- +1 a 1 a +2 b 2 a +3 c 3 a +4 d 4 b +5 e 5 b + -- !test_91 -- 11 22 33 44 @@ -795,10 +808,10 @@ true -- !test_98 -- \N \N \N -abcDeFGhijkLmnOp 682.56 1212 -abcDeFGhijkLmnOp 682.56 1212 -abcDeFGhijkLmnOp 682.56 1212 -abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp \N 1212 +abcDeFGhijkLmnOp \N 1212 +abcDeFGhijkLmnOp \N 1212 +abcDeFGhijkLmnOp \N 1212 -- !test_100 -- 1317017856 1 18752152 809291 1089176 19951117 3-MEDIUM 0 40 4801000 16034243 9 4368910 72015 3 19951228 RAIL Customer#018752152 q4gN2btSpiKXdN,6 ALGERIA 1 ALGERIA AFRICA 10-753-996-8708 MACHINERY Supplier#001089176 ROidEL1L6yeFsJqnUjD EGYPT 5 EGYPT MIDDLE EAST 14-807-108-7869 blanched gainsboro MFGR#4 MFGR#43 MFGR#433 brown MEDIUM BRUSHED STEEL 42 MED BAG @@ -853,13 +866,12 @@ abcDeFGhijkLmnOp 682.56 1212 -- !test_107 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 \N 1212 +0x6162634465464768696A6B4C6D6E4F70 \N 1212 +0x6162634465464768696A6B4C6D6E4F70 \N 1212 +0x6162634465464768696A6B4C6D6E4F70 \N 1212 -- !test_107_desc -- decimal_flba decimal(5,2) Yes false \N NONE interval text Yes false \N NONE uuid varbinary(16) Yes false \N NONE - diff --git a/regression-test/data/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.out b/regression-test/data/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.out new file mode 100644 index 00000000000000..b3b0f227aca48b --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !enclose_state -- +1 20 x +2 30 40 + +-- !matching_escape_enclose -- +1 alice 10 + +-- !quoted_null -- +1 5C4E false +2 \N true + diff --git a/regression-test/pipeline/external/conf/be.conf b/regression-test/pipeline/external/conf/be.conf index f29f342366e273..beea7be7175c38 100644 --- a/regression-test/pipeline/external/conf/be.conf +++ b/regression-test/pipeline/external/conf/be.conf @@ -53,7 +53,8 @@ priority_networks=172.19.0.0/24 enable_fuzzy_mode=true max_depth_of_expr_tree=200 enable_feature_binlog=true -max_sys_mem_available_low_water_mark_bytes=69206016 +mem_limit=35% +max_sys_mem_available_low_water_mark_bytes=2147483648 user_files_secure_path=/ enable_debug_points=true # debug scanner context dead loop diff --git a/regression-test/pipeline/external/conf/fe.conf b/regression-test/pipeline/external/conf/fe.conf index 365c0b9337576e..fc818a862a1a15 100644 --- a/regression-test/pipeline/external/conf/fe.conf +++ b/regression-test/pipeline/external/conf/fe.conf @@ -28,7 +28,7 @@ DATE = `date +%Y%m%d-%H%M%S` JAVA_OPTS="-Xmx4096m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$DORIS_HOME/log/fe.jmap -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintClassHistogramAfterFullGC -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" # For jdk 17+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" +JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" ## ## the lowercase properties are read by main program. diff --git a/regression-test/suites/export_p0/outfile/parquet/test_outfile_parquet_complex_type.groovy b/regression-test/suites/export_p0/outfile/parquet/test_outfile_parquet_complex_type.groovy index 6452576d8ffa1e..b46a9a0bb67f64 100644 --- a/regression-test/suites/export_p0/outfile/parquet/test_outfile_parquet_complex_type.groovy +++ b/regression-test/suites/export_p0/outfile/parquet/test_outfile_parquet_complex_type.groovy @@ -298,6 +298,17 @@ suite("test_outfile_parquet_complex_type", "p0") { // test outfile to s3 def outfile_url = outfile_to_S3() + sql """ set enable_file_scanner_v2 = false; """ + qt_select_load7 """ SELECT * FROM S3 ( + "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.parquet", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "parquet", + "region" = "${region}" + ); + """ + + sql """ set enable_file_scanner_v2 = true; """ qt_select_load7 """ SELECT * FROM S3 ( "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.parquet", "ACCESS_KEY"= "${ak}", diff --git a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy index 0ead2bd030478f..5c95098e91236d 100644 --- a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy +++ b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy @@ -74,7 +74,8 @@ suite("test_hive_read_parquet", "external,hive,external_docker") { FORMAT AS ${format} PROPERTIES ( "fs.defaultFS"="${defaultFS}", - "hadoop.username" = "${hdfsUserName}" + "hadoop.username" = "${hdfsUserName}", + "enable_int96_timestamps" = "true" ); """ logger.info("outfile success path: " + res[0][3]); diff --git a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_complex_type.groovy similarity index 99% rename from regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy rename to regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_complex_type.groovy index 7fca91f12ff522..499137e5e608ef 100644 --- a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy +++ b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_complex_type.groovy @@ -102,7 +102,8 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") { INTO OUTFILE "${uri}" FORMAT AS ${format} PROPERTIES ( - "hadoop.username" = "${hdfsUserName}" + "hadoop.username" = "${hdfsUserName}", + "enable_int96_timestamps" = "true" ); """ logger.info("outfile success path: " + res[0][3]); diff --git a/regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy b/regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy index a29cc71107747d..b15f7b7034cc68 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy @@ -60,49 +60,49 @@ suite("test_hive_compress_type", "p0,external,hive,external_docker,external_dock order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal """ - order_qt_lzo_1 """ select * from parquet_lzo_compression + order_qt_lzo_1 """ select * from parquet_lzo_compression order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal - limit 20; + limit 20; """ - order_qt_lzo_2 """ select * from parquet_lzo_compression where col_int > 1000 + order_qt_lzo_2 """ select * from parquet_lzo_compression where col_int > 1000 order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal - limit 10; + limit 10; """ - order_qt_lzo_3 """ select * from parquet_lzo_compression where col_float > 5.1 and col_boolean = 1 + order_qt_lzo_3 """ select * from parquet_lzo_compression where col_float > 5.1 and col_boolean = 1 order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal - limit 10; + limit 10; """ - order_qt_lzo_4 """ select * from parquet_lzo_compression where col_float > 1000 and col_boolean != 1 + order_qt_lzo_4 """ select * from parquet_lzo_compression where col_float > 1000 and col_boolean != 1 order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal - limit 10; + limit 10; """ - order_qt_lzo_5 """ select * from parquet_lzo_compression where col_double < 17672101476 and col_char !='ft' + order_qt_lzo_5 """ select * from parquet_lzo_compression where col_double < 17672101476 and col_char !='ft' order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal - limit 10; + limit 10; """ order_qt_lzo_6 """ select * from parquet_lzo_compression where col_string='nuXBDInOfoaWz' order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal - limit 10; + limit 10; """ order_qt_lzo_7 """ select * from parquet_lzo_compression where col_decimal > 86208 and year(col_timestamp) = 2023 order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal - limit 10; + limit 10; """ order_qt_lzo_8 """ select * from parquet_lzo_compression where year(col_date)!=2023 and year(col_timestamp) = 2023 order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal - limit 10; + limit 10; """ } } diff --git a/regression-test/suites/external_table_p0/hive/test_hive_date_timezone.groovy b/regression-test/suites/external_table_p0/hive/test_hive_date_timezone.groovy index 26371b8f5c7ed4..ef9d8bf30e927e 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_date_timezone.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_date_timezone.groovy @@ -51,6 +51,34 @@ suite("test_hive_date_timezone", "p0,external") { sql """set time_zone = 'America/Mexico_City'""" qt_orc_date_west_tz """select date_col from orc_primitive_types_to_date order by id""" qt_parquet_date_west_tz """select date_col from parquet_primitive_types_to_date order by id""" + + // The parquet timestamp table exercises the optimized timestamp convert path. + // Querying the same data under multiple timezone spellings lets this suite cover + // both fixed-offset normalization and named-timezone lookup behavior. + sql """set time_zone = 'UTC'""" + def parquetTimestampUtc = sql """select timestamp_col from parquet_primitive_types_to_timestamp order by id""" + sql """set time_zone = 'Etc/UTC'""" + def parquetTimestampEtcUtc = sql """select timestamp_col from parquet_primitive_types_to_timestamp order by id""" + sql """set time_zone = '+08:00'""" + def parquetTimestampFixedOffset = sql """select timestamp_col from parquet_primitive_types_to_timestamp order by id""" + sql """set time_zone = '+8:00'""" + def parquetTimestampShortOffset = sql """select timestamp_col from parquet_primitive_types_to_timestamp order by id""" + sql """set time_zone = 'Etc/GMT-8'""" + def parquetTimestampEtcGmtMinus8 = sql """select timestamp_col from parquet_primitive_types_to_timestamp order by id""" + sql """set time_zone = '-06:00'""" + def parquetTimestampFixedMexicoOffset = sql """select timestamp_col from parquet_primitive_types_to_timestamp order by id""" + sql """set time_zone = 'America/Mexico_City'""" + def parquetTimestampMexicoCity = sql """select timestamp_col from parquet_primitive_types_to_timestamp order by id""" + + // Equivalent UTC spellings should stay on the same result set. + assertEquals(parquetTimestampUtc, parquetTimestampEtcUtc) + // These inputs are normalized to the same fixed offset and should match exactly. + assertEquals(parquetTimestampFixedOffset, parquetTimestampShortOffset) + // Etc/GMT-8 is a fixed-offset TZDB name. The sign is POSIX-style, so it means UTC+8. + assertEquals(parquetTimestampFixedOffset, parquetTimestampEtcGmtMinus8) + // America/Mexico_City must still read through the named-timezone path, not a constant + // -06:00 offset. This fixture contains a 2022 DST timestamp that makes the results differ. + assertEquals(parquetTimestampUtc.size(), parquetTimestampMexicoCity.size()) } finally { sql """set time_zone = default""" sql """switch internal""" diff --git a/regression-test/suites/external_table_p0/hive/test_parquet_lazy_mat_profile.groovy b/regression-test/suites/external_table_p0/hive/test_parquet_lazy_mat_profile.groovy index 3cfdccaa41b741..9e83828d33ef98 100644 --- a/regression-test/suites/external_table_p0/hive/test_parquet_lazy_mat_profile.groovy +++ b/regression-test/suites/external_table_p0/hive/test_parquet_lazy_mat_profile.groovy @@ -100,6 +100,18 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa return matcher.find() ? matcher.group(1).trim() : null } + def metricValueAsLong = { String value -> + if (value == null) { + return -1L + } + def formatted = value =~ /.*\((\d+)\).*/ + if (formatted.matches()) { + return formatted[0][1].toLong() + } + def plain = value.replaceAll("[^0-9-]", "") + return plain == "" ? -1L : plain.toLong() + } + // session vars sql "unset variable all;" sql "set profile_level=2;" @@ -230,6 +242,29 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa return extractProfileBlockMetrics(profileText, "ParquetReader") } + def q8 = { + sql """ set enable_file_scanner_v2 = true; """ + sql """ set enable_parquet_filter_by_min_max = false; """ + sql """ set enable_parquet_lazy_materialization = true; """ + def t1 = UUID.randomUUID().toString() + def sql_result = sql """ + select *, "${t1}" from alltypes_tiny_pages_plain where id > 2 and id < 10 order by id; + """ + def idColumnIndex = 7 + assertEquals(7, sql_result.size()) + assertEquals("3", sql_result[0][idColumnIndex].toString()) + assertEquals("9", sql_result[6][idColumnIndex].toString()) + + def profileText = getProfileWithToken(t1) + assertTrue(profileText.contains("ParquetReader"), "Profile does not contain ParquetReader") + def metrics = extractProfileBlockMetrics(profileText, "ParquetReader") + logger.info("metrics = ${metrics}") + assertTrue(metricValueAsLong(metrics["FilteredRowsByLazyRead"]) > 0) + assertTrue(metricValueAsLong(metrics["RawRowsRead"]) >= 7) + assertTrue(metricValueAsLong(metrics["RowsFilteredByConjunct"]) > 0) + assertTrue(metricValueAsLong(metrics["ReaderSelectRows"]) > 0) + } + def test_true_true = { @@ -325,6 +360,8 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa def test_true_false = { sql """ set enable_parquet_filter_by_min_max = true; """ sql """ set enable_parquet_lazy_materialization = false; """ + // in v2 lazy materialization is always enabled. + sql """ set enable_file_scanner_v2=false; """ def metrics = q1() logger.info("metrics = ${metrics}") @@ -595,6 +632,7 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa test_true_false(); test_false_false(); test_false_true(); + q8(); sql """drop catalog ${catalog_name};""" diff --git a/regression-test/suites/external_table_p0/iceberg/action/test_iceberg_execute_actions.groovy b/regression-test/suites/external_table_p0/iceberg/action/test_iceberg_execute_actions.groovy index bf8697aaefd950..283437a1291a55 100644 --- a/regression-test/suites/external_table_p0/iceberg/action/test_iceberg_execute_actions.groovy +++ b/regression-test/suites/external_table_p0/iceberg/action/test_iceberg_execute_actions.groovy @@ -30,7 +30,10 @@ suite("test_iceberg_optimize_actions_ddl", "p0,external,doris,external_docker,ex .optionalStart() .appendLiteral(' ') .optionalEnd() - .appendPattern("HH:mm:ss") + .appendPattern("HH:mm") + .optionalStart() + .appendPattern(":ss") + .optionalEnd() .optionalStart() .appendFraction(ChronoField.MILLI_OF_SECOND, 0, 3, true) .optionalEnd() diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy index b730761ac85950..8136df3cef400f 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_iceberg_deletion_vector", "p0,external,doris,external_docker,external_docker_doris") { +suite("test_iceberg_deletion_vector", "p0,external,nonConcurrent") { String enabled = context.config.otherConfigs.get("enableIcebergTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { logger.info("disable iceberg test.") @@ -42,30 +42,356 @@ suite("test_iceberg_deletion_vector", "p0,external,doris,external_docker,externa sql """switch ${catalog_name};""" sql """ use format_v3;""" + sql """ set file_split_size=1;""" + try { + def executeCommandWithStatus = { String cmd, int timeoutSeconds = 300, Boolean logFailure = true, + Boolean logCommand = true -> + StringBuilder stdout = new StringBuilder() + StringBuilder stderr = new StringBuilder() + try { + if (logCommand) { + logger.info("execute ${cmd}") + } + def proc = new ProcessBuilder("/bin/bash", "-c", cmd).start() + proc.consumeProcessOutput(stdout, stderr) + proc.waitForOrKill(timeoutSeconds * 1000) + int exitcode = proc.exitValue() + String output = stdout.toString() + String error = stderr.toString() + if (exitcode != 0 && logFailure) { + logger.info("exit code: ${exitcode}, stdout\n: ${output}\nstderr\n: ${error}") + } + return [exitCode: exitcode, stdout: output, stderr: error] + } catch (IOException e) { + assertTrue(false, "Execute failed: ${cmd}, err: ${e.message}") + } + } + + def executeCommand = { String cmd, Boolean mustSuc, int timeoutSeconds = 300 -> + def result = executeCommandWithStatus(cmd, timeoutSeconds) + if (mustSuc && result.exitCode != 0) { + assertTrue(false, + "Execute failed: ${cmd}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}") + } + return result.stdout + } + + String dockerCommand = context.config.otherConfigs.get("externalDockerCommand") ?: "docker" + def listDockerContainers = { + String containers = + executeCommand("${dockerCommand} ps --format '{{.ID}}\t{{.Names}}\t{{.Image}}'", true, 30) ?: + "" + return containers.readLines().collect { it.trim() }.findAll { !it.isEmpty() } + } + + def findRequiredDockerContainer = { String role, String configKey, String probeCommand -> + String configuredContainer = context.config.otherConfigs.get(configKey) + if (configuredContainer != null && !configuredContainer.isEmpty()) { + def probe = executeCommandWithStatus( + "${dockerCommand} exec ${configuredContainer} bash -lc '${probeCommand}'", + 30) + assertEquals(0, probe.exitCode, + "${role} container configured by ${configKey}=${configuredContainer} is not usable") + return configuredContainer + } + + def matchedContainers = [] + listDockerContainers().each { String containerLine -> + def fields = containerLine.split(/\t/, 3) + assertTrue(fields.length >= 2, "Unexpected docker ps output: ${containerLine}") + String containerId = fields[0].trim() + String containerName = fields[1].trim() + String containerImage = fields.length >= 3 ? fields[2].trim() : "" + def probe = executeCommandWithStatus( + "${dockerCommand} exec ${containerId} bash -lc '${probeCommand}'", + 30, + false, + false) + if (probe.exitCode == 0) { + matchedContainers.add( + [id: containerId, name: containerName, image: containerImage]) + } + } + + assertFalse(matchedContainers.isEmpty(), + "No usable ${role} container found. Set ${configKey} to the exact container name " + + "or start the required external environment.") + assertEquals(1, matchedContainers.size(), + "Multiple usable ${role} containers found: ${matchedContainers}. " + + "Set ${configKey} to the exact container name.") + logger.info("use ${role} container ${matchedContainers[0].name} " + + "(${matchedContainers[0].image})") + return matchedContainers[0].id + } + + String sparkContainerName = findRequiredDockerContainer( + "Spark Iceberg", "icebergSparkContainer", + "command -v spark-sql >/dev/null && test -f /mnt/SUCCESS && " + + "spark-sql -e \"SHOW NAMESPACES IN demo\" >/dev/null") + + def encodeBase64 = { String text -> + return text.getBytes("UTF-8").encodeBase64().toString() + } + + def runInSparkContainer = { String command, int timeout = 300 -> + executeCommand("${dockerCommand} exec ${sparkContainerName} bash -lc '${command}'", true, + timeout) + } + + def runSparkSql = { String sqlText, int timeout = 600 -> + String encodedSql = encodeBase64(sqlText) + runInSparkContainer( + "echo ${encodedSql} | base64 -d >/tmp/test_iceberg_deletion_vector.sql && " + + "spark-sql -f /tmp/test_iceberg_deletion_vector.sql", + timeout + ) + } + + String setupSql = """ +use format_v3; + +-- Delete-type matrix fixtures: reuse existing v2 equality/position-delete tables and create +-- v3 MOR tables that Spark writes with Iceberg deletion vectors. +drop table if exists dv_delete_matrix_equality_only; +CALL demo.system.register_table( + table => 'format_v3.dv_delete_matrix_equality_only', + metadata_file => 's3a://warehouse/wh/test_db/customer_flink_one/metadata/00002-c38103a7-237b-4350-aa98-05fe2ad14d16.metadata.json' +); + +drop table if exists dv_delete_matrix_position_only; +CALL demo.system.register_table( + table => 'format_v3.dv_delete_matrix_position_only', + metadata_file => 's3a://warehouse/wh/test_db/iceberg_position_parquet/metadata/00005-6ff36319-7a9a-4357-a9fc-965d5f280c9b.metadata.json' +); + +drop table if exists dv_delete_matrix_position_and_dv; +CALL demo.system.register_table( + table => 'format_v3.dv_delete_matrix_position_and_dv', + metadata_file => 's3a://warehouse/wh/test_db/iceberg_position_parquet/metadata/00005-6ff36319-7a9a-4357-a9fc-965d5f280c9b.metadata.json' +); +alter table dv_delete_matrix_position_and_dv set tblproperties ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' +); +delete from dv_delete_matrix_position_and_dv where id = 2; + +drop table if exists dv_delete_matrix_equality_and_dv; +create table dv_delete_matrix_equality_and_dv ( + id int, + batch int, + data string +) +using iceberg +tblproperties ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' +); +insert into dv_delete_matrix_equality_and_dv values + (1, 1, 'a'), (2, 1, 'b'), (3, 1, 'c'), (4, 1, 'd'), + (5, 1, 'e'), (6, 1, 'f'), (7, 1, 'g'), (8, 1, 'h'), + (9, 2, 'i'), (10, 2, 'j'), (11, 2, 'k'), (12, 2, 'l'); +delete from dv_delete_matrix_equality_and_dv where id % 2 = 1; + +-- Split/cache fixture: one sufficiently large data file plus one DV. Doris reads it with a tiny +-- file_split_size so the same data file is split across scanners and the shared scan-node cache +-- must prevent the DV from being parsed once per split. +drop table if exists dv_split_cache_single_file; +create table dv_split_cache_single_file ( + id bigint, + grp int, + payload string +) +using iceberg +tblproperties ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read', + 'write.parquet.row-group-size-bytes' = '1024' +); +set spark.sql.shuffle.partitions = 1; +insert into dv_split_cache_single_file +select /*+ COALESCE(1) */ + id, + cast(id % 8 as int) as grp, + repeat(concat('payload_', cast(id as string)), 20) as payload + from range(0, 4096); +delete from dv_split_cache_single_file where id % 2 = 1; +reset spark.sql.shuffle.partitions; +""" + runSparkSql(setupSql) + + String javaSource = ''' +import java.util.HashMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.parquet.GenericParquetWriter; +import org.apache.iceberg.deletes.EqualityDeleteWriter; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.parquet.Parquet; + +public class AppendEqualityDelete { + public static void main(String[] args) throws Exception { + Catalog catalog = IcebergRestCatalog.load(); + Table table = catalog.loadTable(TableIdentifier.of(args[0], args[1])); + Schema eqSchema = table.schema().select(args[2]); + int fieldId = table.schema().findField(args[2]).fieldId(); + String location = table.location() + "/data/equality-delete-" + args[2] + "-" + + args[3] + "-" + System.currentTimeMillis() + ".parquet"; + OutputFile outputFile = table.io().newOutputFile(location); + EqualityDeleteWriter writer = Parquet.writeDeletes(outputFile) + .forTable(table) + .rowSchema(eqSchema) + .withSpec(table.spec()) + .createWriterFunc(GenericParquetWriter::create) + .equalityFieldIds(fieldId) + .overwrite() + .buildEqualityWriter(); + GenericRecord record = GenericRecord.create(eqSchema); + record.setField(args[2], Integer.valueOf(args[3])); + writer.write(record); + writer.close(); + DeleteFile deleteFile = writer.toDeleteFile(); + table.newRowDelta().addDeletes(deleteFile).commit(); + } +} + +class ReadIcebergRows { + public static void main(String[] args) throws Exception { + Catalog catalog = IcebergRestCatalog.load(); + Table table = catalog.loadTable(TableIdentifier.of(args[0], args[1])); + String[] columns = Arrays.copyOfRange(args, 2, args.length); + List rows = new ArrayList<>(); + try (CloseableIterable records = IcebergGenerics.read(table).select(columns).build()) { + for (Record record : records) { + List values = new ArrayList<>(); + for (String column : columns) { + values.add(String.valueOf(record.getField(column))); + } + rows.add(String.join("\\t", values)); + } + } + Collections.sort(rows, Comparator.comparingInt(row -> Integer.parseInt(row.split("\\t")[0]))); + for (String row : rows) { + System.out.println(row); + } + } +} +class IcebergRestCatalog { + static Catalog load() { + Map props = new HashMap<>(); + props.put("type", "rest"); + props.put("uri", "http://rest:8181"); + props.put("warehouse", "s3://warehouse/wh/"); + props.put("io-impl", "org.apache.iceberg.aws.s3.S3FileIO"); + props.put("s3.endpoint", "http://minio:9000"); + props.put("s3.path-style-access", "true"); + props.put("s3.region", "us-east-1"); + return CatalogUtil.buildIcebergCatalog("demo", props, null); + } +} +''' + String encodedJavaSource = encodeBase64(javaSource) + runInSparkContainer( + "echo ${encodedJavaSource} | base64 -d >/tmp/AppendEqualityDelete.java && " + + "javac -cp \"/opt/spark/jars/*\" /tmp/AppendEqualityDelete.java && " + + "java -cp \"/tmp:/opt/spark/jars/*\" AppendEqualityDelete " + + "format_v3 dv_delete_matrix_equality_and_dv id 10", + 300 + ) + + + // Baseline DV correctness across Parquet, ORC, no-delete, and v2-to-v3 upgrade fixtures. qt_q1 """ SELECT * FROM dv_test ORDER BY id; """ qt_q2 """ SELECT * FROM dv_test_v2 ORDER BY id; """ + qt_q3 """ SELECT * FROM dv_test_orc ORDER BY id; """ + qt_no_delete """ SELECT * FROM dv_test_no_delete ORDER BY id; """ + + def enableFileScannerV2Rows = sql """SHOW VARIABLES LIKE 'enable_file_scanner_v2'""" + assertTrue(enableFileScannerV2Rows.size() > 0, + "Session variable enable_file_scanner_v2 is not found") + String originalEnableFileScannerV2 = enableFileScannerV2Rows[0][1].toString() + try { + sql """set enable_file_scanner_v2=false""" + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().enableDebugPointForAllBEs( + "IcebergDeleteFileReader.read_deletion_vector.io_error") + test { + sql """ SELECT count(*) FROM dv_test; """ + exception "injected Iceberg deletion vector read failure" + } + + sql """set enable_file_scanner_v2=true""" + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().enableDebugPointForAllBEs( + "TableReader.parse_deletion_vector.io_error") + test { + sql """ SELECT count(*) FROM dv_test; """ + exception "injected format v2 deletion vector read failure" + } + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + sql """set enable_file_scanner_v2=${originalEnableFileScannerV2}""" + } + + // Delete-type matrix checks cover equality-only, position-only, DV-only, DV+position, + // and DV+equality combinations. + qt_equality_only """ SELECT * FROM dv_delete_matrix_equality_only ORDER BY id; """ + qt_position_only_count """ SELECT count(*) FROM dv_delete_matrix_position_only; """ + qt_position_and_dv_count """ SELECT count(*) FROM dv_delete_matrix_position_and_dv; """ + qt_equality_and_dv """ SELECT * FROM dv_delete_matrix_equality_and_dv ORDER BY id; """ + qt_split_cache_single_file_count """ + SELECT count(*), sum(id) + FROM dv_split_cache_single_file; + """ qt_q4 """ SELECT * FROM dv_test where id = 2 ORDER BY id ; """ qt_q5 """ SELECT * FROM dv_test_v2 where id = 2 ORDER BY id ; """ + qt_q5_orc """ SELECT * FROM dv_test_orc where id = 2 ORDER BY id; """ qt_q6 """ SELECT * FROM dv_test where id %2 =1 ORDER BY id; """ qt_q7 """ SELECT * FROM dv_test_v2 where id %2 =1 ORDER BY id; """ + qt_q7_orc """ SELECT * FROM dv_test_orc where id %2 =1 ORDER BY id; """ qt_q8 """ SELECT * FROM dv_test where data < 'f' ORDER BY id; """ qt_q9 """ SELECT * FROM dv_test_v2 where data < 'f' ORDER BY id; """ + qt_q9_orc """ SELECT * FROM dv_test_orc where data < 'f' ORDER BY id; """ qt_q10 """ SELECT count(*) FROM dv_test ; """ qt_q11 """ SELECT count(*) FROM dv_test_v2 ; """ + qt_q11_orc """ SELECT count(*) FROM dv_test_orc; """ + qt_no_delete_count """ SELECT count(*) FROM dv_test_no_delete; """ qt_q12 """ SELECT count(id) FROM dv_test ; """ qt_q13 """ SELECT count(id) FROM dv_test_v2 ; """ + qt_q13_orc """ SELECT count(id) FROM dv_test_orc; """ qt_q14 """ SELECT count(batch) FROM dv_test ; """ qt_q15 """ SELECT count(batch) FROM dv_test_v2 ; """ + qt_q15_orc """ SELECT count(batch) FROM dv_test_orc; """ qt_q16 """ SELECT count(*) FROM dv_test_1w ; """ @@ -74,9 +400,236 @@ suite("test_iceberg_deletion_vector", "p0,external,doris,external_docker,externa qt_q19 """ SELECT count(value) FROM dv_test_1w where id%2 = 1; """ qt_q20 """ SELECT count(id) FROM dv_test_1w where id%3 = 1; """ qt_q21 """ SELECT count(ts) FROM dv_test_1w where id%3 != 1; """ + qt_q22 """ SELECT * FROM dv_test ORDER BY data DESC LIMIT 3; """ + qt_q23 """ SELECT * FROM dv_test_v2 ORDER BY data DESC LIMIT 3; """ + qt_q24 """ SELECT batch, count(*), sum(id) FROM dv_test GROUP BY batch ORDER BY batch; """ + qt_q25 """ SELECT batch, count(*), sum(id) FROM dv_test_v2 GROUP BY batch ORDER BY batch; """ + qt_q26 """ SELECT batch, count(*), sum(id) FROM dv_test_orc GROUP BY batch ORDER BY batch; """ + qt_q27 """ SELECT id, data FROM dv_test WHERE id BETWEEN 2 AND 10 ORDER BY id LIMIT 2; """ + qt_q28 """ SELECT id, data FROM dv_test_v2 WHERE id BETWEEN 2 AND 10 ORDER BY id LIMIT 2; """ + qt_q29 """ SELECT id, data FROM dv_test_orc WHERE id BETWEEN 2 AND 10 ORDER BY id LIMIT 2; """ + qt_q30 """ SELECT count(*) FROM (SELECT * FROM dv_test_1w WHERE id % 7 = 0 ORDER BY id LIMIT 100) t; """ + qt_q31 """ SELECT count(*) FROM (SELECT * FROM dv_test_1w WHERE id % 11 = 0 ORDER BY id LIMIT 100) t; """ + // Metadata matrix verifies the physical delete-file content types that back the logical + // scenario checks above. + qt_delete_type_matrix """ + SELECT 'dv_equality' scenario, content, file_format, count(*) files, sum(record_count) records + FROM `dv_delete_matrix_equality_and_dv\$files` + GROUP BY content, file_format + UNION ALL + SELECT 'dv_only' scenario, content, file_format, count(*) files, sum(record_count) records + FROM `dv_test\$files` + GROUP BY content, file_format + UNION ALL + SELECT 'dv_position' scenario, content, file_format, count(*) files, sum(record_count) records + FROM `dv_delete_matrix_position_and_dv\$files` + GROUP BY content, file_format + UNION ALL + SELECT 'equality_only' scenario, content, file_format, count(*) files, sum(record_count) records + FROM `dv_delete_matrix_equality_only\$files` + GROUP BY content, file_format + UNION ALL + SELECT 'no_delete' scenario, content, file_format, count(*) files, sum(record_count) records + FROM `dv_test_no_delete\$files` + GROUP BY content, file_format + UNION ALL + SELECT 'position_only' scenario, content, file_format, count(*) files, sum(record_count) records + FROM `dv_delete_matrix_position_only\$files` + GROUP BY content, file_format + ORDER BY scenario, content, file_format; + """ + // Iceberg v3 semantics: when a data file has a DV, old position deletes for that file must be + // ignored, while equality deletes still filter matching rows. + qt_position_delete_ignored_with_dv """ + SELECT id, name + FROM dv_delete_matrix_position_and_dv + WHERE id IN (1, 2, 5, 10, 15) + ORDER BY id, name + LIMIT 20; + """ + qt_equality_delete_active_with_dv """ + SELECT id, batch, data + FROM dv_delete_matrix_equality_and_dv + ORDER BY id; + """ + def normalizeExternalRows = { String output -> + return output.readLines() + .collect { it.trim() } + .findAll { it ==~ /^-?[0-9].*/ && it.contains("\t") } + .join("\n") + } + + String expectedRows = ["2\t1\tb", "4\t1\td", "6\t1\tf", "8\t1\th", "12\t2\tl"].join("\n") + String sparkRows = normalizeExternalRows(runSparkSql( + "use format_v3; select id, batch, data from dv_delete_matrix_equality_and_dv order by id;", + 300 + )) + assertEquals(expectedRows, sparkRows) + String javaRows = normalizeExternalRows(runInSparkContainer( + "java -cp \"/tmp:/opt/spark/jars/*\" ReadIcebergRows " + + "format_v3 dv_delete_matrix_equality_and_dv id batch data", + 300 + )) + assertEquals(expectedRows, javaRows) + + String trinoContainerName = findRequiredDockerContainer( + "Trino", "icebergTrinoContainer", + "test -d /etc/trino/catalog && command -v trino >/dev/null") + String trinoExternalEnvIp = externalEnvIp + if (trinoExternalEnvIp == "127.0.0.1" || trinoExternalEnvIp.equalsIgnoreCase("localhost")) { + trinoExternalEnvIp = executeCommand("hostname -I | cut -d' ' -f1", true, 30)?.trim() + } + String trinoCatalogProps = """ +connector.name=iceberg +iceberg.catalog.type=rest +iceberg.rest-catalog.uri=http://${trinoExternalEnvIp}:${rest_port} +fs.native-s3.enabled=true +s3.endpoint=http://${trinoExternalEnvIp}:${minio_port} +s3.aws-access-key=admin +s3.aws-secret-key=password +s3.region=us-east-1 +s3.path-style-access=true +""" + String encodedTrinoCatalogProps = encodeBase64(trinoCatalogProps) + executeCommand( + "${dockerCommand} exec ${trinoContainerName} bash -lc " + + "'echo ${encodedTrinoCatalogProps} | " + + "base64 -d >/etc/trino/catalog/iceberg.properties'", + true, + 30 + ) + executeCommand("${dockerCommand} restart ${trinoContainerName}", true, 60) + String trinoRows = "" + for (int i = 0; i < 12; i++) { + Thread.sleep(5000) + trinoRows = normalizeExternalRows(executeCommand( + "${dockerCommand} exec ${trinoContainerName} trino --output-format TSV " + + "--catalog iceberg --schema format_v3 --execute " + + "\"SELECT id, batch, data " + + "FROM dv_delete_matrix_equality_and_dv ORDER BY id\"", + false, + 120 + )) + if (!trinoRows.isEmpty()) { + break + } + } + assertEquals(expectedRows, trinoRows) + + def profileCounterValues = { String profileText, String counterName -> + def values = [] + def matcher = profileText =~ ("(?m)^\\s*(?:-\\s*)?" + + java.util.regex.Pattern.quote(counterName) + ":\\s+([^\\n]+)") + while (matcher.find()) { + String valueText = matcher.group(1).toString() + def exact = valueText =~ /\(([0-9,]+)\)/ + String rawValue + if (exact.find()) { + rawValue = exact.group(1) + } else { + def number = valueText =~ /([0-9,]+)/ + rawValue = number.find() ? number.group(1) : null + } + if (rawValue != null) { + values.add(Long.parseLong(rawValue.replace(",", ""))) + } + } + return values + } + + def profileInfoValueCount = { String profileText, String infoName -> + def matcher = profileText =~ (java.util.regex.Pattern.quote(infoName) + ":\\s*\\[([^\\]]*)\\]") + if (!matcher.find()) { + return 0 + } + return matcher.group(1).split(",").collect { it.trim() }.findAll { !it.isEmpty() }.size() + } + + def aliveBackendCount = { + def backends = sql_return_maparray("show backends") + int aliveCount = backends.count { be -> + be.Alive == null || be.Alive.toString().equalsIgnoreCase("true") + } + return aliveCount > 0 ? aliveCount : backends.size() + } + + // Split/cache scenario for 3.4: this query scans one data file with file_split_size=1. The + // result is captured by qt_split_cache_single_file_count; the profile assertion checks the DV + // rows were materialized once per backend-local scan cache, not once per split. + String splitCacheProfileTag = "iceberg_dv_split_cache_" + UUID.randomUUID().toString() + profile(splitCacheProfileTag) { + run { + sql """ + /* ${splitCacheProfileTag} */ + SELECT /*+ SET_VAR(parallel_pipeline_task_num=1) */ count(*), sum(id) + FROM dv_split_cache_single_file; + """ + // The detailed scanner counters are populated asynchronously after the query returns. + Thread.sleep(5000) + } + check { profileString, exception -> + if (exception != null) { + throw exception + } + String mergedProfile = profileString + if (profileString.contains("MergedProfile:")) { + mergedProfile = profileString.substring(profileString.indexOf("MergedProfile:")) + int mergedProfileEnd = mergedProfile.length() + ["DetailProfile(", "Execution Profile:", "Appendix:"].each { String sectionName -> + int sectionIndex = mergedProfile.indexOf(sectionName) + if (sectionIndex > 0) { + mergedProfileEnd = Math.min(mergedProfileEnd, sectionIndex) + } + } + mergedProfile = mergedProfile.substring(0, mergedProfileEnd) + } + def numDeleteRowsValues = profileCounterValues(mergedProfile, "NumDeleteRows") + numDeleteRowsValues = numDeleteRowsValues.findAll { it > 0L } + long numDeleteRows = numDeleteRowsValues.sum(0L) + long expectedDeleteRowsPerDv = 2048L + int maxBackendLocalLoads = aliveBackendCount() + int scannerCount = profileInfoValueCount(profileString, "PerScannerRowsRead") + assertFalse(numDeleteRowsValues.isEmpty(), + "Expected NumDeleteRows counter for split DV scan, profile: ${profileString}") + assertEquals(0L, numDeleteRows % expectedDeleteRowsPerDv, + "Expected DV rows to be counted in backend-local DV loads, " + + "NumDeleteRows counters: ${numDeleteRowsValues}, profile: ${profileString}") + assertTrue(numDeleteRows >= expectedDeleteRowsPerDv && + numDeleteRows <= expectedDeleteRowsPerDv * maxBackendLocalLoads, + "Expected DV rows to be materialized at most once per backend-local cache, " + + "alive backends: ${maxBackendLocalLoads}, NumDeleteRows counters: " + + "${numDeleteRowsValues}, profile: ${profileString}") + assertTrue(scannerCount > 1, + "Expected multiple scanner entries for split DV scan, profile: ${profileString}") + } + } + + // Cross-reader comparison protects against Doris accepting a DV/equality-delete result that + // diverges from Spark, Trino, or Iceberg's Java reader. + explain { + sql("SELECT count(*) FROM dv_test;") + verbose(true) + contains "deleteFileNum" + } + + explain { + sql("SELECT count(*) FROM dv_test_orc;") + verbose(true) + contains "deleteFileNum" + } + + explain { + sql("SELECT count(*) FROM dv_test_v2;") + verbose(true) + contains "deleteFileNum" + } + + } finally { + sql """ unset variable file_split_size;""" + } } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy index b4394755c9aa97..f24b075443ab4d 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy @@ -42,9 +42,17 @@ suite("test_iceberg_equality_delete_with_schema_change", "p0,external,doris,exte sql """switch ${catalog_name};""" sql """ use multi_catalog;""" - + def enableFileScannerV2Rows = sql """show variables like 'enable_file_scanner_v2'""" + assertTrue(enableFileScannerV2Rows.size() > 0, + "Session variable enable_file_scanner_v2 is not found") + String originalEnableFileScannerV2 = enableFileScannerV2Rows[0][1].toString() + // The existing output was generated by the legacy scanner and is kept as the differential + // baseline. Run the complete schema-evolution matrix through V2 so missing/renamed equality + // delete keys must produce byte-for-byte identical query results. + sql """set enable_file_scanner_v2=true""" - for (String format: ["par","orc"]) { + try { + for (String format: ["par","orc"]) { // Basic full table scan order_qt_q1 """ SELECT * FROM equality_delete_${format}_1 ORDER BY new_new_id; """ @@ -190,5 +198,9 @@ suite("test_iceberg_equality_delete_with_schema_change", "p0,external,doris,exte order_qt_q70 """ SELECT new_new_id, new_name, data FROM equality_delete_${format}_1 WHERE new_new_id = 1 UNION ALL SELECT new_new_id, new_name, data FROM equality_delete_${format}_2 WHERE new_new_id = 1; """ + + } + } finally { + sql """set enable_file_scanner_v2=${originalEnableFileScannerV2}""" } } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy index 4815de33584b5a..a1d074d943eb89 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy @@ -92,7 +92,9 @@ suite("test_iceberg_optimize_count", "p0,external,doris,external_docker,external } // batch mode + sql """set enable_external_table_batch_mode=true""" sql """set num_files_in_batch_mode=1""" + sql """set enable_file_scanner_v2=false""" explain { sql("""select * from sample_cow_orc""") contains "approximate" @@ -132,7 +134,9 @@ suite("test_iceberg_optimize_count", "p0,external,doris,external_docker,external } // don't use push down count + sql """set enable_external_table_batch_mode=false""" sql """ set enable_count_push_down_for_external_table=false; """ + sql """set enable_file_scanner_v2=true""" qt_q05 """${sqlstr1}""" qt_q06 """${sqlstr2}""" @@ -178,8 +182,8 @@ suite("test_iceberg_optimize_count", "p0,external,doris,external_docker,external } finally { sql """ set enable_count_push_down_for_external_table=true; """ + sql """set enable_external_table_batch_mode=false""" sql """set num_partitions_in_batch_mode=1024""" // sql """drop catalog if exists ${catalog_name}""" } } - diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.groovy index 18f9aa56957d4f..e6142771df4a06 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.groovy @@ -95,6 +95,28 @@ suite("test_iceberg_struct_schema_evolution", "p0,external,doris,external_docker // Test 8: DISTINCT query on struct fields qt_struct_distinct """SELECT DISTINCT element_at(a_struct, 'renamed'), element_at(a_struct, 'added'), element_at(a_struct, 'keep') FROM ${table_name} ORDER BY 1, 2, 3""" + // Reproduce Spark Iceberg struct child type evolution: old files keep col.a as INT while + // current Iceberg schema exposes it as BIGINT. Reading col.a must cast the materialized struct + // child without assuming the declared nullable file type matches the actual column nullability. + def type_evolution_table_name = "test_struct_child_type_evolution" + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.test_db.${type_evolution_table_name}; + CREATE TABLE demo.test_db.${type_evolution_table_name} ( + id INT, + col STRUCT + ) USING iceberg + TBLPROPERTIES ('write.format.default' = 'parquet'); + INSERT INTO demo.test_db.${type_evolution_table_name} + SELECT 1, named_struct('a', 10, 'b', 20, 'c', 30); + ALTER TABLE demo.test_db.${type_evolution_table_name} ALTER COLUMN col.a TYPE BIGINT; + """ + sql """REFRESH CATALOG ${catalog_name}""" + sql """ + SELECT /*+ SET_VAR(enable_prune_nested_column=true) */ col.a, col.b, col.c + FROM ${type_evolution_table_name} + ORDER BY id + """ + // ============================================================ // Test with ORC format (for completeness) // ============================================================ diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy index b5ca34e5a18a48..91d38cf729438e 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy @@ -307,7 +307,7 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ test { sql """select * from dup_columns_table;""" - exception "Duplicate column name found: id" + exception "Duplicate column name found: ID" } sql """ set force_jni_scanner=false; """ @@ -332,4 +332,3 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ } } - diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy index fade251ed56f4a..63a2fe13bc3345 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy @@ -38,6 +38,9 @@ suite("test_paimon_deletion_vector", "p0,external,doris,external_docker,external def test_cases = { String force -> sql """ set force_jni_scanner=${force} """ + // Force tiny splits so both native and JNI paths read Paimon DV tables through multiple + // scanner ranges while still returning the same filtered rows. + sql """ set file_split_size=1 """ qt_1 """select count(*) from deletion_vector_orc;""" qt_2 """select count(*) from deletion_vector_parquet;""" qt_3 """select count(*) from deletion_vector_orc where id > 2;""" @@ -47,10 +50,16 @@ suite("test_paimon_deletion_vector", "p0,external,doris,external_docker,external qt_7 """select * from deletion_vector_table_1_0 order by id;""" qt_8 """select count(*) from deletion_vector_table_1_0;""" qt_9 """select count(*) from deletion_vector_table_1_0 where id > 2;""" + qt_10 """select * from deletion_vector_orc where id > 2 order by id limit 1;""" + qt_11 """select * from deletion_vector_parquet where id > 2 order by id limit 1;""" + qt_12 """select count(*) from (select * from deletion_vector_orc where id > 2 union all select * from deletion_vector_orc where id > 2) t;""" + qt_13 """select count(*) from (select * from deletion_vector_parquet where id > 2 union all select * from deletion_vector_parquet where id > 2) t;""" } def test_table_count_push_down = { String force -> sql """ set force_jni_scanner=${force} """ + // DV tables cannot use table-level count pushdown because deleted rows must be applied; + // the no-DV v1.0 table keeps the positive pushdown baseline. explain { sql("select count(*) from deletion_vector_orc;") contains "pushdown agg=COUNT (-1)" @@ -68,6 +77,8 @@ suite("test_paimon_deletion_vector", "p0,external,doris,external_docker,external def test_not_table_count_push_down = { String force -> sql """ set enable_count_push_down_for_external_table=false; """ sql """ set force_jni_scanner=${force} """ + // With count pushdown disabled, all DV and non-DV tables must report NONE regardless + // of the selected scanner path. explain { sql("select count(*) from deletion_vector_orc;") contains "pushdown agg=NONE" @@ -90,7 +101,8 @@ suite("test_paimon_deletion_vector", "p0,external,doris,external_docker,external test_not_table_count_push_down("true") } finally { sql """ set enable_count_push_down_for_external_table=true; """ + sql """unset variable file_split_size""" sql """set force_jni_scanner=false""" } -} \ No newline at end of file +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy index 3974051f9f6c94..b5f233aac67515 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy @@ -267,7 +267,7 @@ suite("test_paimon_jdbc_catalog", "p0,external") { assertSystemTableReadable( "paimon_jdbc_row_tracking_tbl\$row_tracking", - ["_row_id", "_sequence_number"], + ["_ROW_ID", "_SEQUENCE_NUMBER"], 1 ) } finally { diff --git a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_agg_table_select.groovy b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_agg_table_select.groovy index a494c539cf4cc3..c70d96ef0c6da2 100644 --- a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_agg_table_select.groovy +++ b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_agg_table_select.groovy @@ -277,7 +277,7 @@ suite("test_remote_doris_agg_table_select", "p0,external,doris,external_docker,e test { sql "select typ_id, typ_name, hll_cardinality(pv) from `${catalog_arrow_name}`.`${db_name}`.test_remote_doris_agg_table_select_hll order by typ_id,typ_name" // check exception message contains - exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type HLL. cur path: /dummyPath" + exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type HLL" } // BITMAP @@ -299,7 +299,7 @@ suite("test_remote_doris_agg_table_select", "p0,external,doris,external_docker,e ) final; """ // check exception message contains - exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type BITMAP. cur path: /dummyPath" + exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type BITMAP" } sql """ DROP DATABASE IF EXISTS `${db_name}` """ diff --git a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy index d0fe187b27f3cc..46dbcdd5519747 100644 --- a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy +++ b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy @@ -208,7 +208,7 @@ suite("test_remote_doris_unique_table_select", "p0,external,doris,external_docke test { sql "select typ_id, typ_name, hll_cardinality(pv) from `${catalog_arrow_name}`.`${db_name}`.test_remote_doris_unique_table_select_hll order by typ_id,typ_name" // check exception message contains - exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type HLL. cur path: /dummyPath" + exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type HLL" } // BITMAP @@ -230,7 +230,7 @@ suite("test_remote_doris_unique_table_select", "p0,external,doris,external_docke ) final; """ // check exception message contains - exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type BITMAP. cur path: /dummyPath" + exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type BITMAP" } sql """ DROP DATABASE IF EXISTS `${db_name}` """ diff --git a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy index 406f6da7dbd34f..6aa297aa98bfdb 100644 --- a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy +++ b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy @@ -112,7 +112,7 @@ suite("test_remote_doris_variant_select", "p0,external,doris,external_docker,ext select * from `${catalog_arrow_name}`.`${db_name}`.`test_remote_doris_variant_select_t` order by id """ // check exception message contains - exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type variant. cur path: /dummyPath" + exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type variant" } qt_sql """ diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy index 4473fa8bf17cb4..2808e7b4557d14 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy @@ -167,13 +167,10 @@ suite("test_hdfs_parquet_group0","external,hive,tvf,external_docker") { uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group0/nation.dict-malformed.parquet" - test { - sql """ select * from HDFS( + order_qt_test_20 """ select nation_key, name, region_key, rtrim(comment_col) from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet"); """ - exception "[IO_ERROR]Out-of-bounds Access" - } uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group0/lz4_raw_compressed_larger.parquet" @@ -329,10 +326,9 @@ suite("test_hdfs_parquet_group0","external,hive,tvf,external_docker") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet"); """ - exception "Out-of-bounds access in parquet data decoder" + exception "Unexpected end of stream" } - uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group0/lz4_raw_compressed.parquet" order_qt_test_43 """ select * from HDFS( "uri" = "${uri}", diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy index 998e5b44c9cdb8..4d00533fdc3b52 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy @@ -244,10 +244,13 @@ suite("test_hdfs_parquet_group2","external,hive,tvf,external_docker") { uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group2/group-field-with-enum-as-logical-annotation.parquet" - order_qt_test_31 """ select * from HDFS( + test { + sql """ select * from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ + exception "Logical type Enum cannot be applied to group node" + } uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group2/timemillis-in-i64.parquet" diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group4.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group4.groovy index 3ee952737e14a4..5248c75df952ea 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group4.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group4.groovy @@ -865,7 +865,7 @@ suite("test_hdfs_parquet_group4","external,hive,tvf,external_docker") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ - exception "The column type of 'member0' is not supported" + exception "Parquet TIME with isAdjustedToUTC=true is not supported" } @@ -2045,7 +2045,7 @@ suite("test_hdfs_parquet_group4","external,hive,tvf,external_docker") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ - exception "The column type of 'COLUMN1' is not supported" + exception "Parquet TIME with isAdjustedToUTC=true is not supported" } diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group5.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group5.groovy index 083398a6b5641b..902be6fb4b3de1 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group5.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group5.groovy @@ -123,7 +123,7 @@ suite("test_hdfs_parquet_group5","external,hive,tvf,external_docker") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ - exception "The column type of 'timestamp' is not supported" + exception "Parquet TIME with isAdjustedToUTC=true is not supported" } @@ -272,7 +272,7 @@ suite("test_hdfs_parquet_group5","external,hive,tvf,external_docker") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ - exception "The column type of 'timestamp' is not supported" + exception "Parquet TIME with isAdjustedToUTC=true is not supported" } diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy index 522ad648e5df07..dfd6f961872966 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy @@ -427,7 +427,7 @@ suite("test_hdfs_parquet_group6","external,hive,tvf,external_docker") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ - exception "The column type of 'time_millis' is not supported" + exception "Parquet TIME with isAdjustedToUTC=true is not supported" } @@ -649,13 +649,10 @@ suite("test_hdfs_parquet_group6","external,hive,tvf,external_docker") { "format" = "parquet") limit 10; """ uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/test_parquet_time_type.parquet" - test { - sql """ select * from HDFS( + order_qt_test_87 """ select * from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ - exception "The column type of 'c2' is not supported" - } uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/json.parquet" @@ -673,13 +670,10 @@ suite("test_hdfs_parquet_group6","external,hive,tvf,external_docker") { uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/ARROW-17100.parquet" - test { - sql """ select * from HDFS( + order_qt_test_90 """ select * from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet"); """ - exception "Can't read enough bytes in plain decode" - } uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/parquet_cpp_example.parquet" @@ -744,7 +738,7 @@ suite("test_hdfs_parquet_group6","external,hive,tvf,external_docker") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ - exception "The column type of 'time_micros' is not supported" + exception "Parquet TIME with isAdjustedToUTC=true is not supported" } diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.groovy new file mode 100644 index 00000000000000..331faa8ddb3ff6 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_local_tvf_csv_enclose_consistency", "p0,external") { + def backends = sql "show backends" + assertTrue(backends.size() > 0) + def beId = backends[0][0] + def dataPath = context.config.dataPath + "/external_table_p0/tvf" + def filePath = "/" + + def files = [ + "csv_enclose_state.csv", + "csv_matching_escape_enclose.csv", + "csv_quoted_null.csv" + ] + for (def backend : backends) { + for (def file : files) { + scpFiles("root", backend[1], "${dataPath}/${file}", filePath, false) + } + } + + sql "set enable_file_scanner_v2 = true" + + order_qt_enclose_state """ + select id, score, extra + from local( + "file_path" = "${filePath}/csv_enclose_state.csv", + "backend_id" = "${beId}", + "format" = "csv_with_names", + "column_separator" = ",", + "enclose" = "\\\"", + "escape" = "\\\\") + order by id + """ + + order_qt_matching_escape_enclose """ + select id, name, score + from local( + "file_path" = "${filePath}/csv_matching_escape_enclose.csv", + "backend_id" = "${beId}", + "format" = "csv_with_names", + "column_separator" = ",", + "enclose" = "\\\"", + "escape" = "\\\"") + order by id + """ + + order_qt_quoted_null """ + select id, hex(name), name is null + from local( + "file_path" = "${filePath}/csv_quoted_null.csv", + "backend_id" = "${beId}", + "format" = "csv_with_names", + "column_separator" = ",", + "trim_double_quotes" = "true", + "null_format" = "\\\\N") + order by id + """ +}