[branch-4.1] Backport FileScannerV2 and follow-up fixes

.github/CODEOWNERS

@@ -14,4 +14,148 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-* @yiguolei
+
+# for more syntax help and usage example
+# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax
+
+be/src/agent/be_exec_version_manager.cpp @BiteTheDDDDt @zclllyybb
+fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman
+**/pom.xml @CalvinKirs @morningman
+fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @dataroaring @morningman @yiguolei @xiaokang
+fe/fe-core/src/main/java/org/apache/doris/fs @CalvinKirs
+fe/fe-core/src/main/java/org/apache/doris/fsv2 @CalvinKirs
+be/src/vec/functions @zclllyybb
+be/**/CMakeLists.txt @zclllyybb @BiteTheDDDDt
+be/src/olap/rowset/segment_v2/variant @eldenmoon @csun5285
+be/src/storage/index/inverted/ @airborne12 @eldenmoon @csun5285
+be/src/exprs/function/function_search.h @airborne12 @eldenmoon @csun5285
+be/src/exprs/function/function_search.cpp @airborne12 @eldenmoon @csun5285
+be/src/exprs/function/match.cpp @airborne12 @eldenmoon @csun5285
+be/src/exprs/function/match.h @airborne12 @eldenmoon @csun5285
+be/src/exprs/vsearch.h @airborne12 @eldenmoon @csun5285
+be/src/exprs/vsearch.cpp @airborne12 @eldenmoon @csun5285
+.claude/ @zclllyybb
+AGENTS.md @zclllyybb
+
+be/src/cloud                                                 @liaoxin01 @luwei16 @gavinchou
+be/src/io                                                    @liaoxin01 @gavinchou @morningman @Gabriel39
+be/src/io/cache                                              @liaoxin01 @gavinchou @morningman @Gabriel39
+be/src/io/fs                                                 @liaoxin01 @gavinchou @morningman @Gabriel39
+be/src/io/tools                                              @liaoxin01 @gavinchou
+be/src/format_v2                                             @Gabriel39 @yiguolei
+be/src/load                                                  @liaoxin01 @gavinchou
+be/src/load/channel                                          @liaoxin01 @gavinchou
+be/src/load/delta_writer                                     @liaoxin01 @gavinchou
+be/src/load/group_commit                                     @liaoxin01 @gavinchou
+be/src/load/memtable                                         @liaoxin01 @gavinchou
+be/src/load/routine_load                                     @liaoxin01 @gavinchou
+be/src/load/stream_load                                      @liaoxin01 @gavinchou
+be/src/storage                                               @gavinchou
+be/src/storage/olap_common.h                                 @gavinchou @yiguolei
+be/src/storage/cache                                         @liaoxin01 @gavinchou
+be/src/storage/compaction                                    @luwei16 @gavinchou
+be/src/storage/delete                                        @luwei16 @gavinchou
+be/src/storage/index                                         @airborne12 @eldenmoon @csun5285
+be/src/storage/iterator                                      @yiguolei @gavinchou
+be/src/storage/predicate                                     @yiguolei @gavinchou
+be/src/storage/rowset                                        @gavinchou
+be/src/storage/schema_change                                 @luwei16 @gavinchou
+be/src/storage/segment                                       @yiguolei @liaoxin01 @gavinchou @eldenmoon @csun5285 @airborne12
+be/src/storage/snapshot                                      @deardeng @luwei16 @gavinchou
+be/src/storage/tablet                                        @gavinchou
+be/src/storage/task                                          @deardeng @gavinchou
+be/src/storage/txn                                           @liaoxin01 @gavinchou
+be/test/cloud                                                @luwei16 @liaoxin01 @gavinchou
+be/test/io                                                   @liaoxin01 @gavinchou @morningman @Gabriel39
+be/test/io/cache                                             @liaoxin01 @gavinchou @morningman @Gabriel39
+be/test/io/client                                            @gavinchou
+be/test/io/fs                                                @luwei16 @liaoxin01 @gavinchou @morningman @Gabriel39
+be/test/load                                                 @liaoxin01 @gavinchou
+be/test/load/channel                                         @liaoxin01 @gavinchou
+be/test/load/delta_writer                                    @liaoxin01 @gavinchou
+be/test/load/memtable                                        @liaoxin01 @gavinchou
+be/test/olap                                                 @gavinchou
+be/test/olap/rowset                                          @gavinchou
+be/test/storage                                              @gavinchou
+be/test/storage/cache                                        @liaoxin01 @gavinchou
+be/test/storage/compaction                                   @luwei16 @gavinchou
+be/test/storage/delete                                       @gavinchou
+be/test/storage/index                                        @airborne12 @eldenmoon @csun5285
+be/test/storage/iterator                                     @yiguolei @gavinchou
+be/test/storage/predicate                                    @yiguolei @gavinchou
+be/test/storage/rowset                                       @gavinchou
+be/test/storage/schema_change                                @luwei16 @gavinchou
+be/test/storage/segment                                      @yiguolei @gavinchou @eldenmoon @csun5285 @airborne12
+be/test/storage/snapshot                                     @deardeng @luwei16 @gavinchou
+be/test/storage/tablet                                       @gavinchou
+be/test/storage/txn                                          @liaoxin01 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/alter              @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/backup             @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/binlog             @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/clone              @deardeng @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud              @liaoxin01 @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/alter        @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/backup       @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/catalog      @morningman @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/common       @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/datasource   @morningman @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/load         @liaoxin01 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/master       @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/persist      @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/qe           @morrySnow @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/rpc          @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/snapshot     @luwei16 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/stage        @liaoxin01 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/storage      @luwei16 @liaoxin01 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/system       @deardeng @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cloud/transaction  @mymeiyi @liaoxin01 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/cooldown           @liaoxin01 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/fs                 @calvinkirs @morningman @liaoxin01 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/ha                 @mymeiyi @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/journal            @mymeiyi @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/load               @liaoxin01 @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/master             @mymeiyi @deardeng @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/mtmv               @morrySnow @seawinde
+fe/fe-core/src/main/java/org/apache/doris/nereids            @morrySnow @924060929 @englefly @starocean999
+fe/fe-core/src/main/java/org/apache/doris/persist            @mymeiyi @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/persist/gson       @mymeiyi @gavinchou
+fe/fe-core/src/main/java/org/apache/doris/persist/meta       @mymeiyi @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/alter              @luwei16 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/backup             @luwei16 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/binlog             @luwei16 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/clone              @deardeng @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud              @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/backup       @luwei16 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/cache        @liaoxin01 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/catalog      @morningman @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/common       @liaoxin01 @luwei16 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/load         @liaoxin01 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/master       @deardeng @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/rpc          @luwei16 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/stage        @liaoxin01 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/storage      @liaoxin01 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/system       @deardeng @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/cloud/transaction  @liaoxin01 @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/fs                 @calvinkirs @morningman @gavinchou @Gabriel39
+fe/fe-core/src/test/java/org/apache/doris/journal            @mymeiyi @gavinchou
+fe/fe-core/src/test/java/org/apache/doris/load               @liaoxin01 @gavinchou
+fe/fe-filesystem                                             @calvinkirs @morningman @gavinchou
+fe/fe-sql-parser/src/main/antlr4                             @morrySnow @starocean999 @924060929 @englefly
+cloud                                                        @gavinchou
+cloud/cmake                                                  @liaoxin01 @luwei16 @gavinchou
+cloud/conf                                                   @liaoxin01 @luwei16 @gavinchou
+cloud/script                                                 @gavinchou @luwei16
+cloud/src                                                    @gavinchou
+cloud/src/common                                             @liaoxin01 @luwei16 @gavinchou
+cloud/src/gen-cpp                                            @liaoxin01 @luwei16 @gavinchou
+cloud/src/meta-service                                       @liaoxin01 @luwei16 @gavinchou
+cloud/src/meta-store                                         @luwei16 @gavinchou
+cloud/src/rate-limiter                                       @luwei16 @gavinchou
+cloud/src/recycler                                           @liaoxin01 @gavinchou
+cloud/src/resource-manager                                   @deardeng @gavinchou
+cloud/src/snapshot                                           @luwei16 @gavinchou
+cloud/test                                                   @liaoxin01 @luwei16 @gavinchou
+gensrc/proto/cloud.proto                                     @luwei16 @gavinchou
+gensrc/proto/file_cache.proto                                @liaoxin01 @gavinchou
+gensrc/proto/olap_file.proto                                 @yiguolei @liaoxin01 @gavinchou
+gensrc/proto/segment_v2.proto                                @yiguolei @liaoxin01 @gavinchou @eldenmoon @csun5285 @airborne12

.gitignore

@@ -149,3 +149,4 @@ compile_commands.json
 .github
 
 .worktrees/
+.worktree_initialized

be/src/common/check.cpp

@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "common/check.h"
+
+#include "common/exception.h"
+#include "common/status.h"
+
+namespace doris {
+
+void doris_check_fail(std::string_view message) {
+    throw Exception(Status::FatalError("{}", message));
+}
+
+} // namespace doris

be/src/common/check.h

@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <fmt/format.h>
+#include <glog/logging.h>
+
+#include <cstddef>
+#include <ios>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <utility>
+
+namespace doris {
+
+[[noreturn]] void doris_check_fail(std::string_view message);
+
+namespace detail {
+template <typename T>
+concept OstreamPrintable = requires(std::ostream& os, const T& value) { os << value; };
+
+class DorisCheckMessage {
+public:
+    explicit DorisCheckMessage(std::string_view message) { _stream << message; }
+
+    template <typename T>
+    DorisCheckMessage& operator<<(const T& value) {
+        _stream << value;
+        return *this;
+    }
+
+    DorisCheckMessage& operator<<(std::ostream& (*func)(std::ostream&)) {
+        func(_stream);
+        return *this;
+    }
+
+    DorisCheckMessage& operator<<(std::ios& (*func)(std::ios&)) {
+        func(_stream);
+        return *this;
+    }
+
+    DorisCheckMessage& operator<<(std::ios_base& (*func)(std::ios_base&)) {
+        func(_stream);
+        return *this;
+    }
+
+    [[noreturn]] void fail() { doris_check_fail(_stream.str()); }
+
+private:
+    std::ostringstream _stream;
+};
+
+class DorisCheckMessageVoidify {
+public:
+    [[noreturn]] void operator&(DorisCheckMessage& message) const { message.fail(); }
+    [[noreturn]] void operator&(DorisCheckMessage&& message) const { message.fail(); }
+};
+
+class DorisCheckResult {
+public:
+    explicit DorisCheckResult(bool ok) : _ok(ok) {}
+    explicit DorisCheckResult(std::string message) : _ok(false), _message(std::move(message)) {}
+
+    bool ok() const { return _ok; }
+    const std::string& message() const { return _message; }
+
+private:
+    bool _ok;
+    std::string _message;
+};
+
+template <typename T>
+std::string doris_check_value_to_string(const T& value) {
+    if constexpr (std::is_same_v<std::decay_t<T>, std::nullptr_t>) {
+        return "nullptr";
+    } else if constexpr (std::is_same_v<std::decay_t<T>, bool>) {
+        return value ? "true" : "false";
+    } else if constexpr (OstreamPrintable<T>) {
+        std::ostringstream oss;
+        oss << std::boolalpha << value;
+        return oss.str();
+    } else {
+        return "<unprintable>";
+    }
+}
+
+template <typename Lhs, typename Rhs, typename Comparator>
+DorisCheckResult doris_check_binary_op_result(const Lhs& lhs, const Rhs& rhs,
+                                              std::string_view lhs_expr, std::string_view rhs_expr,
+                                              std::string_view op_expr, Comparator comparator) {
+    if (static_cast<bool>(comparator(lhs, rhs))) {
+        return DorisCheckResult(true);
+    }
+    return DorisCheckResult(fmt::format("Check failed: {} {} {} ({} vs {})", lhs_expr, op_expr,
+                                        rhs_expr, doris_check_value_to_string(lhs),
+                                        doris_check_value_to_string(rhs)));
+}
+} // namespace detail
+
+} // namespace doris
+
+// core in Debug mode, exception in Release mode.
+#define DORIS_CHECK(stmt)                                                  \
+    if (bool _doris_check_ok = static_cast<bool>(stmt); _doris_check_ok) { \
+    } else [[unlikely]]                                                    \
+        ::doris::detail::DorisCheckMessageVoidify() &                      \
+                ::doris::detail::DorisCheckMessage("Check failed: " #stmt)
+
+// Use DORIS_CHECK_* only for invariants that must also be checked in Release builds.
+// Keep DCHECK_* in loops or other hot paths where Release checks would add overhead.
+#ifndef NDEBUG
+#define DORIS_CHECK_EQ(val1, val2) DCHECK_EQ(val1, val2)
+#define DORIS_CHECK_NE(val1, val2) DCHECK_NE(val1, val2)
+#define DORIS_CHECK_LT(val1, val2) DCHECK_LT(val1, val2)
+#define DORIS_CHECK_LE(val1, val2) DCHECK_LE(val1, val2)
+#define DORIS_CHECK_GT(val1, val2) DCHECK_GT(val1, val2)
+#define DORIS_CHECK_GE(val1, val2) DCHECK_GE(val1, val2)
+#else
+#define DORIS_CHECK_BINARY_OP(val1, val2, op, op_str)                             \
+    if (auto _doris_check_result = ::doris::detail::doris_check_binary_op_result( \
+                (val1), (val2), #val1, #val2, op_str,                             \
+                [](const auto& _doris_check_lhs, const auto& _doris_check_rhs) {  \
+                    return _doris_check_lhs op _doris_check_rhs;                  \
+                });                                                               \
+        _doris_check_result.ok()) {                                               \
+    } else [[unlikely]]                                                           \
+        ::doris::detail::DorisCheckMessageVoidify() &                             \
+                ::doris::detail::DorisCheckMessage(_doris_check_result.message())
+
+#define DORIS_CHECK_EQ(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, ==, "==")
+#define DORIS_CHECK_NE(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, !=, "!=")
+#define DORIS_CHECK_LT(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, <, "<")
+#define DORIS_CHECK_LE(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, <=, "<=")
+#define DORIS_CHECK_GT(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, >, ">")
+#define DORIS_CHECK_GE(val1, val2) DORIS_CHECK_BINARY_OP(val1, val2, >=, ">=")
+#endif

be/src/common/config.cpp

@@ -1050,9 +1050,8 @@ DEFINE_mInt32(merged_hdfs_min_io_size, "8192");
 
 // OrcReader
 DEFINE_mInt32(orc_natural_read_size_mb, "8");
-DEFINE_mInt64(big_column_size_buffer, "65535");
-DEFINE_mInt64(small_column_size_buffer, "100");
-
+DEFINE_Validator(orc_natural_read_size_mb,
+                 [](const int config) -> bool { return config > 0 && config <= 1024; });
 // Perform the always_true check at intervals determined by runtime_filter_sampling_frequency
 DEFINE_mInt32(runtime_filter_sampling_frequency, "32");
 DEFINE_mInt32(execution_max_rpc_timeout_sec, "3600");