diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index 33a77bd10e1258..c1ad4a729654ce 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -38,6 +38,7 @@ #include "storage/rowset/beta_rowset.h" #include "storage/rowset/rowset.h" #include "storage/rowset/rowset_factory.h" +#include "storage/schema.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_fwd.h" @@ -237,14 +238,12 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque // FIXME(cyx): Should trigger compaction on base_tablet if there are too many rowsets to convert. - // Create a new tablet schema, should merge with dropped columns in light weight schema change + // Build the visible base schema. Historical delete columns are resolved by DeleteHandler. _base_tablet_schema = std::make_shared(); _base_tablet_schema->update_tablet_columns(*_base_tablet->tablet_schema(), request.columns); _new_tablet_schema = _new_tablet->tablet_schema(); - std::vector return_columns; - return_columns.resize(_base_tablet_schema->num_columns()); - std::iota(return_columns.begin(), return_columns.end(), 0); + ReadSchemaSPtr read_schema = std::make_shared(_base_tablet_schema->columns()); // delete handlers to filter out deleted rows DeleteHandler delete_handler; @@ -252,12 +251,15 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque for (auto& split : rs_splits) { auto& rs_meta = split.rs_reader->rowset()->rowset_meta(); if (rs_meta->has_delete_predicate()) { - _base_tablet_schema->merge_dropped_columns(*rs_meta->tablet_schema()); delete_predicates.push_back(rs_meta); } } - RETURN_IF_ERROR(delete_handler.init(_base_tablet_schema, delete_predicates, - start_resp.alter_version())); + std::vector dropped_columns; + RETURN_IF_ERROR(delete_handler.init(delete_predicates, start_resp.alter_version(), read_schema, + dropped_columns)); + for (auto& column : dropped_columns) { + read_schema->append_column(std::make_shared(std::move(column))); + } // reader_context is stack variables, it's lifetime MUST keep the same with rs_readers RowsetReaderContext reader_context; @@ -265,8 +267,7 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque reader_context.tablet_schema = _base_tablet_schema; reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; - reader_context.return_columns = &return_columns; - reader_context.sequence_id_idx = reader_context.tablet_schema->sequence_col_idx(); + reader_context.read_schema = read_schema; reader_context.is_unique = _base_tablet->keys_type() == UNIQUE_KEYS; reader_context.batch_size = ALTER_TABLE_BATCH_SIZE; reader_context.delete_bitmap = _base_tablet->tablet_meta()->delete_bitmap_ptr(); @@ -274,11 +275,11 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque std::vector cluster_key_idxes; if (!_base_tablet_schema->cluster_key_uids().empty()) { for (const auto& uid : _base_tablet_schema->cluster_key_uids()) { - cluster_key_idxes.emplace_back(_base_tablet_schema->field_index(uid)); + cluster_key_idxes.emplace_back(read_schema->ordinal_by_uid(uid)); } reader_context.read_orderby_key_columns = &cluster_key_idxes; reader_context.is_unique = false; - reader_context.sequence_id_idx = -1; + reader_context.use_sequence_column_for_merge_order = false; } for (auto& split : rs_splits) { diff --git a/be/src/exec/operator/file_scan_operator.h b/be/src/exec/operator/file_scan_operator.h index 7d8fae51b78439..bffa424ced33f2 100644 --- a/be/src/exec/operator/file_scan_operator.h +++ b/be/src/exec/operator/file_scan_operator.h @@ -112,17 +112,6 @@ class FileScanOperatorX final : public ScanOperatorX { return _batch_split_mode ? 1 : ScanOperatorX::parallelism(state); } - int get_column_id(const std::string& col_name) const override { - int column_id_counter = 0; - for (const auto& slot : _output_tuple_desc->slots()) { - if (slot->col_name() == col_name) { - return column_id_counter; - } - column_id_counter++; - } - return column_id_counter; - } - bool can_push_down_column_predicate(const SlotDescriptor* slot) const override; private: diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index dda268de829252..b789b8d1f0e92b 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -298,8 +298,6 @@ Status OlapScanLocalState::_init_profile() { ADD_CHILD_TIMER(_scanner_profile, "TabletReaderInitTimer", "ReaderInitTime"); _tablet_reader_capture_rs_readers_timer = ADD_CHILD_TIMER( _scanner_profile, "TabletReaderCaptureRsReadersTimer", "TabletReaderInitTimer"); - _tablet_reader_init_return_columns_timer = ADD_CHILD_TIMER( - _scanner_profile, "TabletReaderInitReturnColumnsTimer", "TabletReaderInitTimer"); _tablet_reader_init_keys_param_timer = ADD_CHILD_TIMER( _scanner_profile, "TabletReaderInitKeysParamTimer", "TabletReaderInitTimer"); _tablet_reader_init_orderby_keys_param_timer = ADD_CHILD_TIMER( @@ -329,8 +327,8 @@ Status OlapScanLocalState::_init_profile() { _segment_iterator_init_timer = ADD_CHILD_TIMER(_scanner_profile, "SegmentIteratorInitTimer", "BlockFetchTime"); - _segment_iterator_init_return_column_iterators_timer = - ADD_CHILD_TIMER(_scanner_profile, "SegmentIteratorInitReturnColumnIteratorsTimer", + _segment_iterator_init_column_iterators_timer = + ADD_CHILD_TIMER(_scanner_profile, "SegmentIteratorInitColumnIteratorsTimer", "SegmentIteratorInitTimer"); _segment_iterator_init_index_iterators_timer = ADD_CHILD_TIMER( _scanner_profile, "SegmentIteratorInitIndexIteratorsTimer", "SegmentIteratorInitTimer"); diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 9f7ddca05274a0..78443a0cac803e 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -301,7 +301,6 @@ class OlapScanLocalState final : public ScanLocalState { // timer about tablet reader RuntimeProfile::Counter* _tablet_reader_init_timer = nullptr; RuntimeProfile::Counter* _tablet_reader_capture_rs_readers_timer = nullptr; - RuntimeProfile::Counter* _tablet_reader_init_return_columns_timer = nullptr; RuntimeProfile::Counter* _tablet_reader_init_keys_param_timer = nullptr; RuntimeProfile::Counter* _tablet_reader_init_orderby_keys_param_timer = nullptr; RuntimeProfile::Counter* _tablet_reader_init_conditions_param_timer = nullptr; @@ -318,7 +317,7 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _rowset_reader_load_segments_timer = nullptr; RuntimeProfile::Counter* _segment_iterator_init_timer = nullptr; - RuntimeProfile::Counter* _segment_iterator_init_return_column_iterators_timer = nullptr; + RuntimeProfile::Counter* _segment_iterator_init_column_iterators_timer = nullptr; RuntimeProfile::Counter* _segment_iterator_init_index_iterators_timer = nullptr; RuntimeProfile::Counter* _segment_iterator_init_segment_prefetchers_timer = nullptr; @@ -375,14 +374,6 @@ class OlapScanOperatorX final : public ScanOperatorX { Status prepare(RuntimeState* state) override; - int get_column_id(const std::string& col_name) const override { - if (!_tablet_schema) { - return -1; - } - const auto& column = *DORIS_TRY(_tablet_schema->column(col_name)); - return _tablet_schema->field_index(column.unique_id()); - } - private: friend class OlapScanLocalState; TOlapScanNode _olap_scan_node; diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index e1aff774e7a39d..8af7196952978c 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -51,6 +51,7 @@ #include "runtime/descriptors.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_profile_counter_names.h" +#include "storage/predicate/like_column_predicate.h" #include "storage/predicate/null_predicate.h" #include "storage/predicate/predicate_creator.h" @@ -482,8 +483,10 @@ Status ScanLocalState::_normalize_predicate(VExprContext* context, cons } // `node_type` of function filter is FUNCTION_CALL or COMPOUND_PRED if (state()->enable_function_pushdown()) { - RETURN_IF_PUSH_DOWN(_normalize_function_filters(context, slot, &pdt), - status); + RETURN_IF_PUSH_DOWN( + _normalize_function_filters( + context, slot, _slot_id_to_predicates[slot->id()], &pdt), + status); } }, *range); @@ -529,7 +532,7 @@ Status ScanLocalStateBase::_normalize_bloom_filter( *pdt = _should_push_down_bloom_filter(); if (*pdt != PushDownType::UNACCEPTABLE) { pred = create_bloom_filter_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? expr->get_child(0)->data_type() : slot->type(), expr->get_bloom_filter_func()); @@ -562,8 +565,9 @@ Status ScanLocalStateBase::_normalize_topn_filter( return Status::OK(); } -Status ScanLocalStateBase::_normalize_function_filters(VExprContext* expr_ctx, SlotDescriptor* slot, - PushDownType* pdt) { +Status ScanLocalStateBase::_normalize_function_filters( + VExprContext* expr_ctx, SlotDescriptor* slot, + std::vector>& predicates, PushDownType* pdt) { auto expr = expr_ctx->root()->is_rf_wrapper() ? expr_ctx->root()->get_impl() : expr_ctx->root(); bool opposite = false; VExpr* fn_expr = expr.get(); @@ -580,8 +584,10 @@ Status ScanLocalStateBase::_normalize_function_filters(VExprContext* expr_ctx, S RETURN_IF_ERROR(_should_push_down_function_filter(assert_cast(fn_expr), expr_ctx, &val, &fn_ctx, temp_pdt)); if (temp_pdt != PushDownType::UNACCEPTABLE) { - std::string col = slot->col_name(); - _push_down_functions.emplace_back(opposite, col, fn_ctx, val); + const auto column_id = + cast_set(output_tuple_desc()->get_column_id(slot->id())); + predicates.emplace_back(LikeColumnPredicate::create_shared( + opposite, column_id, slot->col_name(), fn_ctx, val)); *pdt = temp_pdt; } } @@ -786,15 +792,13 @@ Status ScanLocalStateBase::_normalize_in_predicate( } } pred = is_in ? create_in_list_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), - slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? root->get_child(0)->data_type() : slot->type(), hybrid_set, false) : create_in_list_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), - slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? root->get_child(0)->data_type() : slot->type(), @@ -843,7 +847,7 @@ Status ScanLocalStateBase::_normalize_binary_predicate( switch (op) { case SQLFilterOp::FILTER_EQ: pred = create_comparison_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? root->get_child(0)->data_type() : slot->type(), @@ -851,7 +855,7 @@ Status ScanLocalStateBase::_normalize_binary_predicate( break; case SQLFilterOp::FILTER_NE: pred = create_comparison_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? root->get_child(0)->data_type() : slot->type(), @@ -859,7 +863,7 @@ Status ScanLocalStateBase::_normalize_binary_predicate( break; case SQLFilterOp::FILTER_LESS: pred = create_comparison_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? root->get_child(0)->data_type() : slot->type(), @@ -867,7 +871,7 @@ Status ScanLocalStateBase::_normalize_binary_predicate( break; case SQLFilterOp::FILTER_LARGER: pred = create_comparison_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? root->get_child(0)->data_type() : slot->type(), @@ -875,7 +879,7 @@ Status ScanLocalStateBase::_normalize_binary_predicate( break; case SQLFilterOp::FILTER_LESS_OR_EQUAL: pred = create_comparison_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? root->get_child(0)->data_type() : slot->type(), @@ -883,7 +887,7 @@ Status ScanLocalStateBase::_normalize_binary_predicate( break; case SQLFilterOp::FILTER_LARGER_OR_EQUAL: pred = create_comparison_predicate( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), + output_tuple_desc()->get_column_id(slot->id()), slot->col_name(), slot->type()->get_primitive_type() == TYPE_VARIANT ? root->get_child(0)->data_type() : slot->type(), @@ -978,17 +982,15 @@ Status ScanLocalStateBase::_normalize_is_null_predicate( auto fn_call = assert_cast(root.get()); if (fn_call->fn().name.function_name == "is_null_pred") { - pred = NullPredicate::create_shared( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), true, - T); + pred = NullPredicate::create_shared(output_tuple_desc()->get_column_id(slot->id()), + slot->col_name(), true, T); auto temp_range = ColumnValueRange::create_empty_column_value_range( slot->is_nullable(), range.precision(), range.scale()); temp_range.set_contain_null(true); range.intersection(temp_range); } else if (fn_call->fn().name.function_name == "is_not_null_pred") { - pred = NullPredicate::create_shared( - _parent->intermediate_row_desc().get_column_id(slot->id()), slot->col_name(), false, - T); + pred = NullPredicate::create_shared(output_tuple_desc()->get_column_id(slot->id()), + slot->col_name(), false, T); auto temp_range = ColumnValueRange::create_empty_column_value_range( slot->is_nullable(), range.precision(), range.scale()); temp_range.set_contain_null(false); @@ -1284,8 +1286,7 @@ Status ScanOperatorX::prepare(RuntimeState* state) { .slot_ref.slot_id]; DCHECK(s != nullptr); if (can_push_down_column_predicate(s)) { - auto col_name = s->col_name(); - cid = get_column_id(col_name); + cid = _output_tuple_desc->get_column_id(s->id()); } } RETURN_IF_ERROR(state->get_query_ctx()->get_runtime_predicate(id).init_target( diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index e11eb506a772ab..4071405b4f6d45 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -32,7 +32,6 @@ #include "exec/runtime_filter/runtime_filter_partition_pruner.h" #include "exec/scan/scan_node.h" #include "exec/scan/scanner_context.h" -#include "exprs/function_filter.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vin_predicate.h" #include "runtime/descriptors.h" @@ -156,9 +155,6 @@ class ScanLocalStateBase : public PipelineXLocalState<> { // Moved from ScanLocalState to avoid re-instantiation for each Derived type. std::atomic _eos = false; int _max_pushdown_conditions_per_column = 1024; - // Save all function predicates which may be pushed down to data source. - std::vector _push_down_functions; - // Virtual methods with default implementations; overridden by subclasses when supported. // Declared here so that the normalize methods below (non-Derived-template) can call them. virtual bool _push_down_topn(const RuntimePredicate& predicate) { return false; } @@ -199,6 +195,7 @@ class ScanLocalStateBase : public PipelineXLocalState<> { std::vector>& predicates, PushDownType* pdt); Status _normalize_function_filters(VExprContext* expr_ctx, SlotDescriptor* slot, + std::vector>& predicates, PushDownType* pdt); // Inner PrimitiveType-template methods. Moved to base to avoid N(Derived)×M(PrimitiveType) @@ -345,7 +342,6 @@ class ScanLocalState : public ScanLocalStateBase { // Parsed from conjuncts phmap::flat_hash_map _slot_id_to_value_range; phmap::flat_hash_map>> _slot_id_to_predicates; - std::vector> _or_predicates; std::vector> _filter_dependencies; @@ -386,8 +382,6 @@ class ScanOperatorX : public OperatorX { return &_parsed_partition_boundaries; } - [[nodiscard]] virtual int get_column_id(const std::string& col_name) const { return -1; } - [[nodiscard]] virtual bool can_push_down_column_predicate(const SlotDescriptor*) const { return true; } diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 9ffdb94e149c91..df582c173ae1f5 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -26,7 +26,6 @@ #include #include -#include #include #include @@ -43,7 +42,6 @@ #include "exec/common/variant_util.h" #include "exec/operator/olap_scan_operator.h" #include "exec/scan/scan_node.h" -#include "exprs/function_filter.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "io/cache/block_file_cache_profile.h" @@ -61,6 +59,8 @@ #include "storage/olap_common.h" #include "storage/olap_tuple.h" #include "storage/olap_utils.h" +#include "storage/predicate/predicate_creator.h" +#include "storage/schema.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_schema.h" #ifndef NDEBUG @@ -85,14 +85,12 @@ OlapScanner::OlapScanner(ScanLocalStateBase* parent, OlapScanner::Params&& param .start_key {}, .end_key {}, .predicates {}, - .function_filters {}, .delete_predicates {}, .target_cast_type_for_variants {}, .all_access_paths {}, .predicate_access_paths {}, .rs_splits {}, - .return_columns {}, - .tso_predicate_column_id {}, + .read_schema {}, .output_columns {}, .extra_columns {}, .common_expr_ctxs_push_down {}, @@ -103,9 +101,7 @@ OlapScanner::OlapScanner(ScanLocalStateBase* parent, OlapScanner::Params&& param .collection_statistics {}, .ann_topn_runtime {}, .condition_cache_digest = parent->get_condition_cache_digest(), - .binlog_scan_type = params.binlog_scan_type, - .start_tso = std::nullopt, - .end_tso = std::nullopt}), + .binlog_scan_type = params.binlog_scan_type}), _start_tso(params.start_tso), _end_tso(params.end_tso), _initial_file_cache_stats(std::move(params.initial_file_cache_stats)) { @@ -115,30 +111,6 @@ OlapScanner::OlapScanner(ScanLocalStateBase* parent, OlapScanner::Params&& param _vector_search_params = params.state->get_vector_search_params(); } -static std::string read_columns_to_string(TabletSchemaSPtr tablet_schema, - const std::vector& read_columns) { - // avoid too long for one line, - // it is hard to display in `show profile` stmt if one line is too long. - const int col_per_line = 10; - int i = 0; - std::string read_columns_string; - read_columns_string += "["; - for (auto it = read_columns.cbegin(); it != read_columns.cend(); it++) { - if (it != read_columns.cbegin()) { - read_columns_string += ", "; - } - read_columns_string += tablet_schema->columns().at(*it)->name(); - if (i >= col_per_line) { - read_columns_string += "\n"; - i = 0; - } else { - ++i; - } - } - read_columns_string += "]"; - return read_columns_string; -} - static bool has_file_cache_statistics(const io::FileCacheStatistics& stats) { return stats.num_local_io_total != 0 || stats.num_remote_io_total != 0 || stats.num_peer_io_total != 0 || stats.local_io_timer != 0 || @@ -192,11 +164,16 @@ Status OlapScanner::_prepare_impl() { context->prepare_ann_range_search(_vector_search_params); } - for (auto pair : local_state->_slot_id_to_virtual_column_expr) { - // Scanner will be executed in a different thread, so we need to clone the context. - VExprContextSPtr context; - RETURN_IF_ERROR(pair.second->clone(_state, context)); - _slot_id_to_virtual_column_expr[pair.first] = context; + for (auto* slot : _output_tuple_desc->slots()) { + if (slot->get_virtual_column_expr()) { + auto expr_it = local_state->_slot_id_to_virtual_column_expr.find(slot->id()); + DORIS_CHECK(expr_it != local_state->_slot_id_to_virtual_column_expr.end()); + // Scanner will be executed in a different thread, so we need to clone the context. + VExprContextSPtr context; + RETURN_IF_ERROR(expr_it->second->clone(_state, context)); + _virtual_column_exprs[_output_tuple_desc->get_column_id(slot->id())] = + std::move(context); + } } _score_runtime = local_state->_score_runtime; @@ -276,15 +253,14 @@ Status OlapScanner::_prepare_impl() { } // Initialize tablet_reader_params - RETURN_IF_ERROR(_init_tablet_reader_params( - local_state->_parent->cast()._slot_id_to_slot_desc, _key_ranges, - local_state->_slot_id_to_predicates, local_state->_push_down_functions)); + RETURN_IF_ERROR( + _init_tablet_reader_params(_key_ranges, local_state->_slot_id_to_predicates)); } - // add read columns in profile + // Add the read schema in profile. if (_state->enable_profile()) { _profile->add_info_string("ReadColumns", - read_columns_to_string(tablet_schema, _return_columns)); + _tablet_reader_params.read_schema->read_columns_to_string()); } if (_tablet_reader_params.score_runtime) { @@ -325,55 +301,52 @@ Status OlapScanner::_open_impl(RuntimeState* state) { return Status::OK(); } -// For binlog/snapshot incremental read. Forwards the (start_tso, end_tso] range and the TSO -// column id down to BetaRowsetReader, which builds the comparison predicates directly on read -// options. This bypasses the value/key predicate split in TabletReader::_init_conditions_param, -// guaranteeing the range filter always reaches storage (a correctness requirement for MIN_DELTA). -Status OlapScanner::_init_tso_pushdown() { +Status OlapScanner::_init_tso_predicates() { if (!_start_tso.has_value() && !_end_tso.has_value()) { return Status::OK(); } - auto& tablet_schema = _tablet_reader_params.tablet_schema; - int32_t tso_index = _tablet_reader_params.read_row_binlog ? tablet_schema->binlog_tso_col_idx() - : tablet_schema->commit_tso_col_idx(); + const auto& read_schema = _tablet_reader_params.read_schema; + int32_t tso_ordinal = _tablet_reader_params.read_row_binlog ? read_schema->tso_ordinal() + : read_schema->commit_tso_ordinal(); const std::string& column_name = _tablet_reader_params.read_row_binlog ? BINLOG_TSO_COL : COMMIT_TSO_COL; - if (tso_index < 0) { - return Status::InternalError("Column {} not found in tablet schema after append", - column_name); + if (tso_ordinal < 0) { + return Status::InvalidArgument( + "Column {} must be present in the FE scan schema for incremental read", + column_name); } - // Push the TSO range down as-is; BetaRowsetReader builds the comparison predicates and - // injects them straight into read options, so they cannot be dropped by the value/key - // predicate split in TabletReader::_init_conditions_param. - _tablet_reader_params.start_tso = _start_tso; - _tablet_reader_params.end_tso = _end_tso; + const auto* tso_column = read_schema->column(tso_ordinal); + const auto& tso_data_type = read_schema->data_type(tso_ordinal); + if (_start_tso.has_value()) { + _tablet_reader_params.predicates.push_back(create_comparison_predicate( + tso_ordinal, tso_column->name(), tso_data_type, + Field::create_field(*_start_tso), false)); + } + if (_end_tso.has_value()) { + _tablet_reader_params.predicates.push_back(create_comparison_predicate( + tso_ordinal, tso_column->name(), tso_data_type, + Field::create_field(*_end_tso), false)); + } // The storage-layer statistics fast path (VStatisticsIterator, picked when // push_down_agg_type is COUNT/MINMAX) bypasses SegmentIterator and returns raw // segment row counts without applying any column predicate. The commit-tso // predicate injected above is row-level, so the fast path would both miscount - // (ignoring commit_tso <= snapshot_tso) and crash on a column-count DCHECK when - // the tso predicate column is not in return_columns. Disable it here, matching - // the binlog DETAIL/MIN_DELTA handling. + // (ignoring commit_tso <= snapshot_tso). Disable it here, matching the + // binlog DETAIL/MIN_DELTA handling. _tablet_reader_params.push_down_agg_type_opt = TPushAggOp::NONE; - - // Always carry the tso column id so BetaRowsetReader can build predicates on it. - // Whether the column must be appended to read_columns (because it is not in - // return_columns) is decided downstream in BetaRowsetReader. - _tablet_reader_params.tso_predicate_column_id = static_cast(tso_index); - + // The scan-node digest does not contain the per-range TSO bounds. + _tablet_reader_params.condition_cache_digest = 0; return Status::OK(); } // it will be called under tablet read lock because capture rs readers need Status OlapScanner::_init_tablet_reader_params( - const phmap::flat_hash_map& slot_id_to_slot_desc, const std::vector& key_ranges, const phmap::flat_hash_map>>& - slot_to_predicates, - const std::vector& function_filters) { + slot_to_predicates) { // if the table with rowset [0-x] or [0-1] [2-y], and [0-1] is empty const bool single_version = _tablet_reader_params.has_single_version(); @@ -391,12 +364,12 @@ Status OlapScanner::_init_tablet_reader_params( } RETURN_IF_ERROR(_init_variant_columns()); - RETURN_IF_ERROR(_init_return_columns()); + RETURN_IF_ERROR(_init_read_schema()); _tablet_reader_params.push_down_agg_type_opt = _local_state->get_push_down_agg_type(); - // Binlog DETAIL/MIN_DELTA scans widen `return_columns` with key/tso/op/before - // columns to drive the row-level merge in BlockReader. The storage-layer + // Binlog DETAIL/MIN_DELTA scan schemas contain key/tso/op/before columns + // needed by the row-level merge in BlockReader. The storage-layer // statistics fast path (VStatisticsIterator, picked when push_down_agg_type // is COUNT/MINMAX) bypasses SegmentIterator entirely, returning raw segment // row counts without binlog op filtering and with a schema that does not @@ -416,31 +389,11 @@ Status OlapScanner::_init_tablet_reader_params( for (const auto& ele : ((OlapScanLocalState*)_local_state)->_cast_types_for_variants) { _tablet_reader_params.target_cast_type_for_variants[ele.first] = ele.second; }; - auto& tablet_schema = _tablet_reader_params.tablet_schema; - for (auto& predicates : slot_to_predicates) { - const int sid = predicates.first; - DCHECK(slot_id_to_slot_desc.contains(sid)); - int32_t index = - tablet_schema->field_index(slot_id_to_slot_desc.find(sid)->second->col_name()); - if (index < 0) { - throw Exception( - Status::InternalError("Column {} not found in tablet schema", - slot_id_to_slot_desc.find(sid)->second->col_name())); - } - for (auto& predicate : predicates.second) { - _tablet_reader_params.predicates.push_back(predicate->clone(index)); + for (const auto& predicates : slot_to_predicates) { + for (const auto& predicate : predicates.second) { + _tablet_reader_params.predicates.push_back(predicate->clone(predicate->column_id())); } } - - std::copy(function_filters.cbegin(), function_filters.cend(), - std::inserter(_tablet_reader_params.function_filters, - _tablet_reader_params.function_filters.begin())); - - // Merge the columns in delete predicate that not in latest schema in to current tablet schema - for (auto& del_pred : _tablet_reader_params.delete_predicates) { - tablet_schema->merge_dropped_columns(*del_pred->tablet_schema()); - } - // Push key ranges to the tablet reader. // Skip the "full scan" placeholder (has_lower_bound == false) — when no key // predicates exist, start_key/end_key remain empty and the reader does a full scan. @@ -459,116 +412,18 @@ Status OlapScanner::_init_tablet_reader_params( _tablet_reader_params.profile = _local_state->custom_profile(); _tablet_reader_params.runtime_state = _state; - _tablet_reader_params.origin_return_columns = &_return_columns; - _tablet_reader_params.tablet_columns_convert_to_null_set = &_tablet_columns_convert_to_null_set; - - auto add_return_column_if_absent = [&](uint32_t cid) { - if (std::find(_tablet_reader_params.return_columns.begin(), - _tablet_reader_params.return_columns.end(), - cid) == _tablet_reader_params.return_columns.end()) { - _tablet_reader_params.return_columns.push_back(cid); - } - }; - - // MIN_DELTA / DETAIL row-binlog scans reconstruct change rows in BlockReader through a - // key-ordered merge. They must read every key column, every requested value column, the - // binlog meta columns (tso / op) and their __BEFORE__ mirrors. APPEND_ONLY streams rows - // as-is and stays on the plain projection paths below. const bool is_binlog_merge_scan = _tablet_reader_params.binlog_scan_type == TBinlogScanType::MIN_DELTA || _tablet_reader_params.binlog_scan_type == TBinlogScanType::DETAIL; - if (is_binlog_merge_scan) { - for (size_t i = 0; i < tablet_schema->num_key_columns(); ++i) { - add_return_column_if_absent(static_cast(i)); - } - for (auto cid : _return_columns) { - add_return_column_if_absent(cid); - } - - if (int32_t tso_idx = tablet_schema->binlog_tso_col_idx(); tso_idx >= 0) { - add_return_column_if_absent(static_cast(tso_idx)); - } - if (int32_t op_idx = tablet_schema->binlog_op_col_idx(); op_idx >= 0) { - add_return_column_if_absent(static_cast(op_idx)); - } - - for (auto cid : _return_columns) { - if (cid >= tablet_schema->num_key_columns()) { - const auto& col_name = tablet_schema->column(cid).name(); - std::string before_col_name; - before_col_name.append("__BEFORE__"); - before_col_name.append(col_name); - before_col_name.append("__"); - if (int32_t before_idx = tablet_schema->field_index(before_col_name); - before_idx >= 0) { - add_return_column_if_absent(static_cast(before_idx)); - } - } - } - } else if (_tablet_reader_params.direct_mode) { - _tablet_reader_params.return_columns = _return_columns; - } else { - // we need to fetch all key columns to do the right aggregation on storage engine side. - for (size_t i = 0; i < tablet_schema->num_key_columns(); ++i) { - _tablet_reader_params.return_columns.push_back(i); - } - for (auto index : _return_columns) { - if (tablet_schema->column(index).is_key()) { - continue; - } - _tablet_reader_params.return_columns.push_back(index); - } - // expand the sequence column - if (tablet_schema->has_sequence_col() || tablet_schema->has_seq_map()) { - bool has_replace_col = false; - for (auto col : _return_columns) { - if (tablet_schema->column(col).aggregation() == - FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) { - has_replace_col = true; - break; - } - } - if (auto sequence_col_idx = tablet_schema->sequence_col_idx(); - has_replace_col && tablet_schema->has_sequence_col() && - std::find(_return_columns.begin(), _return_columns.end(), sequence_col_idx) == - _return_columns.end()) { - _tablet_reader_params.return_columns.push_back(sequence_col_idx); - } - if (has_replace_col) { - const auto& val_to_seq = tablet_schema->value_col_idx_to_seq_col_idx(); - std::set return_seq_columns; - - for (auto col : _tablet_reader_params.return_columns) { - // we need to add the necessary sequence column in _return_columns, and - // Avoid adding the same seq column twice - const auto val_iter = val_to_seq.find(col); - if (val_iter != val_to_seq.end()) { - auto seq = val_iter->second; - if (std::find(_tablet_reader_params.return_columns.begin(), - _tablet_reader_params.return_columns.end(), - seq) == _tablet_reader_params.return_columns.end()) { - return_seq_columns.insert(seq); - } - } - } - _tablet_reader_params.return_columns.insert( - std::end(_tablet_reader_params.return_columns), - std::begin(return_seq_columns), std::end(return_seq_columns)); - } - } - } - - RETURN_IF_ERROR(_init_tso_pushdown()); + RETURN_IF_ERROR(_init_tso_predicates()); // Row-binlog scans must not be re-ordered or truncated by ORDER BY / TopN pushdowns, // so reset every reorder-related param for all binlog scan types. // // Only MIN_DELTA / DETAIL additionally force the storage layer to deliver rows strictly // in primary-key order, so the BlockReader can group consecutive same-key changes - // (MIN_DELTA) or emit BEFORE/AFTER pairs in deterministic order (DETAIL). Their storage - // projection is widened above with the full key prefix, which the key-ordered merge - // comparator relies on: with read_orderby_key_num_prefix_columns == 0 the comparator - // falls back to comparing the first num_key_columns block positions. + // (MIN_DELTA) or emit BEFORE/AFTER pairs in deterministic order (DETAIL). Their FE scan + // tuples include the storage columns needed by the key-ordered merge. // // APPEND_ONLY does no key grouping and keeps the raw SQL projection, which may omit // some or even all key columns. Forcing a key-ordered merge would make the fallback @@ -658,15 +513,11 @@ Status OlapScanner::_init_tablet_reader_params( // set push down topn filter _tablet_reader_params.topn_filter_source_node_ids = olap_scan_local_state->get_topn_filter_source_node_ids(_state, true); - if (!_tablet_reader_params.topn_filter_source_node_ids.empty()) { - _tablet_reader_params.topn_filter_target_node_id = - olap_scan_local_state->parent()->node_id(); - } } // If this is a Two-Phase read query, and we need to delay the release of Rowset // by rowset->update_delayed_expired_timestamp().This could expand the lifespan of Rowset - if (tablet_schema->field_index(BeConsts::ROWID_COL) >= 0) { + if (_tablet_reader_params.tablet_schema->field_index(BeConsts::ROWID_COL) >= 0) { constexpr static int delayed_s = 60; for (auto rs_reader : _tablet_reader_params.rs_splits) { uint64_t delayed_expired_timestamp = @@ -679,7 +530,7 @@ Status OlapScanner::_init_tablet_reader_params( } } - if (tablet_schema->has_global_row_id()) { + if (_tablet_reader_params.tablet_schema->has_global_row_id()) { auto& id_file_map = _state->get_id_file_map(); for (auto rs_reader : _tablet_reader_params.rs_splits) { id_file_map->add_temp_rowset(rs_reader.rs_reader->rowset()); @@ -726,13 +577,17 @@ Status OlapScanner::_init_variant_columns() { return Status::OK(); } -Status OlapScanner::_init_return_columns() { +Status OlapScanner::_init_read_schema() { // For OLAP scan, _output_tuple_desc is the storage-aligned scan tuple // descriptor. extra_key_column_slot_ids marks extra key slots that are // present only for scan-schema alignment. For example, on an AGG table with // keys (k1, k2), a query returning only k2 may still scan (k1, k2); k1 is // an extra column and can be removed by the projection output tuple. - for (auto* slot : _output_tuple_desc->slots()) { + std::vector read_columns; + std::vector expected_types; + read_columns.reserve(_output_tuple_desc->slots().size()); + expected_types.reserve(_output_tuple_desc->slots().size()); + for (uint32_t ordinal = 0; auto* slot : _output_tuple_desc->slots()) { // variant column using path to index a column int32_t index = 0; auto& tablet_schema = _tablet_reader_params.tablet_schema; @@ -751,13 +606,6 @@ Status OlapScanner::_init_return_columns() { slot->col_name(), tablet_schema->get_all_field_names(), slot->col_unique_id()); } - if (slot->get_virtual_column_expr()) { - _virtual_column_exprs[index] = _slot_id_to_virtual_column_expr[slot->id()]; - - VLOG_DEBUG << fmt::format("Virtual column, slot id: {}, cid {}, type: {}", slot->id(), - index, slot->get_data_type_ptr()->get_name()); - } - const auto& column = tablet_schema->column(index); auto* olap_local_state = static_cast(_local_state); const auto& olap_scan_node = olap_local_state->olap_scan_node(); @@ -768,7 +616,7 @@ Status OlapScanner::_init_return_columns() { // Direct readers can synthesize extra storage keys because they are only // placeholders before the scan projection removes them. Merge/aggregation // readers must still read real key values to preserve storage semantics. - _tablet_reader_params.extra_columns.insert(index); + _tablet_reader_params.extra_columns.insert(ordinal); } } int32_t unique_id = @@ -782,29 +630,27 @@ Status OlapScanner::_init_return_columns() { {unique_id, slot->predicate_access_paths()}); } - if ((slot->type()->get_primitive_type() == PrimitiveType::TYPE_STRUCT || - slot->type()->get_primitive_type() == PrimitiveType::TYPE_MAP || - slot->type()->get_primitive_type() == PrimitiveType::TYPE_ARRAY) && - !slot->all_access_paths().empty()) { - tablet_schema->add_pruned_columns_data_type(column.unique_id(), slot->type()); - } - - _return_columns.push_back(index); - if (slot->is_nullable() && !tablet_schema->column(index).is_nullable()) { - _tablet_columns_convert_to_null_set.emplace(index); - } else if (!slot->is_nullable() && tablet_schema->column(index).is_nullable()) { + read_columns.push_back(tablet_schema->columns()[index]); + expected_types.push_back(slot->get_data_type_ptr()); + if (!slot->is_nullable() && tablet_schema->column(index).is_nullable()) { return Status::Error( "slot(id: {}, name: {})'s nullable does not match " "column(tablet id: {}, index: {}, name: {}) ", slot->id(), slot->col_name(), tablet_schema->table_id(), index, tablet_schema->column(index).name()); } + ++ordinal; } - if (_return_columns.empty()) { + if (read_columns.empty()) { return Status::InternalError("failed to build storage scanner, no materialized slot!"); } + // The FE physical scan tuple is the read-path schema. It already includes + // every storage key, sequence, TSO and binlog column required below. + _tablet_reader_params.read_schema = + std::make_shared(std::move(read_columns), std::move(expected_types)); + return Status::OK(); } @@ -1068,8 +914,6 @@ void OlapScanner::_collect_profile_before_close() { COUNTER_UPDATE(local_state->_tablet_reader_init_timer, stats.tablet_reader_init_timer_ns); COUNTER_UPDATE(local_state->_tablet_reader_capture_rs_readers_timer, stats.tablet_reader_capture_rs_readers_timer_ns); - COUNTER_UPDATE(local_state->_tablet_reader_init_return_columns_timer, - stats.tablet_reader_init_return_columns_timer_ns); COUNTER_UPDATE(local_state->_tablet_reader_init_keys_param_timer, stats.tablet_reader_init_keys_param_timer_ns); COUNTER_UPDATE(local_state->_tablet_reader_init_orderby_keys_param_timer, @@ -1095,8 +939,8 @@ void OlapScanner::_collect_profile_before_close() { stats.rowset_reader_load_segments_timer_ns); COUNTER_UPDATE(local_state->_segment_iterator_init_timer, stats.segment_iterator_init_timer_ns); - COUNTER_UPDATE(local_state->_segment_iterator_init_return_column_iterators_timer, - stats.segment_iterator_init_return_column_iterators_timer_ns); + COUNTER_UPDATE(local_state->_segment_iterator_init_column_iterators_timer, + stats.segment_iterator_init_column_iterators_timer_ns); COUNTER_UPDATE(local_state->_segment_iterator_init_index_iterators_timer, stats.segment_iterator_init_index_iterators_timer_ns); COUNTER_UPDATE(local_state->_segment_iterator_init_segment_prefetchers_timer, diff --git a/be/src/exec/scan/olap_scanner.h b/be/src/exec/scan/olap_scanner.h index 0802140960e87a..337c2c2cfea2ff 100644 --- a/be/src/exec/scan/olap_scanner.h +++ b/be/src/exec/scan/olap_scanner.h @@ -44,7 +44,6 @@ namespace doris { struct OlapScanRange; -class FunctionFilter; class RuntimeProfile; class RuntimeState; class TPaloScanRange; @@ -105,14 +104,12 @@ class OlapScanner : public Scanner { private: Status _init_tablet_reader_params( - const phmap::flat_hash_map& slot_id_to_slot_desc, const std::vector& key_ranges, const phmap::flat_hash_map>>& - predicates, - const std::vector& function_filters); + predicates); - [[nodiscard]] Status _init_tso_pushdown(); - [[nodiscard]] Status _init_return_columns(); + [[nodiscard]] Status _init_tso_predicates(); + [[nodiscard]] Status _init_read_schema(); [[nodiscard]] Status _init_variant_columns(); #ifndef NDEBUG Status _check_ann_cache_hit_debug_points(const OlapReaderStatistics& stats); @@ -126,14 +123,8 @@ class OlapScanner : public Scanner { std::optional _end_tso; public: - std::vector _return_columns; - - std::unordered_set _tablet_columns_convert_to_null_set; io::FileCacheStatistics _initial_file_cache_stats; - // This field is copied from OlapScanLocalState. - std::map _slot_id_to_virtual_column_expr; - // ColumnId of virtual column to its expr context std::map _virtual_column_exprs; std::shared_ptr _score_runtime; diff --git a/be/src/exec/scan/scanner_scheduler.cpp b/be/src/exec/scan/scanner_scheduler.cpp index 230a54048a2813..3036cbdf6ee099 100644 --- a/be/src/exec/scan/scanner_scheduler.cpp +++ b/be/src/exec/scan/scanner_scheduler.cpp @@ -368,11 +368,8 @@ void ScannerScheduler::_make_sure_virtual_col_is_materialized( } std::string error_msg = fmt::format( - "Column in idx {} is nothing, block columns {}, normal_columns " - "{}, " - "virtual_column_ids [{}]", - idx, free_block->columns(), olap_scanner->_return_columns.size(), - fmt::join(virtual_column_ids, ",")); + "Column in idx {} is nothing, block columns {}, virtual_column_ids [{}]", idx, + free_block->columns(), fmt::join(virtual_column_ids, ",")); throw doris::Exception(ErrorCode::INTERNAL_ERROR, error_msg); } #endif diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index b20c6613199d52..6a8695be913600 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -20,13 +20,11 @@ #include "common/exception.h" #include "common/status.h" #include "core/data_type/define_primitive_type.h" -#include "exprs/function_filter.h" #include "exprs/hybrid_set.h" #include "exprs/minmax_predicate.h" #include "storage/predicate/bloom_filter_predicate.h" #include "storage/predicate/column_predicate.h" #include "storage/predicate/in_list_predicate.h" -#include "storage/predicate/like_column_predicate.h" namespace doris { @@ -173,58 +171,4 @@ inline auto create_bloom_filter(PrimitiveType type, bool null_aware) { return create_predicate_function(type, null_aware); } -template -std::shared_ptr create_olap_column_predicate( - uint32_t column_id, const std::shared_ptr& filter, const TabletColumn*, - bool null_aware) { - std::shared_ptr filter_olap; - filter_olap.reset(create_bloom_filter(PT, null_aware)); - filter_olap->light_copy(filter.get()); - // create a new filter to match the input filter and PT. For example, filter may be varchar, but PT is char - return BloomFilterColumnPredicate::create_shared(column_id, filter_olap); -} - -template -std::shared_ptr create_olap_column_predicate( - uint32_t column_id, const std::shared_ptr& filter, - const TabletColumn* column, bool) { - return create_in_list_predicate(column_id, filter, - column->length()); -} - -template -std::shared_ptr create_olap_column_predicate( - uint32_t column_id, const std::shared_ptr& filter, - const TabletColumn* column, bool) { - // currently only support like predicate - if constexpr (PT == TYPE_CHAR || PT == TYPE_VARCHAR || PT == TYPE_STRING) { - return LikeColumnPredicate::create_shared(filter->_opposite, column_id, column->name(), - filter->_fn_ctx, filter->_string_param); - } - throw Exception(ErrorCode::INTERNAL_ERROR, "function filter do not support type {}", PT); -} - -template -std::shared_ptr create_column_predicate(uint32_t column_id, - const std::shared_ptr& filter, - FieldType type, const TabletColumn* column, - bool null_aware = false) { - switch (type) { -#define M(NAME) \ - case FieldType::OLAP_FIELD_##NAME: { \ - return create_olap_column_predicate(column_id, filter, column, null_aware); \ - } - APPLY_FOR_PRIMTYPE(M) -#undef M - case FieldType::OLAP_FIELD_TYPE_DECIMAL: { - return create_olap_column_predicate(column_id, filter, column, null_aware); - } - case FieldType::OLAP_FIELD_TYPE_BOOL: { - return create_olap_column_predicate(column_id, filter, column, null_aware); - } - default: - return nullptr; - } -} - } // namespace doris diff --git a/be/src/exprs/function_filter.h b/be/src/exprs/function_filter.h deleted file mode 100644 index 54bd39e223398d..00000000000000 --- a/be/src/exprs/function_filter.h +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -#pragma once - -#include - -#include "core/string_ref.h" // IWYU pragma: keep -#include "exprs/function_context.h" - -namespace doris { - -class FunctionFilter { -public: - FunctionFilter(bool opposite, const std::string& col_name, doris::FunctionContext* fn_ctx, - doris::StringRef string_param) - : _opposite(opposite), - _col_name(col_name), - _fn_ctx(fn_ctx), - _string_param(string_param) {} - - bool _opposite; - std::string _col_name; - // these pointer's life time controlled by scan node - doris::FunctionContext* _fn_ctx = nullptr; - // only one param from conjunct, because now only support like predicate - doris::StringRef _string_param; -}; - -} // namespace doris diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index 6e9f25be48d864..c2a73742a34272 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -887,8 +887,7 @@ void VectorizedFnCall::prepare_ann_range_search( Status VectorizedFnCall::evaluate_ann_range_search( const segment_v2::AnnRangeSearchRuntime& range_search_runtime, - const std::vector>& cid_to_index_iterators, - const std::vector& idx_to_cid, + const std::vector>& index_iterators, const std::vector>& column_iterators, size_t rows_of_segment, roaring::Roaring& row_bitmap, segment_v2::AnnIndexStats& ann_index_stats, bool enable_result_cache, @@ -902,17 +901,12 @@ Status VectorizedFnCall::evaluate_ann_range_search( range_search_runtime.to_string()); size_t origin_num = row_bitmap.cardinality(); - const auto idx_in_block = range_search_runtime.src_col_idx; - DCHECK_LT(idx_in_block, idx_to_cid.size()) - << "idx_in_block: " << idx_in_block << ", idx_to_cid.size(): " << idx_to_cid.size(); - - ColumnId src_col_cid = idx_to_cid[idx_in_block]; - DCHECK(src_col_cid < cid_to_index_iterators.size()); - segment_v2::IndexIterator* index_iterator = cid_to_index_iterators[src_col_cid].get(); + const auto src_col_idx = range_search_runtime.src_col_idx; + DCHECK_LT(src_col_idx, index_iterators.size()); + segment_v2::IndexIterator* index_iterator = index_iterators[src_col_idx].get(); if (index_iterator == nullptr) { VLOG_DEBUG << "ANN range search skipped: " - << fmt::format("No index iterator for column cid {}", src_col_cid); - ; + << fmt::format("No index iterator for column {}", src_col_idx); return Status::OK(); } @@ -920,15 +914,15 @@ Status VectorizedFnCall::evaluate_ann_range_search( dynamic_cast(index_iterator); if (ann_index_iterator == nullptr) { VLOG_DEBUG << "ANN range search skipped: " - << fmt::format("Column cid {} has no ANN index iterator", src_col_cid); + << fmt::format("Column {} has no ANN index iterator", src_col_idx); return Status::OK(); } DCHECK(ann_index_iterator->get_reader(AnnIndexReaderType::ANN) != nullptr) - << "Ann index iterator should have reader. Column cid: " << src_col_cid; + << "Ann index iterator should have reader. Column: " << src_col_idx; std::shared_ptr ann_index_reader = std::dynamic_pointer_cast( ann_index_iterator->get_reader(segment_v2::AnnIndexReaderType::ANN)); DCHECK(ann_index_reader != nullptr) - << "Ann index reader should not be null. Column cid: " << src_col_cid; + << "Ann index reader should not be null. Column: " << src_col_idx; // Check if metrics type is match. if (ann_index_reader->get_metric_type() != range_search_runtime.metric_type) { VLOG_DEBUG << "ANN range search skipped: " @@ -966,7 +960,7 @@ Status VectorizedFnCall::evaluate_ann_range_search( SCOPED_TIMER(&(stats->load_index_costs_ns)); if (!ann_index_iterator->try_load_index()) { VLOG_DEBUG << "ANN range search skipped: " - << fmt::format("Failed to load ANN index for column cid {}", src_col_cid); + << fmt::format("Failed to load ANN index for column {}", src_col_idx); ann_index_stats.fall_back_brute_force_cnt += 1; return Status::OK(); } @@ -1005,10 +999,10 @@ Status VectorizedFnCall::evaluate_ann_range_search( // Typical situation: range search and operator is LE or LT. if (result.distance != nullptr) { DCHECK(result.row_ids != nullptr); - ColumnId dst_col_cid = idx_to_cid[range_search_runtime.dst_col_idx]; - DCHECK(dst_col_cid < column_iterators.size()); - DCHECK(column_iterators[dst_col_cid] != nullptr); - segment_v2::ColumnIterator* column_iterator = column_iterators[dst_col_cid].get(); + const auto dst_col_idx = cast_set(range_search_runtime.dst_col_idx); + DCHECK_LT(dst_col_idx, column_iterators.size()); + DCHECK(column_iterators[dst_col_idx] != nullptr); + segment_v2::ColumnIterator* column_iterator = column_iterators[dst_col_idx].get(); DCHECK(column_iterator != nullptr); segment_v2::VirtualColumnIterator* virtual_column_iterator = dynamic_cast(column_iterator); diff --git a/be/src/exprs/vectorized_fn_call.h b/be/src/exprs/vectorized_fn_call.h index 4e1544c7380bd6..e50ac0e8bcf472 100644 --- a/be/src/exprs/vectorized_fn_call.h +++ b/be/src/exprs/vectorized_fn_call.h @@ -110,8 +110,7 @@ class VectorizedFnCall : public VExpr { Status evaluate_ann_range_search( const segment_v2::AnnRangeSearchRuntime& runtime, - const std::vector>& cid_to_index_iterators, - const std::vector& idx_to_cid, + const std::vector>& index_iterators, const std::vector>& column_iterators, size_t rows_of_segment, roaring::Roaring& row_bitmap, segment_v2::AnnIndexStats& ann_index_stats, bool enable_result_cache, diff --git a/be/src/exprs/vexpr.cpp b/be/src/exprs/vexpr.cpp index 5b1f835ace5a26..c24177cba285aa 100644 --- a/be/src/exprs/vexpr.cpp +++ b/be/src/exprs/vexpr.cpp @@ -937,8 +937,7 @@ Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBase auto* column_slot_ref = assert_cast(cast_expr->get_child(0).get()); auto column_id = column_slot_ref->column_id(); const auto* storage_name_type = - context->get_index_context()->get_storage_name_and_type_by_column_id( - column_id); + context->get_index_context()->get_storage_name_and_type(column_id); auto storage_type = remove_nullable(storage_name_type->second); auto target_type = remove_nullable(cast_expr->get_target_type()); auto origin_primitive_type = storage_type->get_primitive_type(); @@ -980,14 +979,13 @@ Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBase if (child->is_slot_ref()) { auto* column_slot_ref = assert_cast(child.get()); auto column_id = column_slot_ref->column_id(); - auto* iter = context->get_index_context()->get_inverted_index_iterator_by_column_id( - column_id); + auto* iter = context->get_index_context()->get_inverted_index_iterator(column_id); //column does not have inverted index if (iter == nullptr) { continue; } const auto* storage_name_type = - context->get_index_context()->get_storage_name_and_type_by_column_id(column_id); + context->get_index_context()->get_storage_name_and_type(column_id); if (storage_name_type == nullptr) { auto err_msg = fmt::format( "storage_name_type cannot be found for column {} while in {} " @@ -1109,7 +1107,6 @@ bool VExpr::equals(const VExpr& other) { Status VExpr::evaluate_ann_range_search( const segment_v2::AnnRangeSearchRuntime& runtime, const std::vector>& index_iterators, - const std::vector& idx_to_cid, const std::vector>& column_iterators, size_t rows_of_segment, roaring::Roaring& row_bitmap, AnnIndexStats& ann_index_stats, bool enable_result_cache, AnnRangeSearchEvaluationResult& result) { diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index c0b99ab7a8f4dc..a9a74e93409c1c 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -433,8 +433,7 @@ class VExpr { #endif virtual Status evaluate_ann_range_search( const segment_v2::AnnRangeSearchRuntime& runtime, - const std::vector>& cid_to_index_iterators, - const std::vector& idx_to_cid, + const std::vector>& index_iterators, const std::vector>& column_iterators, size_t rows_of_segment, roaring::Roaring& row_bitmap, segment_v2::AnnIndexStats& ann_index_stats, bool enable_result_cache, diff --git a/be/src/exprs/vexpr_context.cpp b/be/src/exprs/vexpr_context.cpp index 13a2e0b3d445f8..c7f83e22ddfc73 100644 --- a/be/src/exprs/vexpr_context.cpp +++ b/be/src/exprs/vexpr_context.cpp @@ -496,8 +496,7 @@ void VExprContext::prepare_ann_range_search(const doris::VectorSearchUserParams& } Status VExprContext::evaluate_ann_range_search( - const std::vector>& cid_to_index_iterators, - const std::vector& idx_to_cid, + const std::vector>& index_iterators, const std::vector>& column_iterators, const std::unordered_map>& common_expr_to_slotref_map, @@ -513,8 +512,8 @@ Status VExprContext::evaluate_ann_range_search( AnnRangeSearchEvaluationResult evaluation_result; RETURN_IF_ERROR(_root->evaluate_ann_range_search( - _ann_range_search_runtime, cid_to_index_iterators, idx_to_cid, column_iterators, - rows_of_segment, row_bitmap, ann_index_stats, enable_result_cache, evaluation_result)); + _ann_range_search_runtime, index_iterators, column_iterators, rows_of_segment, + row_bitmap, ann_index_stats, enable_result_cache, evaluation_result)); if (!evaluation_result.executed) { return Status::OK(); @@ -534,7 +533,7 @@ Status VExprContext::evaluate_ann_range_search( return Status::OK(); } - DCHECK_LT(_ann_range_search_runtime.src_col_idx, idx_to_cid.size()); + DCHECK_LT(_ann_range_search_runtime.src_col_idx, index_iterators.size()); const auto src_col_idx = cast_set(_ann_range_search_runtime.src_col_idx); const auto src_col_key = cast_set(_ann_range_search_runtime.src_col_idx); auto slot_ref_map_it = common_expr_to_slotref_map.find(this); @@ -550,10 +549,8 @@ Status VExprContext::evaluate_ann_range_search( _index_context->set_true_for_index_status(slot_ref_expr_addr, src_col_idx); VLOG_DEBUG << fmt::format( - "Evaluate ann range search for expr {}, src_col_idx {}, cid {}, row_bitmap " - "cardinality {}", - _root->debug_string(), src_col_idx, idx_to_cid[_ann_range_search_runtime.src_col_idx], - row_bitmap.cardinality()); + "Evaluate ann range search for expr {}, src_col_idx {}, row_bitmap cardinality {}", + _root->debug_string(), src_col_idx, row_bitmap.cardinality()); return Status::OK(); } diff --git a/be/src/exprs/vexpr_context.h b/be/src/exprs/vexpr_context.h index 3f5e33510cda55..7c20d1d3b99d28 100644 --- a/be/src/exprs/vexpr_context.h +++ b/be/src/exprs/vexpr_context.h @@ -62,56 +62,36 @@ using ScoreRuntimeSPtr = std::shared_ptr; class IndexExecContext { public: - IndexExecContext(const std::vector& col_ids, - const std::vector>& index_iterators, + IndexExecContext(const std::vector>& index_iterators, const std::vector& storage_name_and_type_vec, std::unordered_map>& common_expr_index_status, ScoreRuntimeSPtr score_runtime, segment_v2::Segment* segment, const segment_v2::ColumnIteratorOptions& column_iter_opts) - : _col_ids(col_ids), - _index_iterators(index_iterators), + : _index_iterators(index_iterators), _storage_name_and_type(storage_name_and_type_vec), _expr_index_status(common_expr_index_status), _score_runtime(std::move(score_runtime)), _segment(segment), _column_iter_opts(column_iter_opts) {} - segment_v2::IndexIterator* get_inverted_index_iterator_by_column_id(int column_index) const { - if (column_index < 0 || column_index >= _col_ids.size()) { + segment_v2::IndexIterator* get_inverted_index_iterator(int32_t read_ordinal) const { + if (read_ordinal < 0 || static_cast(read_ordinal) >= _index_iterators.size()) { return nullptr; } - const auto& column_id = _col_ids[column_index]; - if (column_id >= _index_iterators.size()) { + const auto& iterator = _index_iterators[read_ordinal]; + if (!iterator) { return nullptr; } - if (!_index_iterators[column_id]) { - return nullptr; - } - return _index_iterators[column_id].get(); + return iterator.get(); } - const IndexFieldNameAndTypePair* get_storage_name_and_type_by_column_id( - int column_index) const { - if (column_index < 0 || column_index >= _col_ids.size()) { - return nullptr; - } - const auto& column_id = _col_ids[column_index]; - if (column_id >= _storage_name_and_type.size()) { + const IndexFieldNameAndTypePair* get_storage_name_and_type(int32_t read_ordinal) const { + if (read_ordinal < 0 || + static_cast(read_ordinal) >= _storage_name_and_type.size()) { return nullptr; } - return &_storage_name_and_type[column_id]; - } - - bool get_column_id(int column_index, ColumnId* column_id) const { - if (column_id == nullptr) { - return false; - } - if (column_index < 0 || column_index >= _col_ids.size()) { - return false; - } - *column_id = _col_ids[column_index]; - return true; + return &_storage_name_and_type[read_ordinal]; } segment_v2::Segment* segment() const { return _segment; } @@ -148,14 +128,10 @@ class IndexExecContext { _index_result_column[expr] = std::move(column); } - void set_true_for_index_status(const VExpr* expr, int column_index) { - if (column_index < 0 || column_index >= _col_ids.size()) { - return; - } - const auto& column_id = _col_ids[column_index]; - if (_expr_index_status.contains(column_id)) { - if (_expr_index_status[column_id].contains(expr)) { - _expr_index_status[column_id][expr] = true; + void set_true_for_index_status(const VExpr* expr, int32_t read_ordinal) { + if (_expr_index_status.contains(read_ordinal)) { + if (_expr_index_status[read_ordinal].contains(expr)) { + _expr_index_status[read_ordinal][expr] = true; } } } @@ -186,9 +162,6 @@ class IndexExecContext { } private: - // A reference to a vector of column IDs for the current expression's output columns. - const std::vector& _col_ids; - // A reference to a vector of unique pointers to index iterators. const std::vector>& _index_iterators; @@ -344,8 +317,7 @@ class VExprContext { void prepare_ann_range_search(const doris::VectorSearchUserParams& params); Status evaluate_ann_range_search( - const std::vector>& cid_to_index_iterators, - const std::vector& idx_to_cid, + const std::vector>& index_iterators, const std::vector>& column_iterators, const std::unordered_map>& common_expr_to_slotref_map, diff --git a/be/src/exprs/virtual_slot_ref.cpp b/be/src/exprs/virtual_slot_ref.cpp index f898269efba095..d784b7ff405037 100644 --- a/be/src/exprs/virtual_slot_ref.cpp +++ b/be/src/exprs/virtual_slot_ref.cpp @@ -222,8 +222,7 @@ bool VirtualSlotRef::equals(const VExpr& other) { * expression to perform the optimized search. * * @param range_search_runtime Runtime parameters for the range search - * @param cid_to_index_iterators Index iterators for each column - * @param idx_to_cid Column ID mapping + * @param index_iterators Index iterators in read-schema order * @param column_iterators Data column iterators * @param row_bitmap Result bitmap to be updated with matching rows * @param ann_index_stats Performance statistics collector @@ -231,14 +230,13 @@ bool VirtualSlotRef::equals(const VExpr& other) { */ Status VirtualSlotRef::evaluate_ann_range_search( const segment_v2::AnnRangeSearchRuntime& range_search_runtime, - const std::vector>& cid_to_index_iterators, - const std::vector& idx_to_cid, + const std::vector>& index_iterators, const std::vector>& column_iterators, size_t rows_of_segment, roaring::Roaring& row_bitmap, segment_v2::AnnIndexStats& ann_index_stats, bool enable_result_cache, AnnRangeSearchEvaluationResult& result) { return _virtual_column_expr->evaluate_ann_range_search( - range_search_runtime, cid_to_index_iterators, idx_to_cid, column_iterators, - rows_of_segment, row_bitmap, ann_index_stats, enable_result_cache, result); + range_search_runtime, index_iterators, column_iterators, rows_of_segment, row_bitmap, + ann_index_stats, enable_result_cache, result); } } // namespace doris diff --git a/be/src/exprs/virtual_slot_ref.h b/be/src/exprs/virtual_slot_ref.h index ade88a4cd69a32..5994c6d98627a1 100644 --- a/be/src/exprs/virtual_slot_ref.h +++ b/be/src/exprs/virtual_slot_ref.h @@ -95,8 +95,7 @@ class VirtualSlotRef MOCK_REMOVE(final) : public VExpr { * 4. Collects performance statistics * * @param range_search_runtime Runtime info containing query vector, radius, and metrics - * @param cid_to_index_iterators Vector of index iterators for each column - * @param idx_to_cid Mapping from index position to column ID + * @param index_iterators Vector of index iterators in read-schema order * @param column_iterators Vector of column iterators for data access * @param row_bitmap Output bitmap updated with matching row IDs * @param ann_index_stats Statistics collector for performance monitoring @@ -104,8 +103,7 @@ class VirtualSlotRef MOCK_REMOVE(final) : public VExpr { */ Status evaluate_ann_range_search( const segment_v2::AnnRangeSearchRuntime& range_search_runtime, - const std::vector>& cid_to_index_iterators, - const std::vector& idx_to_cid, + const std::vector>& index_iterators, const std::vector>& column_iterators, size_t rows_of_segment, roaring::Roaring& row_bitmap, segment_v2::AnnIndexStats& ann_index_stats, bool enable_result_cache, diff --git a/be/src/exprs/vsearch.cpp b/be/src/exprs/vsearch.cpp index ecaef392db87b2..bc6cd7043f3dca 100644 --- a/be/src/exprs/vsearch.cpp +++ b/be/src/exprs/vsearch.cpp @@ -73,7 +73,7 @@ Status collect_slot_search_input(const VSearchExpr& expr, const VSlotRef& slot_r bundle->field_name_to_column_id[field_name] = column_index; - auto* iterator = index_context->get_inverted_index_iterator_by_column_id(column_index); + auto* iterator = index_context->get_inverted_index_iterator(column_index); if (iterator == nullptr) { // For example, `data.items.message` has its own SlotRef in the scan schema. The // storage layer may inherit index metadata from `data`, but it still constructs a @@ -89,8 +89,7 @@ Status collect_slot_search_input(const VSearchExpr& expr, const VSlotRef& slot_r return Status::OK(); } - const auto* storage_name_type = - index_context->get_storage_name_and_type_by_column_id(column_index); + const auto* storage_name_type = index_context->get_storage_name_and_type(column_index); if (storage_name_type == nullptr) { return Status::InternalError("storage_name_type not found for column {} in {}", column_index, expr.expr_name()); diff --git a/be/src/load/delta_writer/delta_writer_v2.h b/be/src/load/delta_writer/delta_writer_v2.h index 063bcccb4fe3fb..337426515c2796 100644 --- a/be/src/load/delta_writer/delta_writer_v2.h +++ b/be/src/load/delta_writer/delta_writer_v2.h @@ -46,7 +46,6 @@ namespace doris { class FlushToken; class MemTable; -class Schema; class StorageEngine; class TupleDescriptor; class SlotDescriptor; diff --git a/be/src/load/memtable/memtable.h b/be/src/load/memtable/memtable.h index 431608966599d4..5e7c60a31ddf96 100644 --- a/be/src/load/memtable/memtable.h +++ b/be/src/load/memtable/memtable.h @@ -38,7 +38,6 @@ namespace doris { -class Schema; class SlotDescriptor; class TabletSchema; struct TabletAddRowsPayload; diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index 2c072cf1a22aa1..67fa1d22e312fe 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -465,6 +465,16 @@ std::string TupleDescriptor::debug_string() const { return out.str(); } +int TupleDescriptor::get_column_id(SlotId slot_id) const { + for (int column_id = 0; auto* slot : slots()) { + if (slot->id() == slot_id) { + return column_id; + } + ++column_id; + } + return -1; +} + RowDescriptor::RowDescriptor(const DescriptorTbl& desc_tbl, const std::vector& row_tuples) { DCHECK_GT(row_tuples.size(), 0); diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index 234e4d485653f6..cbfa92ad6c51c6 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -362,6 +362,7 @@ class TupleDescriptor { int num_materialized_slots() const { return _num_materialized_slots; } MOCK_FUNCTION const std::vector& slots() const { return _slots; } + int get_column_id(SlotId slot_id) const; bool has_varlen_slots() const { return _has_varlen_slots; } const TableDescriptor* table_desc() const { return _table_desc; } diff --git a/be/src/storage/binlog.h b/be/src/storage/binlog.h index 0907c55a97042d..097b1e6a5cada8 100644 --- a/be/src/storage/binlog.h +++ b/be/src/storage/binlog.h @@ -54,6 +54,17 @@ constexpr std::string_view kBinlogMetaPrefix = "binlog_meta_"; constexpr std::string_view kBinlogDataPrefix = "binlog_data_"; constexpr std::string_view kRowBinlogPrefix = "binlog_row_"; +namespace binlog { + +inline std::string build_before_column_name(std::string_view name) { + std::string before_name = "__BEFORE__"; + before_name.append(name.data(), name.size()); + before_name.append("__"); + return before_name; +} + +} // namespace binlog + constexpr int64_t kBinlogLsnAutoIncId = -1; // used in file directory constexpr std::string_view FDRowBinlogSuffix = "_row_binlog"; diff --git a/be/src/storage/delete/delete_handler.cpp b/be/src/storage/delete/delete_handler.cpp index 1781c7734de7d0..df622218c8c2cf 100644 --- a/be/src/storage/delete/delete_handler.cpp +++ b/be/src/storage/delete/delete_handler.cpp @@ -24,6 +24,7 @@ #include #include +#include "common/cast_set.h" #include "common/config.h" #include "common/logging.h" #include "common/status.h" @@ -523,8 +524,9 @@ DeleteHandler::ConditionParseResult DeleteHandler::parse_condition( template requires(std::is_same_v or std::is_same_v) -Status DeleteHandler::_parse_column_pred(TabletSchemaSPtr complete_schema, - TabletSchemaSPtr delete_pred_related_schema, +Status DeleteHandler::_parse_column_pred(const ReadSchema& read_schema, + std::vector& dropped_columns, + const TabletSchemaSPtr& delete_pred_related_schema, const RepeatedPtrField& sub_pred_list, DeleteConditions* delete_conditions) { for (const auto& sub_predicate : sub_pred_list) { @@ -535,17 +537,15 @@ Status DeleteHandler::_parse_column_pred(TabletSchemaSPtr complete_schema, col_unique_id = sub_predicate.column_unique_id(); } } - if (col_unique_id < 0) { - const auto& column = - *DORIS_TRY(delete_pred_related_schema->column(condition.column_name)); - col_unique_id = column.unique_id(); - } - condition.col_unique_id = col_unique_id; - const auto& column = complete_schema->column_by_uid(col_unique_id); - uint32_t index = complete_schema->field_index(col_unique_id); + ColumnId column_id; + const TabletColumn* column; + RETURN_IF_ERROR(_resolve_column(read_schema, dropped_columns, col_unique_id, + condition.column_name, delete_pred_related_schema, + &column_id, &column)); + condition.col_unique_id = column->unique_id(); std::shared_ptr predicate; - RETURN_IF_ERROR(parse_to_predicate(index, column.name(), column.get_vec_type(), condition, - _predicate_arena, predicate)); + RETURN_IF_ERROR(parse_to_predicate(column_id, column->name(), column->get_vec_type(), + condition, _predicate_arena, predicate)); if (predicate != nullptr) { delete_conditions->column_predicate_vec.push_back(predicate); } @@ -553,27 +553,87 @@ Status DeleteHandler::_parse_column_pred(TabletSchemaSPtr complete_schema, return Status::OK(); } -Status DeleteHandler::init(TabletSchemaSPtr tablet_schema, - const std::vector& delete_preds, int64_t version) { +Status DeleteHandler::_resolve_column(const ReadSchema& read_schema, + std::vector& dropped_columns, + int32_t col_unique_id, const std::string& column_name, + const TabletSchemaSPtr& delete_pred_related_schema, + ColumnId* column_id, const TabletColumn** column) { + // A valid UID is the complete identity. A dropped column and a later + // same-name replacement must remain distinct. + if (col_unique_id >= 0) { + int32_t ordinal = read_schema.ordinal_by_uid(col_unique_id); + if (ordinal >= 0) { + *column_id = ordinal; + *column = read_schema.column(ordinal); + return Status::OK(); + } + } + + int32_t rowset_schema_ordinal = -1; + if (col_unique_id >= 0) { + rowset_schema_ordinal = delete_pred_related_schema->field_index(col_unique_id); + } else { + rowset_schema_ordinal = delete_pred_related_schema->field_index(column_name); + } + if (rowset_schema_ordinal < 0) { + return Status::Error( + "cannot find delete predicate column name={} unique_id={} in rowset schema", + column_name, col_unique_id); + } + + const auto& historical_column = delete_pred_related_schema->columns()[rowset_schema_ordinal]; + int32_t ordinal = read_schema.ordinal_by_column(*historical_column); + if (ordinal >= 0) { + *column_id = ordinal; + *column = read_schema.column(ordinal); + return Status::OK(); + } + + for (size_t i = 0; i < dropped_columns.size(); ++i) { + const auto& dropped_column = dropped_columns[i]; + bool same_column = historical_column->unique_id() >= 0 + ? dropped_column.unique_id() == historical_column->unique_id() + : dropped_column.name() == historical_column->name(); + if (same_column) { + *column_id = cast_set(read_schema.num_read_columns() + i); + *column = &dropped_column; + return Status::OK(); + } + } + + dropped_columns.emplace_back(*historical_column); + *column_id = cast_set(read_schema.num_read_columns() + dropped_columns.size() - 1); + *column = &dropped_columns.back(); + return Status::OK(); +} + +Status DeleteHandler::init(const std::vector& delete_preds, int64_t version, + const ReadSchemaSPtr& read_schema, + std::vector& dropped_columns) { DCHECK(!_is_inited) << "reinitialize delete handler."; DCHECK(version >= 0) << "invalid parameters. version=" << version; + std::vector resolved_dropped_columns; for (const auto& delete_pred : delete_preds) { // Skip the delete condition with large version if (delete_pred->version().first > version) { continue; } - // Need the tablet schema at the delete condition to parse the accurate column + // Resolve against the rowset schema that created this predicate. The current schema may + // have dropped the column and added a same-name column with a different UID or type, while + // legacy predicates without UIDs must still bind by name within this historical schema. const auto& delete_pred_related_schema = delete_pred->tablet_schema(); const auto& delete_condition = delete_pred->delete_predicate(); DeleteConditions temp; temp.filter_version = delete_pred->version().first; if (!delete_condition.sub_predicates_v2().empty()) { - RETURN_IF_ERROR(_parse_column_pred(tablet_schema, delete_pred_related_schema, + RETURN_IF_ERROR(_parse_column_pred(*read_schema, resolved_dropped_columns, + delete_pred_related_schema, delete_condition.sub_predicates_v2(), &temp)); } else { // make it compatible with the former versions - RETURN_IF_ERROR(_parse_column_pred(tablet_schema, delete_pred_related_schema, + RETURN_IF_ERROR(_parse_column_pred(*read_schema, resolved_dropped_columns, + delete_pred_related_schema, delete_condition.sub_predicates(), &temp)); } for (const auto& in_predicate : delete_condition.in_predicates()) { @@ -583,27 +643,21 @@ Status DeleteHandler::init(TabletSchemaSPtr tablet_schema, int32_t col_unique_id = -1; if (in_predicate.has_column_unique_id()) { col_unique_id = in_predicate.column_unique_id(); - } else { - // if upgrade from version 2.0.x, column_unique_id maybe not set - const auto& pre_column = - *DORIS_TRY(delete_pred_related_schema->column(condition.column_name)); - col_unique_id = pre_column.unique_id(); - } - if (col_unique_id == -1) { - return Status::Error( - "cannot get column_unique_id for column {}", condition.column_name); } - condition.col_unique_id = col_unique_id; condition.condition_op = in_predicate.is_not_in() ? PredicateType::NOT_IN_LIST : PredicateType::IN_LIST; for (const auto& value : in_predicate.values()) { condition.value_str.push_back(value); } - const auto& column = tablet_schema->column_by_uid(col_unique_id); - uint32_t index = tablet_schema->field_index(col_unique_id); + ColumnId column_id; + const TabletColumn* column; + RETURN_IF_ERROR(_resolve_column(*read_schema, resolved_dropped_columns, col_unique_id, + condition.column_name, delete_pred_related_schema, + &column_id, &column)); + condition.col_unique_id = column->unique_id(); std::shared_ptr predicate; - RETURN_IF_ERROR(parse_to_in_predicate(index, column.name(), column.get_vec_type(), + RETURN_IF_ERROR(parse_to_in_predicate(column_id, column->name(), column->get_vec_type(), condition, _predicate_arena, predicate)); temp.column_predicate_vec.push_back(predicate); } @@ -611,6 +665,7 @@ Status DeleteHandler::init(TabletSchemaSPtr tablet_schema, _del_conds.emplace_back(std::move(temp)); } + dropped_columns = std::move(resolved_dropped_columns); _is_inited = true; return Status::OK(); diff --git a/be/src/storage/delete/delete_handler.h b/be/src/storage/delete/delete_handler.h index 262233ae0b0345..075df043084939 100644 --- a/be/src/storage/delete/delete_handler.h +++ b/be/src/storage/delete/delete_handler.h @@ -28,6 +28,7 @@ #include "core/arena.h" #include "storage/predicate/column_predicate.h" #include "storage/rowset/rowset_meta.h" +#include "storage/schema.h" #include "storage/tablet/tablet_schema.h" namespace doris { @@ -100,19 +101,23 @@ class DeleteHandler { DeleteHandler() = default; ~DeleteHandler(); - // Initialize DeleteHandler, use the delete conditions of this tablet whose version less than or equal to - // 'version' to fill '_del_conds'. + // Initialize DeleteHandler, use the delete conditions whose version is less + // than or equal to 'version' to fill '_del_conds'. + // + // Delete-condition columns that are absent from `read_schema` are resolved + // against the schema stored in the corresponding delete-predicate rowset and + // returned through `dropped_columns`. // NOTE: You should lock the tablet's header file before calling this function. // input: - // * schema: tablet's schema, the delete conditions and data rows are in this schema // * version: maximum version - // * with_sub_pred_v2: whether to use delete sub predicate v2 (v2 is based on PB and use column uid to specify a column, - // v1 is based on condition string, and relies on regex for parse) + // * read_schema: schema used to bind delete-predicate column ordinals + // output: + // * dropped_columns: missing delete-predicate columns in append order // return: // * Status::Error(): input parameters are not valid // * Status::Error(): alloc memory failed - Status init(TabletSchemaSPtr tablet_schema, - const std::vector& delete_preds, int64_t version); + Status init(const std::vector& delete_preds, int64_t version, + const ReadSchemaSPtr& read_schema, std::vector& dropped_columns); [[nodiscard]] bool empty() const { return _del_conds.empty(); } @@ -126,10 +131,17 @@ class DeleteHandler { requires(std::is_same_v or std::is_same_v) Status _parse_column_pred( - TabletSchemaSPtr complete_schema, TabletSchemaSPtr delete_pred_related_schema, + const ReadSchema& read_schema, std::vector& dropped_columns, + const TabletSchemaSPtr& delete_pred_related_schema, const ::google::protobuf::RepeatedPtrField& sub_pred_list, DeleteConditions* delete_conditions); + static Status _resolve_column(const ReadSchema& read_schema, + std::vector& dropped_columns, int32_t col_unique_id, + const std::string& column_name, + const TabletSchemaSPtr& delete_pred_related_schema, + ColumnId* column_id, const TabletColumn** column); + bool _is_inited = false; // DeleteConditions in _del_conds are in 'OR' relationship std::vector _del_conds; diff --git a/be/src/storage/index/ann/ann_range_search_runtime.h b/be/src/storage/index/ann/ann_range_search_runtime.h index b1fd9dd9e720f3..a0a0f7160cf077 100644 --- a/be/src/storage/index/ann/ann_range_search_runtime.h +++ b/be/src/storage/index/ann/ann_range_search_runtime.h @@ -126,12 +126,12 @@ struct AnnRangeSearchRuntime { // Core search configuration bool is_ann_range_search = false; ///< Flag indicating if ANN range search is enabled bool is_le_or_lt = true; ///< Comparison mode: true for <=, false for < - size_t src_col_idx = 0; ///< Source column index in the schema + size_t src_col_idx = 0; ///< Source read-schema ordinal size_t dim = 0; ///< Dimensionality of the vector space - int64_t dst_col_idx = -1; ///< Destination column index (-1 if not applicable) + int64_t dst_col_idx = -1; ///< Destination read-schema ordinal, or -1 double radius = 0.0; ///< Search radius/distance threshold AnnIndexMetric metric_type; ///< Distance metric (L2, Inner Product, etc.) doris::VectorSearchUserParams user_params; ///< User-defined search parameters ColumnFloat32::Ptr query_value; ///< Query vector data }; -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/src/storage/iterator/binlog_block_reader_utils.h b/be/src/storage/iterator/binlog_block_reader_utils.h index 320fef0691fec2..1dfc3520482f8d 100644 --- a/be/src/storage/iterator/binlog_block_reader_utils.h +++ b/be/src/storage/iterator/binlog_block_reader_utils.h @@ -20,10 +20,7 @@ #include #include #include -#include -#include -#include "core/block/block.h" #include "storage/binlog.h" namespace doris::binlog { @@ -35,26 +32,6 @@ constexpr int64_t STREAM_CHANGE_DELETE = 1; constexpr int64_t STREAM_CHANGE_UPDATE_BEFORE = 2; constexpr int64_t STREAM_CHANGE_UPDATE_AFTER = 3; -// Build the __BEFORE__ column name for a base column. -inline std::string build_before_column_name(std::string_view name) { - std::string before_name = "__BEFORE__"; - before_name.append(name.data(), name.size()); - before_name.append("__"); - return before_name; -} - -// Resolve __BEFORE__ column index for a base column when present. -inline int resolve_before_column_index(const doris::Block& block, int idx, int binlog_op_pos) { - if (idx == binlog_op_pos) { - return idx; - } - - const auto& col_with_name = block.get_by_position(idx); - std::string before_name = build_before_column_name(col_with_name.name); - int tmp_idx = block.get_position_by_name(before_name); - return tmp_idx < 0 ? idx : tmp_idx; -} - enum class MinDeltaResultType { SKIP, INSERT, DELETE, UPDATE_BEFORE_AFTER }; // MIN_DELTA uses row binlog op codes as indices into a 2D lookup table, so we guard the op layout here. @@ -111,4 +88,4 @@ class AggregateFunctionMinDelta { } }; -} // namespace doris::binlog \ No newline at end of file +} // namespace doris::binlog diff --git a/be/src/storage/iterator/block_reader.cpp b/be/src/storage/iterator/block_reader.cpp index 16389c84a9ee02..729682d038fb2f 100644 --- a/be/src/storage/iterator/block_reader.cpp +++ b/be/src/storage/iterator/block_reader.cpp @@ -38,14 +38,12 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" #include "exprs/aggregate/aggregate_function_reader.h" -#include "exprs/function_filter.h" #include "runtime/runtime_state.h" #include "storage/binlog.h" #include "storage/iterator/binlog_block_reader_utils.h" #include "storage/iterator/vcollect_iterator.h" #include "storage/olap_common.h" #include "storage/olap_define.h" -#include "storage/predicate/like_column_predicate.h" #include "storage/rowset/rowset.h" #include "storage/rowset/rowset_reader_context.h" #include "storage/tablet/tablet.h" @@ -77,42 +75,6 @@ Status BlockReader::next_block_with_aggregation(Block* block, bool* eof) { return res; } -// Lazily resolves the positions of the binlog meta columns (tso / lsn / op) inside the -// merged source block, and builds _before_column_idx mapping each non-meta column to its -// __BEFORE__ mirror. The resolved positions are reused across blocks; if the column -// layout changes (detected via _binlog_op_pos sanity check), they are re-resolved. -Status BlockReader::_ensure_binlog_column_pos(const Block& src_block) { - if (_binlog_column_pos_inited) { - if (_binlog_op_pos >= 0 && _binlog_op_pos < src_block.columns() && - src_block.get_by_position(_binlog_op_pos).name == BINLOG_OP_COL) { - return Status::OK(); - } - _binlog_tso_pos = -1; - _binlog_lsn_pos = -1; - _binlog_op_pos = -1; - _binlog_column_pos_inited = false; - } - - const uint32_t col_num = src_block.columns(); - _before_column_idx.resize(col_num); - for (uint32_t i = 0; i < col_num; ++i) { - const auto& name = src_block.get_by_position(i).name; - if (name == BINLOG_TSO_COL) { - _binlog_tso_pos = static_cast(i); - } else if (name == BINLOG_LSN_COL) { - _binlog_lsn_pos = static_cast(i); - } else if (name == BINLOG_OP_COL) { - _binlog_op_pos = static_cast(i); - } else { - std::string before_name = binlog::build_before_column_name(name); - int tmp_idx = src_block.get_position_by_name(before_name); - _before_column_idx[i] = tmp_idx < 0 ? i : tmp_idx; - } - } - _binlog_column_pos_inited = true; - return Status::OK(); -} - int64_t BlockReader::_read_binlog_op(const IColumn& col, size_t row) const { const IColumn* cur = &col; if (const auto* nullable = check_and_get_column(*cur)) { @@ -149,20 +111,11 @@ Status BlockReader::_write_binlog_op(IColumn& col, int64_t op) const { return Status::OK(); } -bool BlockReader::_is_binlog_meta_column(int idx) const { - return idx == _binlog_tso_pos || idx == _binlog_lsn_pos || idx == _binlog_op_pos; -} - // Resolves which source-block column to read from for a given binlog row position. -// When use_before is true and idx is a regular data column, return the index of its -// __BEFORE__ mirror (built in _before_column_idx); otherwise return idx itself. -// Binlog meta columns (tso / lsn / op) have no BEFORE mirror, so they always pass through. -int BlockReader::_resolve_source_column_index(int idx, bool use_before) const { - if (!use_before || _is_binlog_meta_column(idx)) { - return idx; - } - - return _before_column_idx[idx]; +// When use_before is true, return the ordinal of its __BEFORE__ mirror. +// Binlog meta columns map to themselves. +uint32_t BlockReader::_resolve_source_column_ordinal(uint32_t ordinal, bool use_before) const { + return use_before ? _read_schema->before_column_ordinal(ordinal) : ordinal; } void BlockReader::_init_pending_row_columns(const Block& block) { @@ -192,18 +145,16 @@ bool BlockReader::_emit_pending_row(MutableColumns& target_columns, size_t& outp // materialize the BEFORE / AFTER halves of an UPDATE pair (and INSERT / DELETE singletons). Status BlockReader::_append_change_row(MutableColumns& target_columns, const Block& src_block, size_t row_pos, int64_t output_op, bool use_before) { - for (auto idx : _normal_columns_idx) { - int target_col_idx = _return_columns_loc[idx]; - if (target_col_idx < 0) { - continue; - } - if (idx == _binlog_op_pos) { - RETURN_IF_ERROR(_write_binlog_op(*target_columns[target_col_idx], output_op)); + const int32_t op_ordinal = _read_schema->op_ordinal(); + for (size_t ordinal = 0; ordinal < target_columns.size(); ++ordinal) { + if (static_cast(ordinal) == op_ordinal) { + RETURN_IF_ERROR(_write_binlog_op(*target_columns[ordinal], output_op)); continue; } - int source_idx = _resolve_source_column_index(idx, use_before); - target_columns[target_col_idx]->insert_from(*src_block.get_by_position(source_idx).column, - row_pos); + auto source_ordinal = + _resolve_source_column_ordinal(static_cast(ordinal), use_before); + target_columns[ordinal]->insert_from(*src_block.get_by_position(source_ordinal).column, + row_pos); } return Status::OK(); } @@ -225,7 +176,9 @@ Status BlockReader::_min_delta_next_block(Block* block, bool* eof) { auto& target_columns = target_columns_guard.mutable_columns(); size_t output_row_count = 0; _init_pending_row_columns(*block); - RETURN_IF_ERROR(_ensure_binlog_column_pos(*_next_row.block)); + const int32_t tso_ordinal = _read_schema->tso_ordinal(); + const int32_t lsn_ordinal = _read_schema->lsn_ordinal(); + const int32_t op_ordinal = _read_schema->op_ordinal(); while (output_row_count < batch_max_rows()) { if (_emit_pending_row(target_columns, output_row_count)) { continue; @@ -252,78 +205,74 @@ Status BlockReader::_min_delta_next_block(Block* block, bool* eof) { continue; } size_t group_size = _stored_data_columns[0]->size(); - auto first_op = _read_binlog_op(*_stored_data_columns[_binlog_op_pos], 0); - auto last_op = _read_binlog_op(*_stored_data_columns[_binlog_op_pos], group_size - 1); + auto first_op = _read_binlog_op(*_stored_data_columns[op_ordinal], 0); + auto last_op = _read_binlog_op(*_stored_data_columns[op_ordinal], group_size - 1); auto result = binlog::AggregateFunctionMinDelta::calculate_result(first_op, last_op); switch (result) { case binlog::AggregateFunctionMinDelta::ResultType::SKIP: break; case binlog::AggregateFunctionMinDelta::ResultType::INSERT: - for (auto idx : _normal_columns_idx) { - int target_col_idx = _return_columns_loc[idx]; - if (idx == _binlog_op_pos) { - RETURN_IF_ERROR(_write_binlog_op(*target_columns[target_col_idx], + for (size_t ordinal = 0; ordinal < target_columns.size(); ++ordinal) { + if (static_cast(ordinal) == op_ordinal) { + RETURN_IF_ERROR(_write_binlog_op(*target_columns[ordinal], binlog::STREAM_CHANGE_INSERT)); } else { // insert should use most recent value - target_columns[target_col_idx]->insert_from(*_stored_data_columns[idx], - group_size - 1); + target_columns[ordinal]->insert_from(*_stored_data_columns[ordinal], + group_size - 1); } } output_row_count++; break; case binlog::AggregateFunctionMinDelta::ResultType::DELETE: - for (auto idx : _normal_columns_idx) { - int target_col_idx = _return_columns_loc[idx]; - if (idx == _binlog_op_pos) { - RETURN_IF_ERROR(_write_binlog_op(*target_columns[target_col_idx], + for (size_t ordinal = 0; ordinal < target_columns.size(); ++ordinal) { + const auto read_ordinal = static_cast(ordinal); + if (read_ordinal == op_ordinal) { + RETURN_IF_ERROR(_write_binlog_op(*target_columns[ordinal], binlog::STREAM_CHANGE_DELETE)); - } else if (idx == _binlog_lsn_pos || idx == _binlog_tso_pos) { - target_columns[target_col_idx]->insert_from(*_stored_data_columns[idx], - group_size - 1); + } else if (read_ordinal == lsn_ordinal || read_ordinal == tso_ordinal) { + target_columns[ordinal]->insert_from(*_stored_data_columns[ordinal], + group_size - 1); } else { // delete should use first op value - int source_idx = _resolve_source_column_index(idx, true); - target_columns[target_col_idx]->insert_from(*_stored_data_columns[source_idx], - 0); + auto source_ordinal = + _resolve_source_column_ordinal(static_cast(ordinal), true); + target_columns[ordinal]->insert_from(*_stored_data_columns[source_ordinal], 0); } } output_row_count++; break; case binlog::AggregateFunctionMinDelta::ResultType::UPDATE_BEFORE_AFTER: - for (auto idx : _normal_columns_idx) { - int target_col_idx = _return_columns_loc[idx]; - if (idx == _binlog_op_pos) { - RETURN_IF_ERROR(_write_binlog_op(*target_columns[target_col_idx], + for (size_t ordinal = 0; ordinal < target_columns.size(); ++ordinal) { + if (static_cast(ordinal) == op_ordinal) { + RETURN_IF_ERROR(_write_binlog_op(*target_columns[ordinal], binlog::STREAM_CHANGE_UPDATE_BEFORE)); } else { - int source_idx = _resolve_source_column_index(idx, true); - target_columns[target_col_idx]->insert_from(*_stored_data_columns[source_idx], - 0); + auto source_ordinal = + _resolve_source_column_ordinal(static_cast(ordinal), true); + target_columns[ordinal]->insert_from(*_stored_data_columns[source_ordinal], 0); } } output_row_count++; if (output_row_count >= batch_max_rows()) { - for (auto idx : _normal_columns_idx) { - int target_col_idx = _return_columns_loc[idx]; - if (idx == _binlog_op_pos) { - RETURN_IF_ERROR(_write_binlog_op(*_pending_row_columns[target_col_idx], + for (size_t ordinal = 0; ordinal < target_columns.size(); ++ordinal) { + if (static_cast(ordinal) == op_ordinal) { + RETURN_IF_ERROR(_write_binlog_op(*_pending_row_columns[ordinal], binlog::STREAM_CHANGE_UPDATE_AFTER)); } else { - _pending_row_columns[target_col_idx]->insert_from( - *_stored_data_columns[idx], group_size - 1); + _pending_row_columns[ordinal]->insert_from(*_stored_data_columns[ordinal], + group_size - 1); } } _has_pending_row = true; } else { - for (auto idx : _normal_columns_idx) { - int target_col_idx = _return_columns_loc[idx]; - if (idx == _binlog_op_pos) { - RETURN_IF_ERROR(_write_binlog_op(*target_columns[target_col_idx], + for (size_t ordinal = 0; ordinal < target_columns.size(); ++ordinal) { + if (static_cast(ordinal) == op_ordinal) { + RETURN_IF_ERROR(_write_binlog_op(*target_columns[ordinal], binlog::STREAM_CHANGE_UPDATE_AFTER)); } else { - target_columns[target_col_idx]->insert_from(*_stored_data_columns[idx], - group_size - 1); + target_columns[ordinal]->insert_from(*_stored_data_columns[ordinal], + group_size - 1); } } output_row_count++; @@ -351,7 +300,7 @@ Status BlockReader::_detail_change_next_block(Block* block, bool* eof) { auto& target_columns = target_columns_guard.mutable_columns(); size_t output_row_count = 0; _init_pending_row_columns(*block); - RETURN_IF_ERROR(_ensure_binlog_column_pos(*_next_row.block)); + const int32_t op_ordinal = _read_schema->op_ordinal(); while (output_row_count < batch_max_rows()) { if (_emit_pending_row(target_columns, output_row_count)) { continue; @@ -364,7 +313,7 @@ Status BlockReader::_detail_change_next_block(Block* block, bool* eof) { } const Block& source_block = *_next_row.block; const size_t row = _next_row.row_pos; - int64_t op = _read_binlog_op(*source_block.get_by_position(_binlog_op_pos).column, row); + int64_t op = _read_binlog_op(*source_block.get_by_position(op_ordinal).column, row); if (op == ROW_BINLOG_UPDATE) { RETURN_IF_ERROR(_append_change_row(target_columns, source_block, row, binlog::STREAM_CHANGE_UPDATE_BEFORE, true)); @@ -500,14 +449,12 @@ Status BlockReader::_init_agg_state(const ReaderParams& read_params) { _stored_has_null_tag.resize(_stored_data_columns.size()); _stored_has_variable_length_tag.resize(_stored_data_columns.size()); - auto& tablet_schema = *_tablet_schema; for (auto idx : _agg_columns_idx) { - auto column = tablet_schema.column( - read_params.origin_return_columns->at(_return_columns_loc[idx])); - AggregateFunctionPtr function = - column.get_aggregate_function(AGG_READER_SUFFIX, read_params.get_be_exec_version()); + const auto& column = *_read_schema->column(idx); + AggregateFunctionPtr function = column.get_aggregate_function( + AGG_READER_SUFFIX, read_params.get_be_exec_version(), _read_schema->data_type(idx)); - // to avoid coredump when something goes wrong(i.e. column missmatch) + // Every aggregate value column must resolve its reader aggregate function. if (!function) { return Status::InternalError( "Failed to init reader when init agg state: " @@ -539,61 +486,25 @@ Status BlockReader::init(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns); RETURN_IF_ERROR(TabletReader::init(read_params)); - auto return_column_size = read_params.origin_return_columns->size(); - _return_columns_loc.resize(read_params.return_columns.size(), -1); - std::unordered_map pos_map; - for (int i = 0; i < return_column_size; ++i) { - auto cid = read_params.origin_return_columns->at(i); - // For each original cid, find the index in return_columns - for (int j = 0; j < read_params.return_columns.size(); ++j) { - if (read_params.return_columns[j] == cid) { - if (j < _tablet->num_key_columns() || _tablet->keys_type() != AGG_KEYS) { - pos_map[cid] = (int32_t)_normal_columns_idx.size(); - _normal_columns_idx.emplace_back(j); - } else { - _agg_columns_idx.emplace_back(j); - } - _return_columns_loc[j] = i; - break; - } - } + const bool use_sequence_map = _tablet_schema->has_seq_map() && + _tablet_schema->keys_type() == UNIQUE_KEYS && !_direct_mode && + read_params.binlog_scan_type != TBinlogScanType::MIN_DELTA && + read_params.binlog_scan_type != TBinlogScanType::DETAIL && + !(read_params.reader_type == ReaderType::READER_QUERY && + _tablet->enable_unique_key_merge_on_write()); + if (use_sequence_map) { + auto read_schema = std::make_shared(*_read_schema); + RETURN_IF_ERROR(read_schema->init_sequence_map(*_tablet_schema)); + _read_schema = std::move(read_schema); } - if (_tablet_schema->has_seq_map()) { - if (_tablet_schema->has_sequence_col()) { - auto msg = "sequence columns conflict, both seq_col and seq_map are true!"; - LOG(WARNING) << msg; - return Status::InternalError(msg); - } - _has_seq_map = true; - for (auto seq_val_iter = _tablet_schema->seq_col_idx_to_value_cols_idx().cbegin(); - seq_val_iter != _tablet_schema->seq_col_idx_to_value_cols_idx().cend(); - ++seq_val_iter) { - int seq_loc = -1; - for (int i = 0; i < read_params.return_columns.size(); ++i) { - if (read_params.return_columns[i] == seq_val_iter->first) { - seq_loc = i; - break; - } - } - if (seq_loc == -1) { - // don't need to deal with this seq col - continue; - } - - std::vector pos_vec; - for (auto agg_cid : seq_val_iter->second) { - const auto& val_pos_iter = pos_map.find(agg_cid); - if (val_pos_iter == pos_map.end()) { - continue; - } - pos_vec.emplace_back(val_pos_iter->second); - } - if (_return_columns_loc[seq_loc] == -1) { - _seq_map_not_in_origin_block.emplace(seq_loc, pos_vec); - } else { - _seq_map_in_origin_block.emplace(seq_loc, pos_vec); - } + // Every merge and caller Block has the exact read-schema layout. Cache only + // the non-key columns of AGG tables that go through the aggregate machinery. + const auto num_block_columns = static_cast(_read_schema->num_block_columns()); + const bool is_agg_keys = _tablet->keys_type() == AGG_KEYS; + for (uint32_t ordinal = 0; ordinal < num_block_columns; ++ordinal) { + if (is_agg_keys && !_read_schema->column(ordinal)->is_key()) { + _agg_columns_idx.emplace_back(ordinal); } } @@ -622,15 +533,6 @@ Status BlockReader::init(const ReaderParams& read_params) { _next_block_func = &BlockReader::_direct_next_block; return Status::OK(); } - if (_has_seq_map && !_eof) { - for (auto it = _seq_map_not_in_origin_block.cbegin(); - it != _seq_map_not_in_origin_block.cend(); ++it) { - auto seq_idx = it->first; - _seq_columns.insert( - {seq_idx, _next_row.block->get_by_position(seq_idx).column->clone_empty()}); - } - } - switch (_tablet_schema->keys_type()) { case KeysType::DUP_KEYS: _next_block_func = &BlockReader::_direct_next_block; @@ -639,7 +541,7 @@ Status BlockReader::init(const ReaderParams& read_params) { if (read_params.reader_type == ReaderType::READER_QUERY && _reader_context.enable_unique_key_merge_on_write) { _next_block_func = &BlockReader::_direct_next_block; - } else if (_has_seq_map) { + } else if (use_sequence_map) { _next_block_func = &BlockReader::_replace_key_next_block; } else { _next_block_func = &BlockReader::_unique_key_next_block; @@ -677,10 +579,6 @@ Status BlockReader::_direct_next_block(Block* block, bool* eof) { return Status::OK(); } -Status BlockReader::_direct_agg_key_next_block(Block* block, bool* eof) { - return Status::OK(); -} - Status BlockReader::_replace_key_next_block(Block* block, bool* eof) { if (UNLIKELY(_eof)) { *eof = true; @@ -698,12 +596,6 @@ Status BlockReader::_replace_key_next_block(Block* block, bool* eof) { auto merged_row = 0; while (target_block_row < batch_max_rows() && !_eof) { RETURN_IF_ERROR(_insert_data_normal(target_columns)); - // use the first line to init _seq_columns - for (auto it = _seq_map_not_in_origin_block.cbegin(); - it != _seq_map_not_in_origin_block.cend(); ++it) { - auto seq_idx = it->first; - _update_last_mutil_seq(seq_idx); - } if (UNLIKELY(_reader_context.record_rowids)) { _block_row_locations[target_block_row] = _vcollect_iter.current_row_location(); } @@ -757,12 +649,10 @@ void BlockReader::_compare_sequence_map_and_replace(MutableColumns& columns) { auto src_block = _next_row.block.get(); auto src_pos = _next_row.row_pos; - // use seq column in origin block to compare and replace - for (auto it = _seq_map_in_origin_block.cbegin(); it != _seq_map_in_origin_block.cend(); ++it) { - auto seq_idx = it->first; - auto dst_seq_column = columns[_return_columns_loc[seq_idx]].get(); + for (const auto& [seq_ordinal, value_ordinals] : _read_schema->sequence_map()) { + auto* dst_seq_column = columns[seq_ordinal].get(); auto dst_pos = dst_seq_column->size() - 1; - auto src_seq_column = src_block->get_by_position(seq_idx).column; + auto src_seq_column = src_block->get_by_position(seq_ordinal).column; // the rowset version of dst is higher . auto res = dst_seq_column->compare_at(dst_pos, src_pos, *src_seq_column, -1); if (res >= 0) { @@ -770,10 +660,9 @@ void BlockReader::_compare_sequence_map_and_replace(MutableColumns& columns) { } // update value and seq column - for (auto& p : it->second) { - auto val_idx = _normal_columns_idx[p]; - auto src_column = src_block->get_by_position(val_idx).column; - auto dst_column = columns[_return_columns_loc[val_idx]].get(); + for (auto value_ordinal : value_ordinals) { + auto src_column = src_block->get_by_position(value_ordinal).column; + auto* dst_column = columns[value_ordinal].get(); dst_column->pop_back(1); dst_column->insert_from(*src_column, src_pos); } @@ -781,36 +670,6 @@ void BlockReader::_compare_sequence_map_and_replace(MutableColumns& columns) { dst_seq_column->pop_back(1); dst_seq_column->insert_from(*src_seq_column, src_pos); } - - // use temp seq block to compare and replace because origin block not contains these seq columns - for (auto it = _seq_map_not_in_origin_block.cbegin(); it != _seq_map_not_in_origin_block.cend(); - ++it) { - auto seq_idx = it->first; - auto dst_seq_column = _seq_columns[seq_idx].get(); - auto src_seq_column = src_block->get_by_position(seq_idx).column; - // the rowset version of dst is higher . - auto res = dst_seq_column->compare_at(0, src_pos, *src_seq_column, -1); - if (res >= 0) { - continue; - } - - // update value and seq column (if need to return) - for (auto& p : it->second) { - auto val_idx = _normal_columns_idx[p]; - auto src_column = src_block->get_by_position(val_idx).column; - auto dst_column = columns[_return_columns_loc[val_idx]].get(); - dst_column->pop_back(1); - dst_column->insert_from(*src_column, src_pos); - } - - _update_last_mutil_seq(seq_idx); - } -} - -void BlockReader::_update_last_mutil_seq(int seq_idx) { - auto block = _next_row.block.get(); - _seq_columns[seq_idx]->clear(); - _seq_columns[seq_idx]->insert_from(*block->get_by_position(seq_idx).column, _next_row.row_pos); } Status BlockReader::_agg_key_next_block(Block* block, bool* eof) { @@ -854,7 +713,6 @@ Status BlockReader::_agg_key_next_block(Block* block, bool* eof) { _last_agg_data_counter = 0; RETURN_IF_ERROR(_insert_data_normal(target_columns)); - target_block_row++; } else { merged_row++; @@ -920,15 +778,9 @@ Status BlockReader::_unique_key_next_block(Block* block, bool* eof) { } while (target_block_row < batch_max_rows()); if (_delete_sign_available) { - int delete_sign_idx = _reader_context.tablet_schema->field_index(DELETE_SIGN); - DCHECK(delete_sign_idx > 0); - if (delete_sign_idx <= 0 || delete_sign_idx >= target_columns.size()) { - LOG(WARNING) << "tablet_id: " << tablet()->tablet_id() << " delete sign idx " - << delete_sign_idx - << " not invalid, skip filter delete in base compaction"; - target_columns_guard.restore(); - return Status::OK(); - } + int delete_sign_idx = _read_schema->delete_sign_ordinal(); + DORIS_CHECK_GT(delete_sign_idx, 0); + DORIS_CHECK_LT(delete_sign_idx, static_cast(target_columns.size())); auto delete_filter_column = IColumn::mutate(std::move(_delete_filter_column)); reinterpret_cast(delete_filter_column.get())->resize(target_block_row); @@ -969,9 +821,15 @@ Status BlockReader::_insert_data_normal(MutableColumns& columns) { auto block = _next_row.block.get(); RETURN_IF_CATCH_EXCEPTION({ - for (auto idx : _normal_columns_idx) { - columns[_return_columns_loc[idx]]->insert_from(*block->get_by_position(idx).column, - _next_row.row_pos); + size_t aggregate_index = 0; + for (size_t ordinal = 0; ordinal < columns.size(); ++ordinal) { + if (aggregate_index < _agg_columns_idx.size() && + ordinal == _agg_columns_idx[aggregate_index]) { + ++aggregate_index; + continue; + } + columns[ordinal]->insert_from(*block->get_by_position(ordinal).column, + _next_row.row_pos); } }); return Status::OK(); @@ -1066,7 +924,7 @@ void BlockReader::_update_agg_value(MutableColumns& columns, int begin, int end, } if (is_close) { - function->insert_result_into(place, *columns[_return_columns_loc[idx]]); + function->insert_result_into(place, *columns[idx]); // reset aggregate data function->reset(place); } diff --git a/be/src/storage/iterator/block_reader.h b/be/src/storage/iterator/block_reader.h index f1107772b30db6..a93f96ee870953 100644 --- a/be/src/storage/iterator/block_reader.h +++ b/be/src/storage/iterator/block_reader.h @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -37,7 +38,6 @@ namespace doris { class ColumnPredicate; -class FunctionFilter; class RuntimeProfile; class BlockReader final : public TabletReader { @@ -64,10 +64,6 @@ class BlockReader final : public TabletReader { // Directly read row from rowset and pass to upper caller. No need to do aggregation. // This is usually used for DUPLICATE KEY tables Status _direct_next_block(Block* block, bool* eof); - // Just same as _direct_next_block, but this is only for AGGREGATE KEY tables. - // And this is an optimization for AGGR tables. - // When there is only one rowset and is not overlapping, we can read it directly without aggregation. - Status _direct_agg_key_next_block(Block* block, bool* eof); // For normal AGGREGATE KEY tables, read data by a merge heap. Status _agg_key_next_block(Block* block, bool* eof); // For UNIQUE KEY tables, read data by a merge heap. @@ -79,15 +75,11 @@ class BlockReader final : public TabletReader { Status _detail_change_next_block(Block* block, bool* eof); - Status _ensure_binlog_column_pos(const Block& src_block); - int64_t _read_binlog_op(const IColumn& col, size_t row) const; Status _write_binlog_op(IColumn& col, int64_t op) const; - bool _is_binlog_meta_column(int idx) const; - - int _resolve_source_column_index(int idx, bool use_before) const; + uint32_t _resolve_source_column_ordinal(uint32_t ordinal, bool use_before) const; void _init_pending_row_columns(const Block& block); @@ -101,8 +93,6 @@ class BlockReader final : public TabletReader { Status _insert_data_normal(MutableColumns& columns); - // for partial update table - void _update_last_mutil_seq(int seq_idx); void _compare_sequence_map_and_replace(MutableColumns& columns); // Check if the accumulated output columns have reached the preferred byte budget, @@ -129,9 +119,9 @@ class BlockReader final : public TabletReader { std::vector _agg_functions; std::vector _agg_places; - std::vector _normal_columns_idx; // key column on agg mode, all column on uniq mode - std::vector _agg_columns_idx; - std::vector _return_columns_loc; + // Read-schema ordinals of the non-key columns of AGG tables, folded + // through the aggregate machinery. + std::vector _agg_columns_idx; std::vector _agg_data_counters; int _last_agg_data_counter = 0; @@ -163,22 +153,6 @@ class BlockReader final : public TabletReader { bool _is_rowsets_overlapping = true; - int _binlog_tso_pos = -1; - int _binlog_lsn_pos = -1; - int _binlog_op_pos = -1; - bool _binlog_column_pos_inited = false; - - bool _has_seq_map = false; - // for check multi seq - std::unordered_map _seq_columns; - // MutableColumns _seq_columns; - // seq in return_columns, val pos in _normal_columns_idx - std::unordered_map> _seq_map_in_origin_block; - std::unordered_map> _seq_map_not_in_origin_block; - // For each src column index in the binlog block, the index of its companion __BEFORE__ - // column (or itself if no BEFORE mirror exists). Built lazily by _ensure_binlog_column_pos - // and consulted via _resolve_source_column_index when emitting BEFORE rows. - std::vector _before_column_idx; Arena _arena; }; diff --git a/be/src/storage/iterator/vcollect_iterator.cpp b/be/src/storage/iterator/vcollect_iterator.cpp index 390a9deb531320..2ccc4ff06d2672 100644 --- a/be/src/storage/iterator/vcollect_iterator.cpp +++ b/be/src/storage/iterator/vcollect_iterator.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "common/cast_set.h" @@ -34,7 +35,9 @@ #include "common/status.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" +#include "core/column/column_nothing.h" #include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" // IWYU pragma: keep #include "core/field.h" #include "io/io_common.h" @@ -47,6 +50,7 @@ #include "storage/olap_define.h" #include "storage/rowset/rowset.h" #include "storage/rowset/rowset_meta.h" +#include "storage/schema.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_schema.h" @@ -214,7 +218,7 @@ bool VCollectIterator::LevelIteratorComparator::operator()(LevelIterator* lhs, L int cmp_res = UNLIKELY(lhs->compare_columns()) ? lhs_ref.compare(rhs_ref, lhs->compare_columns()) - : lhs_ref.compare(rhs_ref, lhs->tablet_schema().num_key_columns()); + : lhs_ref.compare(rhs_ref, lhs->schema().num_key_columns()); if (cmp_res != 0) { return UNLIKELY(_is_reverse) ? cmp_res < 0 : cmp_res > 0; } @@ -291,12 +295,11 @@ Status VCollectIterator::_topn_next(Block* block) { // separate mutable block with the same schema so stored row positions remain stable. auto clone_block = block->clone_empty(); // Initialize virtual slot columns by schema (avoid runtime type checks). - for (const auto& [cid, expr_ctx] : _reader->_reader_context.virtual_column_exprs) { - auto it = std::find(_reader->_return_columns.begin(), _reader->_return_columns.end(), cid); - DORIS_CHECK(it != _reader->_return_columns.end()); - auto idx = cast_set(std::distance(_reader->_return_columns.begin(), it)); - DORIS_CHECK(idx < clone_block.columns()); - clone_block.get_by_position(idx).column = expr_ctx->root()->data_type()->create_column(); + // virtual_column_exprs is keyed by read-schema ordinal == block position. + for (const auto& [ordinal, expr_ctx] : _reader->_reader_context.virtual_column_exprs) { + DORIS_CHECK(ordinal < clone_block.columns()); + clone_block.get_by_position(ordinal).column = + expr_ctx->root()->data_type()->create_column(); } const size_t clone_block_columns = clone_block.columns(); MutableBlock mutable_block = MutableBlock::build_mutable_block(std::move(clone_block)); @@ -388,14 +391,6 @@ Status VCollectIterator::_topn_next(Block* block) { } if (rows_to_copy > 0) { - // create column that is not in mutable_block but in block - for (size_t j = mutable_block.columns(); j < block->columns(); ++j) { - auto col = block->get_by_position(j).clone_empty(); - mutable_block.mutable_columns().push_back(col.column->assert_mutable()); - mutable_block.data_types().push_back(std::move(col.type)); - mutable_block.get_names().push_back(std::move(col.name)); - } - size_t base = mutable_block.rows(); // append block to mutable_block RETURN_IF_ERROR(mutable_block.add_rows(block, 0, rows_to_copy)); @@ -477,12 +472,15 @@ VCollectIterator::Level0Iterator::Level0Iterator(RowsetReaderSharedPtr rs_reader DCHECK_EQ(RowsetTypePB::BETA_ROWSET, rs_reader->type()); } +std::shared_ptr VCollectIterator::Level0Iterator::_create_read_block() const { + return std::make_shared(_reader->_read_schema->create_read_block()); +} + Status VCollectIterator::Level0Iterator::init(bool get_data_by_ref) { _is_merge_iterator = _rs_reader->is_merge_iterator(); _get_data_by_ref = get_data_by_ref && _is_merge_iterator; if (!_get_data_by_ref) { - _block = std::make_shared(_schema.create_block( - _reader->_return_columns, _reader->_tablet_columns_convert_to_null_set)); + _block = _create_read_block(); } auto st = refresh_current_row(); @@ -525,8 +523,7 @@ Status VCollectIterator::Level0Iterator::refresh_current_row() { do { if (_block == nullptr && !_get_data_by_ref) { - _block = std::make_shared(_schema.create_block( - _reader->_return_columns, _reader->_tablet_columns_convert_to_null_set)); + _block = _create_read_block(); _ref.block = _block; } @@ -690,30 +687,19 @@ Status VCollectIterator::Level1Iterator::init(bool get_data_by_ref) { // Only when there are multiple children that need to be merged if (_merge && _children.size() > 1) { - auto sequence_loc = -1; - for (int loc = 0; loc < _reader->_return_columns.size(); loc++) { - if (_reader->_return_columns[loc] == _reader->_sequence_col_idx) { - sequence_loc = loc; - break; - } - } - - int32_t tso_col_id = _reader->_tablet_schema->binlog_tso_col_idx(); - if (tso_col_id >= 0) { + // Read-schema positions of the tie-break columns. + int32_t sequence_loc = _reader->_read_schema->sequence_ordinal(); + int32_t tso_ordinal = _reader->_read_schema->tso_ordinal(); + if (tso_ordinal >= 0) { DCHECK(sequence_loc == -1); - for (int loc = 0; loc < _reader->_return_columns.size(); ++loc) { - if (_reader->_return_columns[loc] == static_cast(tso_col_id)) { - sequence_loc = loc; - break; - } - } + sequence_loc = tso_ordinal; } RETURN_IF_ERROR(_validate_merge_compare_contract(sequence_loc)); _heap = std::make_unique(LevelIteratorComparator( sequence_loc, _is_reverse, _reader->_reader_context.use_insert_order_when_same, - tso_col_id >= 0)); + tso_ordinal >= 0)); for (auto&& child : _children) { DCHECK(child != nullptr); //DCHECK(child->current_row().ok()); @@ -768,56 +754,49 @@ void VCollectIterator::Level1Iterator::init_level0_iterators_for_union() { } } -// LevelIteratorComparator reads block positions that carry no bounds checks in release -// builds: either the explicit compare-column positions, or positions -// [0, tablet_schema().num_key_columns()) plus the sequence tie-break position. A read -// projection that omits or reorders the leading key columns would turn the first heap -// comparison into an out-of-bounds or semantically wrong positional access (issue #66390). -// Validate the contract against every child's first block before anything enters _heap. -Status VCollectIterator::Level1Iterator::_validate_merge_compare_contract(int sequence_loc) const { - const auto& return_columns = _reader->_return_columns; - const size_t num_key_columns = _schema.num_key_columns(); +Status VCollectIterator::Level1Iterator::_validate_merge_compare_contract( + int sequence_ordinal) const { + const auto& read_schema = *_schema; + const size_t num_key_columns = read_schema.num_key_columns(); + for (const auto& child : _children) { const IteratorRowRef* ref = child->current_row_ref(); if (ref->block == nullptr) { continue; } + const size_t block_columns = ref->block->columns(); auto contract_error = [&](const std::string& detail) { - std::string projected_ids; - for (auto cid : return_columns) { - if (!projected_ids.empty()) { - projected_ids += ','; - } - projected_ids += std::to_string(cid); - } return Status::InternalError( "merge heap compare contract violated: {}, tablet_id={}, block_columns={}, " - "num_key_columns={}, sequence_loc={}, return_columns=[{}]", - detail, _reader->_tablet->tablet_id(), block_columns, num_key_columns, - sequence_loc, projected_ids); + "read_columns={}, num_key_columns={}, sequence_ordinal={}", + detail, _reader->_tablet->tablet_id(), block_columns, + read_schema.num_block_columns(), num_key_columns, sequence_ordinal); }; + if (_compare_columns != nullptr) { - for (uint32_t pos : *_compare_columns) { - if (pos >= block_columns) { - return contract_error( - fmt::format("compare column position {} out of range", pos)); + for (uint32_t ordinal : *_compare_columns) { + if (ordinal >= block_columns) { + return contract_error("compare column ordinal " + std::to_string(ordinal) + + " out of range"); } } } else { - if (num_key_columns > return_columns.size() || num_key_columns > block_columns) { - return contract_error("projection has fewer columns than the key prefix"); + if (num_key_columns > read_schema.num_block_columns() || + num_key_columns > block_columns) { + return contract_error("read schema has fewer columns than the key prefix"); } - for (size_t i = 0; i < num_key_columns; ++i) { - if (return_columns[i] != i) { - return contract_error( - fmt::format("position {} holds column id {} instead of key column {}", - i, return_columns[i], i)); + for (size_t ordinal = 0; ordinal < num_key_columns; ++ordinal) { + const auto& read_column = *read_schema.column(ordinal); + if (!read_column.is_key()) { + return contract_error("read schema ordinal " + std::to_string(ordinal) + + " is not a key column"); } } } - if (sequence_loc != -1 && static_cast(sequence_loc) >= block_columns) { - return contract_error("sequence column position out of range"); + + if (sequence_ordinal != -1 && static_cast(sequence_ordinal) >= block_columns) { + return contract_error("sequence column ordinal out of range"); } } return Status::OK(); @@ -931,10 +910,6 @@ Status VCollectIterator::Level1Iterator::_merge_next(Block* block) { IteratorRowRef cur_row = _ref; IteratorRowRef pre_row_ref = _ref; - // append extra columns (eg. MATCH pred result column) from src_block to block - for (size_t i = block->columns(); i < cur_row.block->columns(); ++i) { - block->insert(cur_row.block->get_by_position(i).clone_empty()); - } auto target_columns_guard = block->mutate_columns_scoped(); auto& target_columns = target_columns_guard.mutable_columns(); size_t column_count = target_columns.size(); diff --git a/be/src/storage/iterator/vcollect_iterator.h b/be/src/storage/iterator/vcollect_iterator.h index 9af05e0ced1186..8518aa5bc2d0e2 100644 --- a/be/src/storage/iterator/vcollect_iterator.h +++ b/be/src/storage/iterator/vcollect_iterator.h @@ -44,7 +44,6 @@ struct pairing_heap_tag; namespace doris { -class TabletSchema; class RuntimeProfile; // Pure-computation helper: estimate whether collected data meets the byte budget @@ -122,7 +121,7 @@ class VCollectIterator { class LevelIterator { public: LevelIterator(TabletReader* reader) - : _schema(reader->tablet_schema()), + : _schema(reader->_read_schema), _compare_columns(reader->_reader_context.read_orderby_key_columns) {} virtual Status init(bool get_data_by_ref = false) = 0; @@ -143,7 +142,8 @@ class VCollectIterator { virtual ~LevelIterator() = default; - const TabletSchema& tablet_schema() const { return _schema; } + // Exact read schema of every block this iterator exposes. + const ReadSchema& schema() const { return *_schema; } const inline std::vector* compare_columns() const { return _compare_columns; } @@ -156,7 +156,7 @@ class VCollectIterator { virtual void update_profile(RuntimeProfile* profile) = 0; protected: - const TabletSchema& _schema; + const ReadSchemaSPtr _schema; IteratorRowRef _ref; std::vector* _compare_columns = nullptr; }; @@ -234,7 +234,8 @@ class VCollectIterator { Status refresh_current_row(); private: - Status _next_by_ref(IteratorRowRef* ref); + // Build an empty block with the exact read-schema layout. + std::shared_ptr _create_read_block() const; bool _is_empty() { if (_get_data_by_ref) { @@ -324,12 +325,8 @@ class VCollectIterator { bool collected_enough_rows(const MutableColumns& columns, int rows_to_merge) const; private: - // Validate that every block position LevelIteratorComparator may touch exists in - // each child's current block and, for the default key-prefix comparison, that the - // read projection really starts with the full ordered key prefix. Called before - // any child is pushed into _heap, so a broken projection surfaces as an error - // instead of an out-of-bounds positional access (issue #66390). - Status _validate_merge_compare_contract(int sequence_loc) const; + // Validate comparator ordinals and the default key-prefix layout. + Status _validate_merge_compare_contract(int sequence_ordinal) const; Status _merge_next(IteratorRowRef* ref); diff --git a/be/src/storage/iterator/vertical_block_reader.cpp b/be/src/storage/iterator/vertical_block_reader.cpp index d355f13d029976..69e2b059b94467 100644 --- a/be/src/storage/iterator/vertical_block_reader.cpp +++ b/be/src/storage/iterator/vertical_block_reader.cpp @@ -138,27 +138,25 @@ Status VerticalBlockReader::_init_collect_iter(const ReaderParams& read_params, } // build heap if key column iterator or build vertical merge iterator if value column - auto ori_return_col_size = _return_columns.size(); if (read_params.is_key_column_group) { - uint32_t seq_col_idx = -1; - if (_tablet_schema->has_sequence_col() && _tablet_schema->cluster_key_uids().empty()) { - seq_col_idx = _tablet_schema->sequence_col_idx(); + int32_t seq_col_idx = -1; + if (_tablet_schema->cluster_key_uids().empty()) { + // ordinal of the sequence column inside this column group's blocks + seq_col_idx = _read_schema->sequence_ordinal(); } if (_tablet_schema->num_key_columns() == 0) { _vcollect_iter = new_vertical_fifo_merge_iterator( std::move(*segment_iters_ptr), iterator_init_flag, rowset_ids, - ori_return_col_size, _tablet_schema->keys_type(), seq_col_idx, - _row_sources_buffer, _context_stats); + _tablet_schema->keys_type(), seq_col_idx, _row_sources_buffer, _context_stats); } else { _vcollect_iter = new_vertical_heap_merge_iterator( std::move(*segment_iters_ptr), iterator_init_flag, rowset_ids, - ori_return_col_size, _tablet_schema->keys_type(), seq_col_idx, - _row_sources_buffer, _context_stats, read_params.key_group_cluster_key_idxes); + _tablet_schema->keys_type(), seq_col_idx, _row_sources_buffer, _context_stats, + read_params.key_group_cluster_key_idxes); } } else { - _vcollect_iter = - new_vertical_mask_merge_iterator(std::move(*segment_iters_ptr), ori_return_col_size, - _row_sources_buffer, _context_stats); + _vcollect_iter = new_vertical_mask_merge_iterator(std::move(*segment_iters_ptr), + _row_sources_buffer, _context_stats); } // init collect iterator StorageReadOptions opts; @@ -186,19 +184,16 @@ void VerticalBlockReader::_init_agg_state(const ReaderParams& read_params) { if (_eof) { return; } - DCHECK(_return_columns.size() == _next_row.block->columns()); + DCHECK_EQ(_read_schema->num_block_columns(), _next_row.block->columns()); auto stored_block = _next_row.block->create_same_struct_block(_reader_context.batch_size); _stored_data_columns = std::move(*stored_block).mutate_columns(); _stored_has_null_tag.resize(_stored_data_columns.size()); _stored_has_variable_length_tag.resize(_stored_data_columns.size()); - auto& tablet_schema = *_tablet_schema; - for (size_t idx = 0; idx < _return_columns.size(); ++idx) { - AggregateFunctionPtr function = - tablet_schema.column(_return_columns.at(idx)) - .get_aggregate_function(AGG_READER_SUFFIX, - read_params.get_be_exec_version()); + for (size_t idx = 0; idx < _read_schema->num_block_columns(); ++idx) { + AggregateFunctionPtr function = _read_schema->column(idx)->get_aggregate_function( + AGG_READER_SUFFIX, read_params.get_be_exec_version(), _read_schema->data_type(idx)); DCHECK(function != nullptr); const auto* column_ptr = _stored_data_columns[idx].get(); const IColumn* columns[] = {column_ptr}; @@ -307,7 +302,7 @@ void VerticalBlockReader::_update_agg_data(MutableColumns& columns) { size_t copy_size = _copy_agg_data(); // calculate has_null_tag - for (size_t idx = 0; idx < _return_columns.size(); ++idx) { + for (size_t idx = 0; idx < _stored_data_columns.size(); ++idx) { _stored_has_null_tag[idx] = _stored_data_columns[idx]->has_null(0, copy_size); } @@ -329,7 +324,7 @@ void VerticalBlockReader::_update_agg_data(MutableColumns& columns) { void VerticalBlockReader::_update_agg_value(MutableColumns& columns, int begin, int end, bool is_close) { - for (size_t idx = 0; idx < _return_columns.size(); ++idx) { + for (size_t idx = 0; idx < _stored_data_columns.size(); ++idx) { AggregateFunctionPtr function = _agg_functions[idx]; AggregateDataPtr place = _agg_places[idx]; auto* column_ptr = _stored_data_columns[idx].get(); @@ -357,7 +352,7 @@ size_t VerticalBlockReader::_copy_agg_data() { auto& ref = _stored_row_ref[i]; _temp_ref_map[ref.block.get()].emplace_back(ref.row_pos, i); } - for (size_t idx = 0; idx < _return_columns.size(); ++idx) { + for (size_t idx = 0; idx < _stored_data_columns.size(); ++idx) { auto& dst_column = _stored_data_columns[idx]; if (_stored_has_variable_length_tag[idx]) { //variable length type should replace ordered @@ -478,14 +473,12 @@ Status VerticalBlockReader::_unique_key_next_block(Block* block, bool* eof) { size_t block_rows = block->rows(); if (_delete_sign_available && block_rows > 0) { - int ori_delete_sign_idx = _reader_context.tablet_schema->field_index(DELETE_SIGN); - if (ori_delete_sign_idx < 0) { + int delete_sign_idx = _read_schema->delete_sign_ordinal(); + if (delete_sign_idx < 0) { *eof = (res.is()); _eof = *eof; return Status::OK(); } - // delete sign column must store in last column of the block - int delete_sign_idx = block->columns() - 1; DCHECK(delete_sign_idx > 0); auto target_columns_guard = block->mutate_columns_scoped(); auto& target_columns = target_columns_guard.mutable_columns(); diff --git a/be/src/storage/iterator/vertical_block_reader.h b/be/src/storage/iterator/vertical_block_reader.h index baba2a215f8d19..3416b15eb77feb 100644 --- a/be/src/storage/iterator/vertical_block_reader.h +++ b/be/src/storage/iterator/vertical_block_reader.h @@ -123,9 +123,6 @@ class VerticalBlockReader final : public TabletReader { std::vector _agg_places; Arena _arena; - std::vector _normal_columns_idx; - std::vector _agg_columns_idx; - std::vector _agg_data_counters; int _last_agg_data_counter = 0; diff --git a/be/src/storage/iterator/vertical_merge_iterator.cpp b/be/src/storage/iterator/vertical_merge_iterator.cpp index 5a0177437f628b..7c0740f06b19e0 100644 --- a/be/src/storage/iterator/vertical_merge_iterator.cpp +++ b/be/src/storage/iterator/vertical_merge_iterator.cpp @@ -329,17 +329,11 @@ void VerticalMergeIteratorContext::_mark_inactive() { Status VerticalMergeIteratorContext::block_reset(const std::shared_ptr& block) { if (!block->columns()) { - const Schema& schema = _iter->schema(); - const auto& column_ids = schema.column_ids(); - for (size_t i = 0; i < schema.num_column_ids(); ++i) { - auto column_desc = schema.column(column_ids[i]); - auto data_type = Schema::get_data_type_ptr(*column_desc); - if (data_type == nullptr) { - return Status::RuntimeError("invalid data type"); - } - auto column = data_type->create_column(); + const ReadSchema& schema = _iter->schema(); + *block = schema.create_read_block(); + auto columns_guard = block->mutate_columns_scoped(); + for (auto& column : columns_guard.mutable_columns()) { column->reserve(_block_row_max); - block->insert(ColumnWithTypeAndName(std::move(column), data_type, column_desc->name())); } } else { block->clear_column_data(); @@ -361,9 +355,8 @@ bool VerticalMergeIteratorContext::compare(const VerticalMergeIteratorContext& r } auto col_cmp_res = 0; if (_seq_col_idx != -1) { - DCHECK(_block->columns() >= _num_key_columns); - auto real_seq_idx = _num_key_columns; - col_cmp_res = _block->compare_column_at(_index_in_block, rhs._index_in_block, real_seq_idx, + DCHECK_LT(static_cast(_seq_col_idx), _block->columns()); + col_cmp_res = _block->compare_column_at(_index_in_block, rhs._index_in_block, _seq_col_idx, *rhs._block, -1); } auto result = (col_cmp_res == 0) ? (_use_insert_order_when_same ? (_order > rhs.order()) @@ -381,7 +374,7 @@ Status VerticalMergeIteratorContext::copy_rows(Block* block, size_t count) { auto start = _index_in_block; _index_in_block += count - 1; RETURN_IF_CATCH_EXCEPTION({ - for (size_t i = 0; i < _ori_return_cols; ++i) { + for (size_t i = 0; i < src.columns(); ++i) { auto& s_col = src.get_by_position(i); auto& d_col = dst.get_by_position(i); @@ -404,7 +397,7 @@ Status VerticalMergeIteratorContext::copy_rows(Block* block, bool advanced) { // copy a row to dst block column by column size_t start = _index_in_block - _cur_batch_num + 1 - advanced; RETURN_IF_CATCH_EXCEPTION({ - for (size_t i = 0; i < _ori_return_cols; ++i) { + for (size_t i = 0; i < src.columns(); ++i) { auto& s_col = src.get_by_position(i); auto& d_col = dst.get_by_position(i); @@ -524,12 +517,6 @@ Status VerticalMergeIteratorContext::_load_next_block() { return st; } } - // erase delete handler columns - if (_block->columns() > _ori_return_cols) { - for (auto i = _block->columns() - 1; i >= _ori_return_cols; --i) { - _block->erase(i); - } - } if (UNLIKELY(_record_rowids)) { RETURN_IF_ERROR(_iter->current_block_row_locations(&_block_row_locations)); for (auto i = 0; i < _block_row_locations.size(); i++) { @@ -640,8 +627,8 @@ Status VerticalHeapMergeIterator::init(const StorageReadOptions& opts, for (size_t seg_order = 0; seg_order < num_iters; ++seg_order) { auto& iter = _origin_iters[seg_order]; auto ctx = std::make_unique( - std::move(iter), _rowset_ids[seg_order], _ori_return_cols, seg_order, _seq_col_idx, - _context_stats, opts.use_insert_order_when_same, _key_group_cluster_key_idxes); + std::move(iter), _rowset_ids[seg_order], seg_order, _seq_col_idx, _context_stats, + opts.use_insert_order_when_same, _key_group_cluster_key_idxes); _ori_iter_ctx.push_back(std::move(ctx)); } _origin_iters.clear(); @@ -707,10 +694,9 @@ Status VerticalFifoMergeIterator::next_batch(Block* block) { cur_order++) { auto& next_iter = _origin_iters[cur_order]; std::unique_ptr next_ctx( - new VerticalMergeIteratorContext(std::move(next_iter), - _rowset_ids[cur_order], _ori_return_cols, - cur_order, _seq_col_idx, _context_stats, - _opts.use_insert_order_when_same)); + new VerticalMergeIteratorContext( + std::move(next_iter), _rowset_ids[cur_order], cur_order, + _seq_col_idx, _context_stats, _opts.use_insert_order_when_same)); RETURN_IF_ERROR(next_ctx->init(_opts)); if (next_ctx->valid()) { _cur_iter_ctx.swap(next_ctx); @@ -749,8 +735,8 @@ Status VerticalFifoMergeIterator::init(const StorageReadOptions& opts, // so this rowset can work in heap for (auto& iter : _origin_iters) { std::unique_ptr ctx(new VerticalMergeIteratorContext( - std::move(iter), _rowset_ids[seg_order], _ori_return_cols, seg_order, _seq_col_idx, - _context_stats, opts.use_insert_order_when_same)); + std::move(iter), _rowset_ids[seg_order], seg_order, _seq_col_idx, _context_stats, + opts.use_insert_order_when_same)); RETURN_IF_ERROR(ctx->init(opts, sample_info)); if (!ctx->valid()) { ++seg_order; @@ -1010,8 +996,8 @@ Status VerticalMaskMergeIterator::init(const StorageReadOptions& opts, RowsetId rs_id; for (auto& iter : _origin_iters) { - auto ctx = std::make_unique( - std::move(iter), rs_id, _ori_return_cols, -1, -1, _context_stats); + auto ctx = std::make_unique(std::move(iter), rs_id, -1, -1, + _context_stats); _origin_iter_ctx.push_back(std::move(ctx)); } _origin_iters.clear(); @@ -1024,30 +1010,28 @@ Status VerticalMaskMergeIterator::init(const StorageReadOptions& opts, // interfaces to create vertical merge iterator std::shared_ptr new_vertical_heap_merge_iterator( std::vector&& inputs, const std::vector& iterator_init_flag, - const std::vector& rowset_ids, size_t ori_return_cols, KeysType keys_type, - uint32_t seq_col_idx, RowSourcesBuffer* row_sources, - VerticalCompactionContextStats* context_stats, + const std::vector& rowset_ids, KeysType keys_type, int32_t seq_col_idx, + RowSourcesBuffer* row_sources, VerticalCompactionContextStats* context_stats, std::vector key_group_cluster_key_idxes) { return std::make_shared( - std::move(inputs), iterator_init_flag, rowset_ids, ori_return_cols, keys_type, - seq_col_idx, row_sources, context_stats, key_group_cluster_key_idxes); + std::move(inputs), iterator_init_flag, rowset_ids, keys_type, seq_col_idx, row_sources, + context_stats, key_group_cluster_key_idxes); } std::shared_ptr new_vertical_fifo_merge_iterator( std::vector&& inputs, const std::vector& iterator_init_flag, - const std::vector& rowset_ids, size_t ori_return_cols, KeysType keys_type, - uint32_t seq_col_idx, RowSourcesBuffer* row_sources, - VerticalCompactionContextStats* context_stats) { + const std::vector& rowset_ids, KeysType keys_type, int32_t seq_col_idx, + RowSourcesBuffer* row_sources, VerticalCompactionContextStats* context_stats) { return std::make_shared(std::move(inputs), iterator_init_flag, - rowset_ids, ori_return_cols, keys_type, - seq_col_idx, row_sources, context_stats); + rowset_ids, keys_type, seq_col_idx, + row_sources, context_stats); } std::shared_ptr new_vertical_mask_merge_iterator( - std::vector&& inputs, size_t ori_return_cols, - RowSourcesBuffer* row_sources, VerticalCompactionContextStats* context_stats) { - return std::make_shared(std::move(inputs), ori_return_cols, - row_sources, context_stats); + std::vector&& inputs, RowSourcesBuffer* row_sources, + VerticalCompactionContextStats* context_stats) { + return std::make_shared(std::move(inputs), row_sources, + context_stats); } } // namespace doris diff --git a/be/src/storage/iterator/vertical_merge_iterator.h b/be/src/storage/iterator/vertical_merge_iterator.h index 8e97f5978f021a..2bc27b19f5bdd9 100644 --- a/be/src/storage/iterator/vertical_merge_iterator.h +++ b/be/src/storage/iterator/vertical_merge_iterator.h @@ -161,14 +161,12 @@ class RowSourcesBuffer { // takes ownership of rowwise iterator class VerticalMergeIteratorContext { public: - VerticalMergeIteratorContext(RowwiseIteratorUPtr&& iter, RowsetId rowset_id, - size_t ori_return_cols, uint32_t order, uint32_t seq_col_idx, - VerticalCompactionContextStats* context_stats, + VerticalMergeIteratorContext(RowwiseIteratorUPtr&& iter, RowsetId rowset_id, uint32_t order, + int32_t seq_col_idx, VerticalCompactionContextStats* context_stats, bool use_insert_order_when_same = false, std::vector key_group_cluster_key_idxes = {}) : _iter(std::move(iter)), _rowset_id(rowset_id), - _ori_return_cols(ori_return_cols), _order(order), _seq_col_idx(seq_col_idx), _num_key_columns(_iter->schema().num_key_columns()), @@ -259,7 +257,6 @@ class VerticalMergeIteratorContext { RowwiseIteratorUPtr _iter; RowsetId _rowset_id; - size_t _ori_return_cols = 0; // segment order, used to compare key const uint32_t _order = 0; @@ -294,15 +291,13 @@ class VerticalHeapMergeIterator : public RowwiseIterator { // VerticalMergeIterator takes the ownership of input iterators VerticalHeapMergeIterator(std::vector&& iters, std::vector iterator_init_flags, - std::vector rowset_ids, size_t ori_return_cols, - KeysType keys_type, int32_t seq_col_idx, - RowSourcesBuffer* row_sources_buf, + std::vector rowset_ids, KeysType keys_type, + int32_t seq_col_idx, RowSourcesBuffer* row_sources_buf, VerticalCompactionContextStats* context_stats, std::vector key_group_cluster_key_idxes) : _origin_iters(std::move(iters)), _iterator_init_flags(std::move(iterator_init_flags)), _rowset_ids(std::move(rowset_ids)), - _ori_return_cols(ori_return_cols), _keys_type(keys_type), _seq_col_idx(seq_col_idx), _row_sources_buf(row_sources_buf), @@ -315,7 +310,7 @@ class VerticalHeapMergeIterator : public RowwiseIterator { Status init(const StorageReadOptions& opts, CompactionSampleInfo* sample_info) override; Status next_batch(Block* block) override; - const Schema& schema() const override { return *_schema; } + const ReadSchema& schema() const override { return *_schema; } uint64_t merged_rows() const override { return _merged_rows; } Status current_block_row_locations(std::vector* block_row_locations) override { DCHECK(_record_rowids); @@ -330,9 +325,8 @@ class VerticalHeapMergeIterator : public RowwiseIterator { std::vector _origin_iters; std::vector _iterator_init_flags; std::vector _rowset_ids; - size_t _ori_return_cols; - const Schema* _schema = nullptr; + const ReadSchema* _schema = nullptr; struct VerticalMergeContextComparator { bool operator()(const VerticalMergeIteratorContext* lhs, @@ -365,14 +359,12 @@ class VerticalFifoMergeIterator : public RowwiseIterator { // VerticalFifoMergeIterator takes the ownership of input iterators VerticalFifoMergeIterator(std::vector&& iters, std::vector iterator_init_flags, - std::vector rowset_ids, size_t ori_return_cols, - KeysType keys_type, int32_t seq_col_idx, - RowSourcesBuffer* row_sources_buf, + std::vector rowset_ids, KeysType keys_type, + int32_t seq_col_idx, RowSourcesBuffer* row_sources_buf, VerticalCompactionContextStats* context_stats) : _origin_iters(std::move(iters)), _iterator_init_flags(std::move(iterator_init_flags)), _rowset_ids(std::move(rowset_ids)), - _ori_return_cols(ori_return_cols), _keys_type(keys_type), _seq_col_idx(seq_col_idx), _row_sources_buf(row_sources_buf), @@ -384,7 +376,7 @@ class VerticalFifoMergeIterator : public RowwiseIterator { Status init(const StorageReadOptions& opts, CompactionSampleInfo* sample_info) override; Status next_batch(Block* block) override; - const Schema& schema() const override { return *_schema; } + const ReadSchema& schema() const override { return *_schema; } uint64_t merged_rows() const override { return _merged_rows; } Status current_block_row_locations(std::vector* block_row_locations) override { DCHECK(_record_rowids); @@ -399,9 +391,8 @@ class VerticalFifoMergeIterator : public RowwiseIterator { std::vector _origin_iters; std::vector _iterator_init_flags; std::vector _rowset_ids; - size_t _ori_return_cols; - const Schema* _schema = nullptr; + const ReadSchema* _schema = nullptr; std::unique_ptr _cur_iter_ctx; int _block_row_max = 0; @@ -430,11 +421,10 @@ struct RowBatch { class VerticalMaskMergeIterator : public RowwiseIterator { public: // VerticalMaskMergeIterator takes the ownership of input iterators - VerticalMaskMergeIterator(std::vector&& iters, size_t ori_return_cols, + VerticalMaskMergeIterator(std::vector&& iters, RowSourcesBuffer* row_sources_buf, VerticalCompactionContextStats* context_stats) : _origin_iters(std::move(iters)), - _ori_return_cols(ori_return_cols), _row_sources_buf(row_sources_buf), _context_stats(context_stats) {} @@ -446,7 +436,7 @@ class VerticalMaskMergeIterator : public RowwiseIterator { Status next_batch(Block* block) override; - const Schema& schema() const override { return *_schema; } + const ReadSchema& schema() const override { return *_schema; } Status next_row(IteratorRowRef* ref) override; @@ -468,11 +458,10 @@ class VerticalMaskMergeIterator : public RowwiseIterator { // released after build ctx std::vector _origin_iters; - size_t _ori_return_cols = 0; std::vector> _origin_iter_ctx; - const Schema* _schema = nullptr; + const ReadSchema* _schema = nullptr; int _block_row_max = 0; size_t _filtered_rows = 0; @@ -485,19 +474,17 @@ class VerticalMaskMergeIterator : public RowwiseIterator { // segment merge iterator std::shared_ptr new_vertical_heap_merge_iterator( std::vector&& inputs, const std::vector& iterator_init_flag, - const std::vector& rowset_ids, size_t _ori_return_cols, KeysType key_type, - uint32_t seq_col_idx, RowSourcesBuffer* row_sources_buf, - VerticalCompactionContextStats* context_stats, + const std::vector& rowset_ids, KeysType key_type, int32_t seq_col_idx, + RowSourcesBuffer* row_sources_buf, VerticalCompactionContextStats* context_stats, std::vector key_group_cluster_key_idxes); std::shared_ptr new_vertical_fifo_merge_iterator( std::vector&& inputs, const std::vector& iterator_init_flag, - const std::vector& rowset_ids, size_t _ori_return_cols, KeysType key_type, - uint32_t seq_col_idx, RowSourcesBuffer* row_sources_buf, - VerticalCompactionContextStats* context_stats); + const std::vector& rowset_ids, KeysType key_type, int32_t seq_col_idx, + RowSourcesBuffer* row_sources_buf, VerticalCompactionContextStats* context_stats); std::shared_ptr new_vertical_mask_merge_iterator( - std::vector&& inputs, size_t ori_return_cols, - RowSourcesBuffer* row_sources_buf, VerticalCompactionContextStats* context_stats); + std::vector&& inputs, RowSourcesBuffer* row_sources_buf, + VerticalCompactionContextStats* context_stats); } // namespace doris diff --git a/be/src/storage/iterator/vgeneric_iterators.cpp b/be/src/storage/iterator/vgeneric_iterators.cpp index 356744ce785fb6..6c93f938ea2bcb 100644 --- a/be/src/storage/iterator/vgeneric_iterators.cpp +++ b/be/src/storage/iterator/vgeneric_iterators.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include "common/status.h" @@ -42,12 +43,12 @@ Status VStatisticsIterator::init(const StorageReadOptions& opts) { if (!_init) { _push_down_agg_type_opt = opts.push_down_agg_type_opt; - for (size_t i = 0; i < _schema.num_column_ids(); i++) { - auto cid = _schema.column_id(i); - auto unique_id = _schema.column(cid)->unique_id(); + for (size_t ordinal = 0; ordinal < _schema.num_block_columns(); ordinal++) { + const auto* column_desc = _schema.column(ordinal); + auto unique_id = column_desc->unique_id(); if (_column_iterators_map.count(unique_id) < 1) { RETURN_IF_ERROR(_segment->new_column_iterator( - opts.tablet_schema->column(cid), &_column_iterators_map[unique_id], &opts)); + *column_desc, &_column_iterators_map[unique_id], &opts)); } _column_iterators.push_back(_column_iterators_map[unique_id].get()); } @@ -60,7 +61,7 @@ Status VStatisticsIterator::init(const StorageReadOptions& opts) { } Status VStatisticsIterator::next_batch(Block* block) { - DCHECK(block->columns() == _column_iterators.size()); + DCHECK_EQ(block->columns(), _column_iterators.size()); if (_output_rows < _target_rows) { block->clear_column_data(); auto columns_guard = block->mutate_columns_scoped(); @@ -76,8 +77,7 @@ Status VStatisticsIterator::next_batch(Block* block) { } else { for (int i = 0; i < columns.size(); ++i) { RETURN_IF_ERROR(_column_iterators[i]->next_batch_of_zone_map(&size, columns[i])); - if (auto cid = _schema.column_id(i); - _schema.column(cid)->type() == FieldType::OLAP_FIELD_TYPE_CHAR) { + if (_schema.column(i)->type() == FieldType::OLAP_FIELD_TYPE_CHAR) { auto col = columns[i]->clone_empty(); for (size_t j = 0; j < columns[i]->size(); ++j) { const auto ref = columns[i]->get_data_at(j).trim_tail_padding_zero(); @@ -93,27 +93,12 @@ Status VStatisticsIterator::next_batch(Block* block) { return Status::EndOfFile("End of VStatisticsIterator"); } -// Build the block using the output schema, which contains only the columns -// the caller requested (return_columns). Delete predicate columns are excluded -// because SegmentIterator handles them independently: -// - _init_current_block() skips predicate columns (including delete predicates) -// via the _is_pred_column[cid] check, so it never accesses the block by those positions. -// - _output_non_pred_columns() checks loc < block->columns() before filling any column, -// so delete predicate columns (whose loc exceeds block->columns()) are simply skipped. -// - Delete predicate evaluation happens entirely through _current_return_columns and -// _evaluate_short_circuit_predicate(), which are independent of the block structure. Status VMergeIteratorContext::block_reset(const std::shared_ptr& block) { if (!block->columns()) { - const auto& column_ids = _output_schema->column_ids(); - for (size_t i = 0; i < _output_schema->num_column_ids(); ++i) { - auto column_desc = _output_schema->column(column_ids[i]); - auto data_type = Schema::get_data_type_ptr(*column_desc); - if (data_type == nullptr) { - return Status::RuntimeError("invalid data type"); - } - auto column = data_type->create_column(); + *block = _read_schema->create_read_block(); + auto columns_guard = block->mutate_columns_scoped(); + for (auto& column : columns_guard.mutable_columns()) { column->reserve(_block_row_max); - block->insert(ColumnWithTypeAndName(std::move(column), data_type, column_desc->name())); } } else { block->clear_column_data(); @@ -152,14 +137,12 @@ bool VMergeIteratorContext::compare(const VMergeIteratorContext& rhs) const { } // Copy rows from the internal _block to the destination block. -// Both blocks are built with the output schema (return_columns only), so they -// have the same number of columns. We iterate over _output_schema->num_column_ids() -// columns to copy from src to dst. +// Both blocks contain the read schema's visible slot prefix. Status VMergeIteratorContext::copy_rows(Block* block, bool advanced) { Block& src = *_block; Block& dst = *block; - DCHECK_EQ(src.columns(), _output_schema->num_column_ids()); - DCHECK_EQ(dst.columns(), _output_schema->num_column_ids()); + DCHECK_EQ(src.columns(), _read_schema->num_block_columns()); + DCHECK_EQ(dst.columns(), _read_schema->num_block_columns()); if (_cur_batch_num == 0) { return Status::OK(); } @@ -168,7 +151,7 @@ Status VMergeIteratorContext::copy_rows(Block* block, bool advanced) { size_t start = _index_in_block - _cur_batch_num + 1 - advanced; RETURN_IF_CATCH_EXCEPTION({ - for (size_t i = 0; i < _output_schema->num_column_ids(); ++i) { + for (size_t i = 0; i < _read_schema->num_block_columns(); ++i) { auto& s_col = src.get_by_position(i); auto& d_col = dst.get_by_position(i); @@ -213,7 +196,7 @@ Status VMergeIteratorContext::copy_rows(BlockView* view, bool advanced) { // (0, 1), (1, 2), (2, 3), (3, 4)... // // Usage: -// Schema schema; +// ReadSchema schema; // VAutoIncrementIterator iter(schema, 1000); // StorageReadOptions opts; // RETURN_IF_ERROR(iter.init(opts)); @@ -224,7 +207,7 @@ Status VMergeIteratorContext::copy_rows(BlockView* view, bool advanced) { class VAutoIncrementIterator : public RowwiseIterator { public: // Will generate num_rows rows in total - VAutoIncrementIterator(const Schema& schema, size_t num_rows) + VAutoIncrementIterator(const ReadSchema& schema, size_t num_rows) : _schema(schema), _num_rows(num_rows), _rows_returned() {} ~VAutoIncrementIterator() override = default; @@ -234,7 +217,7 @@ class VAutoIncrementIterator : public RowwiseIterator { Status next_batch(Block* block) override { int row_idx = 0; while (_rows_returned < _num_rows) { - for (int j = 0; j < _schema.num_columns(); ++j) { + for (int j = 0; j < _schema.num_block_columns(); ++j) { ColumnWithTypeAndName& vc = block->get_by_position(j); IColumn& vi = (IColumn&)(*vc.column); @@ -279,10 +262,10 @@ class VAutoIncrementIterator : public RowwiseIterator { return Status::EndOfFile("End of VAutoIncrementIterator"); } - const Schema& schema() const override { return _schema; } + const ReadSchema& schema() const override { return _schema; } private: - const Schema& _schema; + const ReadSchema& _schema; size_t _num_rows; size_t _rows_returned; }; @@ -304,54 +287,42 @@ Status VMergeIteratorContext::init(const StorageReadOptions& opts) { return Status::OK(); } -// compare() reads block positions that are only DCHECK-bounds-checked in Block::compare_at(), -// so in a release build a projection violating the merge contract turns into an out-of-bounds -// read inside std::push_heap and kills the BE (issue #66390). Verify the contract once the -// first block is loaded and surface a diagnosable error instead: -// - explicit compare columns (_compare_columns) must all point inside the block; -// - otherwise the default comparison touches positions [0, _num_key_columns), where -// _num_key_columns counts the key columns of the WHOLE tablet schema. Key columns always -// occupy column ids [0, num_key_columns) of the tablet schema, so the projection must -// start with exactly those ids, in order, for the positional comparison to be key order; -// - the sequence tie-break column, when present, must point inside the block as well. +// Validate the read-schema ordinals accessed by compare() before building the merge heap. Status VMergeIteratorContext::_validate_compare_contract(const StorageReadOptions& opts) const { const size_t block_columns = _block->columns(); auto contract_error = [&](const std::string& detail) { - std::string projected_ids; - for (auto cid : _output_schema->column_ids()) { - if (!projected_ids.empty()) { - projected_ids += ','; - } - projected_ids += std::to_string(cid); - } return Status::InternalError( "merge iterator compare contract violated: {}, tablet_id={}, rowset_id={}, " - "version={}, block_columns={}, num_key_columns={}, sequence_id_idx={}, " - "projected_column_ids=[{}]", + "version={}, block_columns={}, read_columns={}, num_key_columns={}, " + "sequence_id_idx={}", detail, opts.tablet_id, opts.rowset_id.to_string(), opts.version.to_string(), - block_columns, _num_key_columns, _sequence_id_idx, projected_ids); + block_columns, _read_schema->num_block_columns(), _num_key_columns, + _sequence_id_idx); }; + if (_compare_columns != nullptr) { - for (uint32_t pos : *_compare_columns) { - if (pos >= block_columns) { - return contract_error(fmt::format("compare column position {} out of range", pos)); + for (uint32_t ordinal : *_compare_columns) { + if (ordinal >= block_columns) { + return contract_error("compare column ordinal " + std::to_string(ordinal) + + " out of range"); } } } else { const auto num_key_columns = static_cast(_num_key_columns); - if (num_key_columns > _output_schema->num_column_ids() || num_key_columns > block_columns) { - return contract_error("projection has fewer columns than the key prefix"); + if (num_key_columns > _read_schema->num_block_columns() || + num_key_columns > block_columns) { + return contract_error("read schema has fewer columns than the key prefix"); } - for (size_t i = 0; i < num_key_columns; ++i) { - if (_output_schema->column_ids()[i] != static_cast(i)) { - return contract_error( - fmt::format("position {} holds column id {} instead of key column {}", i, - _output_schema->column_ids()[i], i)); + for (size_t ordinal = 0; ordinal < num_key_columns; ++ordinal) { + if (!_read_schema->column(ordinal)->is_key()) { + return contract_error("read schema ordinal " + std::to_string(ordinal) + + " is not a key column"); } } } + if (_sequence_id_idx != -1 && static_cast(_sequence_id_idx) >= block_columns) { - return contract_error("sequence column position out of range"); + return contract_error("sequence column ordinal out of range"); } return Status::OK(); } @@ -416,7 +387,7 @@ Status VMergeIterator::init(const StorageReadOptions& opts) { for (auto& iter : _origin_iters) { auto ctx = std::make_shared( std::move(iter), _sequence_id_idx, _is_unique, _is_reverse, - opts.use_insert_order_when_same, opts.read_orderby_key_columns, _output_schema, + opts.use_insert_order_when_same, opts.read_orderby_key_columns, _read_schema, _small_seq_first); RETURN_IF_ERROR(ctx->init(opts)); if (!ctx->valid()) { @@ -432,19 +403,16 @@ Status VMergeIterator::init(const StorageReadOptions& opts) { return Status::OK(); } -// VUnionIterator will read data from input iterator one by one. -// Unlike VMergeIterator, VUnionIterator does NOT have its own internal block or copy_rows(). -// It passes the caller's block directly to the underlying SegmentIterator via next_batch(), -// so there is no input-schema vs output-schema mismatch issue here. -// The output_schema parameter is accepted only so that schema() can return the output schema -// consistently with VMergeIterator. +// VUnionIterator reads input iterators one by one. Every input writes the same +// visible slot prefix; a SegmentIterator may retain delete-predicate columns +// privately after that prefix. class VUnionIterator : public RowwiseIterator { public: // Iterators' ownership it transferred to this class. // This class will delete all iterators when destructs // Client should not use iterators anymore. - VUnionIterator(std::vector&& v, SchemaSPtr output_schema) - : _output_schema(std::move(output_schema)), _origin_iters(std::move(v)) {} + VUnionIterator(std::vector&& v, ReadSchemaSPtr read_schema) + : _read_schema(std::move(read_schema)), _origin_iters(std::move(v)) {} ~VUnionIterator() override = default; @@ -452,7 +420,7 @@ class VUnionIterator : public RowwiseIterator { Status next_batch(Block* block) override; - const Schema& schema() const override { return *_output_schema; } + const ReadSchema& schema() const override { return *_read_schema; } Status current_block_row_locations(std::vector* locations) override; @@ -463,7 +431,7 @@ class VUnionIterator : public RowwiseIterator { } private: - const SchemaSPtr _output_schema; + const ReadSchemaSPtr _read_schema; RowwiseIteratorUPtr _cur_iter = nullptr; StorageReadOptions _read_options; std::vector _origin_iters; @@ -512,29 +480,30 @@ Status VUnionIterator::current_block_row_locations(std::vector* loc RowwiseIteratorUPtr new_merge_iterator(std::vector&& inputs, int sequence_id_idx, bool is_unique, bool is_reverse, - uint64_t* merged_rows, SchemaSPtr output_schema, + uint64_t* merged_rows, ReadSchemaSPtr read_schema, bool small_seq_first) { // when the size of inputs is 1, we also need to use VMergeIterator, because the // next_block_view function only be implemented in VMergeIterator. The reason why // the size of inputs is 1 is that the segment was filtered out by zone map or others. return std::make_unique(std::move(inputs), sequence_id_idx, is_unique, - is_reverse, merged_rows, std::move(output_schema), + is_reverse, merged_rows, std::move(read_schema), small_seq_first); } RowwiseIteratorUPtr new_union_iterator(std::vector&& inputs, - SchemaSPtr output_schema) { + ReadSchemaSPtr read_schema) { if (inputs.size() == 1) { return std::move(inputs[0]); } - return std::make_unique(std::move(inputs), std::move(output_schema)); + return std::make_unique(std::move(inputs), std::move(read_schema)); } -RowwiseIterator* new_vstatistics_iterator(std::shared_ptr segment, const Schema& schema) { +RowwiseIterator* new_vstatistics_iterator(std::shared_ptr segment, + const ReadSchema& schema) { return new VStatisticsIterator(segment, schema); } -RowwiseIteratorUPtr new_auto_increment_iterator(const Schema& schema, size_t num_rows) { +RowwiseIteratorUPtr new_auto_increment_iterator(const ReadSchema& schema, size_t num_rows) { return std::make_unique(schema, num_rows); } diff --git a/be/src/storage/iterator/vgeneric_iterators.h b/be/src/storage/iterator/vgeneric_iterators.h index 38eee70cfae23d..220124710674d0 100644 --- a/be/src/storage/iterator/vgeneric_iterators.h +++ b/be/src/storage/iterator/vgeneric_iterators.h @@ -47,7 +47,7 @@ class ColumnIterator; class VStatisticsIterator : public RowwiseIterator { public: // Will generate num_rows rows in total - VStatisticsIterator(std::shared_ptr segment, const Schema& schema) + VStatisticsIterator(std::shared_ptr segment, const ReadSchema& schema) : _segment(std::move(segment)), _schema(schema) {} ~VStatisticsIterator() override = default; @@ -56,11 +56,11 @@ class VStatisticsIterator : public RowwiseIterator { Status next_batch(Block* block) override; - const Schema& schema() const override { return _schema; } + const ReadSchema& schema() const override { return _schema; } private: std::shared_ptr _segment; - const Schema& _schema; + const ReadSchema& _schema; size_t _target_rows = 0; size_t _output_rows = 0; bool _init = false; @@ -85,16 +85,16 @@ class VMergeIteratorContext { public: VMergeIteratorContext(RowwiseIteratorUPtr&& iter, int sequence_id_idx, bool is_unique, bool is_reverse, bool use_insert_order_when_same, - std::vector* read_orderby_key_columns, SchemaSPtr output_schema, - bool small_seq_first = false) + std::vector* read_orderby_key_columns, + ReadSchemaSPtr read_schema, bool small_seq_first = false) : _iter(std::move(iter)), _sequence_id_idx(sequence_id_idx), _is_unique(is_unique), _is_reverse(is_reverse), _use_insert_order_when_same(use_insert_order_when_same), _small_seq_first(small_seq_first), - _output_schema(std::move(output_schema)), - _num_key_columns(cast_set(_output_schema->num_key_columns())), + _read_schema(std::move(read_schema)), + _num_key_columns(cast_set(_read_schema->num_key_columns())), _compare_columns(read_orderby_key_columns) {} VMergeIteratorContext(const VMergeIteratorContext&) = delete; @@ -104,22 +104,7 @@ class VMergeIteratorContext { ~VMergeIteratorContext() = default; - // Reset (or initialize) the internal _block using the output schema. - // - // The output schema contains only the columns the caller requested (return_columns), - // excluding delete predicate columns. For example, if the query reads columns {c1, c2} - // but there is a delete predicate on column c3 (e.g., "DELETE FROM t WHERE c3 = 'foo'"): - // - input schema (iter->schema) = {c1, c2, c3} (3 columns) - // - output schema = {c1, c2} (2 columns) - // - // It is safe to build the block with only the output schema because SegmentIterator - // handles delete predicate columns independently of the block structure: - // - _init_current_block() skips predicate columns (including delete predicates) - // via the _is_pred_column[cid] check, never accessing the block for them. - // - _output_non_pred_columns() checks loc < block->columns() before filling any - // column, so delete predicate columns are simply skipped when the block is smaller. - // - Delete predicate evaluation uses _current_return_columns and - // _evaluate_short_circuit_predicate(), independent of the block. + // Reset (or initialize) the internal block with the visible slot prefix. Status block_reset(const std::shared_ptr& block); // Initialize this context and will prepare data for current_row() @@ -128,8 +113,7 @@ class VMergeIteratorContext { bool compare(const VMergeIteratorContext& rhs) const; // Copy rows from internal _block to the destination block. - // Both blocks have _output_schema columns (return_columns only). - // Only _output_schema->num_column_ids() columns are copied. + // Both blocks have _read_schema->num_block_columns() columns, all of which are copied. // // `advanced = false` when current block finished // when input argument type is block, we do not process same_bit, @@ -177,18 +161,12 @@ class VMergeIteratorContext { void add_cur_batch() { _cur_batch_num++; } - void reset_cur_batch() { _cur_batch_num = 0; } - bool is_cur_block_finished() { return _index_in_block == _block->rows() - 1; } private: // Load next block into _block Status _load_next_block(); - // Validate that every block position compare() may touch actually exists in _block - // and, for the default key-prefix comparison, that the projection really starts with - // the full ordered key prefix. Returns an error instead of letting compare() perform - // an out-of-bounds or semantically wrong positional access (issue #66390). Status _validate_compare_contract(const StorageReadOptions& opts) const; RowwiseIteratorUPtr _iter; @@ -207,14 +185,8 @@ class VMergeIteratorContext { size_t _index_in_block = -1; // 4096 minus 16 + 16 bytes padding that in padding pod array int _block_row_max = 4064; - // The output schema defines which columns are in _block and in the caller's dst block. - // It contains only the requested return_columns, excluding delete predicate columns. - // For example: - // - _iter->schema() (input) = {c1, c2, c3} — c3 for "DELETE WHERE c3='foo'" - // - _output_schema = {c1, c2} — only the requested columns - // block_reset() uses _output_schema to build _block, and copy_rows() iterates over - // _output_schema->num_column_ids() columns to copy from _block to the destination. - const SchemaSPtr _output_schema; + // ReadSchema whose visible slot prefix is shared by every child block and the caller. + const ReadSchemaSPtr _read_schema; int _num_key_columns; std::vector* _compare_columns; std::vector _block_row_locations; @@ -232,10 +204,10 @@ class VMergeIterator : public RowwiseIterator { public: // VMergeIterator takes the ownership of input iterators VMergeIterator(std::vector&& iters, int sequence_id_idx, bool is_unique, - bool is_reverse, uint64_t* merged_rows, SchemaSPtr output_schema, + bool is_reverse, uint64_t* merged_rows, ReadSchemaSPtr read_schema, bool small_seq_first = false) : _origin_iters(std::move(iters)), - _output_schema(std::move(output_schema)), + _read_schema(std::move(read_schema)), _sequence_id_idx(sequence_id_idx), _is_unique(is_unique), _is_reverse(is_reverse), @@ -252,7 +224,7 @@ class VMergeIterator : public RowwiseIterator { } Status next_batch(BlockView* block_view) override { return _next_batch(block_view); } - const Schema& schema() const override { return *_output_schema; } + const ReadSchema& schema() const override { return *_read_schema; } Status current_block_row_locations(std::vector* block_row_locations) override { DCHECK(_record_rowids); @@ -336,9 +308,8 @@ class VMergeIterator : public RowwiseIterator { // It will be released after '_merge_heap' has been built. std::vector _origin_iters; - // The output schema (excludes delete predicate columns). Passed down to each - // VMergeIteratorContext to control how many columns copy_rows() copies. - const SchemaSPtr _output_schema; + // ReadSchema whose visible slot prefix is shared by every input and the output. + const ReadSchemaSPtr _read_schema; struct VMergeContextComparator { bool operator()(const std::shared_ptr& lhs, @@ -372,7 +343,7 @@ class VMergeIterator : public RowwiseIterator { // should delete returned iterator after usage. RowwiseIteratorUPtr new_merge_iterator(std::vector&& inputs, int sequence_id_idx, bool is_unique, bool is_reverse, - uint64_t* merged_rows, SchemaSPtr output_schema, + uint64_t* merged_rows, ReadSchemaSPtr read_schema, bool small_seq_first = false); // Create a union iterator for input iterators. Union iterator will read @@ -380,14 +351,15 @@ RowwiseIteratorUPtr new_merge_iterator(std::vector&& inputs // // Inputs iterators' ownership is taken by created union iterator. RowwiseIteratorUPtr new_union_iterator(std::vector&& inputs, - SchemaSPtr output_schema); + ReadSchemaSPtr read_schema); // Create an auto increment iterator which returns num_rows data in format of schema. // This class aims to be used in unit test. // // Client should delete returned iterator. -RowwiseIteratorUPtr new_auto_increment_iterator(const Schema& schema, size_t num_rows); +RowwiseIteratorUPtr new_auto_increment_iterator(const ReadSchema& schema, size_t num_rows); -RowwiseIterator* new_vstatistics_iterator(std::shared_ptr segment, const Schema& schema); +RowwiseIterator* new_vstatistics_iterator(std::shared_ptr segment, + const ReadSchema& schema); } // namespace doris diff --git a/be/src/storage/iterators.h b/be/src/storage/iterators.h index 28e8fd96222fef..09f8be09cd4499 100644 --- a/be/src/storage/iterators.h +++ b/be/src/storage/iterators.h @@ -37,14 +37,11 @@ namespace doris { -class Schema; +class ReadSchema; class ColumnPredicate; struct IteratorRowRef; -namespace segment_v2 { -struct SubstreamIterator; -} class StorageReadOptions { public: struct KeyRange { @@ -123,7 +120,6 @@ class StorageReadOptions { bool enable_unique_key_merge_on_write = false; bool record_rowids = false; std::vector topn_filter_source_node_ids; - int topn_filter_target_node_id = -1; // used for special optimization for query : ORDER BY key DESC LIMIT n bool read_orderby_key_reverse = false; // For rows with the same key, use ascending order (small-to-large) for tie-breakers. @@ -166,10 +162,6 @@ class StorageReadOptions { std::shared_ptr score_runtime; CollectionStatisticsPtr collection_statistics; - // Cache for sparse column data to avoid redundant reads - // col_unique_id -> cached column_ptr - std::unordered_map sparse_column_cache; - uint64_t condition_cache_digest = 0; }; @@ -245,7 +237,7 @@ class RowwiseIterator { } // return schema for this Iterator - virtual const Schema& schema() const = 0; + virtual const ReadSchema& schema() const = 0; // Return the data id such as segment id, used for keep the insert order when do // merge sort in priority queue diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp index d986b367288853..e9f184ec57049a 100644 --- a/be/src/storage/merger.cpp +++ b/be/src/storage/merger.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -50,6 +49,7 @@ #include "storage/rowset/rowset.h" #include "storage/rowset/rowset_meta.h" #include "storage/rowset/rowset_writer.h" +#include "storage/schema.h" #include "storage/segment/segment.h" #include "storage/segment/segment_writer.h" #include "storage/storage_engine.h" @@ -88,14 +88,8 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, reader_params.version = dst_rowset_writer->version(); - TabletSchemaSPtr merge_tablet_schema = std::make_shared(); - merge_tablet_schema->copy_from(cur_tablet_schema); - - // Merge the columns in delete predicate that not in latest schema in to current tablet schema - for (auto& del_pred_rs : reader_params.delete_predicates) { - merge_tablet_schema->merge_dropped_columns(*del_pred_rs->tablet_schema()); - } - reader_params.tablet_schema = merge_tablet_schema; + reader_params.tablet_schema = std::make_shared(); + reader_params.tablet_schema->copy_from(cur_tablet_schema); if (!tablet->tablet_schema()->cluster_key_uids().empty()) { reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); } @@ -109,12 +103,10 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, stats_output->rowid_conversion->set_dst_rowset_id(dst_rowset_writer->rowset_id()); } - reader_params.return_columns.resize(cur_tablet_schema.num_columns()); - std::iota(reader_params.return_columns.begin(), reader_params.return_columns.end(), 0); - reader_params.origin_return_columns = &reader_params.return_columns; + reader_params.read_schema = std::make_shared(cur_tablet_schema.columns()); RETURN_IF_ERROR(reader.init(reader_params)); - Block block = cur_tablet_schema.create_block(reader_params.return_columns); + Block block = reader_params.read_schema->create_read_block(); size_t output_rows = 0; bool eof = false; while (!eof && !ExecEnv::GetInstance()->storage_engine().stopped()) { @@ -277,14 +269,8 @@ Status Merger::vertical_compact_one_group( reader_params.version = dst_rowset_writer->version(); - TabletSchemaSPtr merge_tablet_schema = std::make_shared(); - merge_tablet_schema->copy_from(tablet_schema); - - for (auto& del_pred_rs : reader_params.delete_predicates) { - merge_tablet_schema->merge_dropped_columns(*del_pred_rs->tablet_schema()); - } - - reader_params.tablet_schema = merge_tablet_schema; + reader_params.tablet_schema = std::make_shared(); + reader_params.tablet_schema->copy_from(tablet_schema); bool has_cluster_key = false; if (!tablet->tablet_schema()->cluster_key_uids().empty()) { reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); @@ -300,12 +286,11 @@ Status Merger::vertical_compact_one_group( stats_output->rowid_conversion->set_dst_rowset_id(dst_rowset_writer->rowset_id()); } - reader_params.return_columns = column_group; - reader_params.origin_return_columns = &reader_params.return_columns; + reader_params.read_schema = std::make_shared(tablet_schema.columns(), column_group); reader_params.batch_size = batch_size; RETURN_IF_ERROR(reader.init(reader_params, sample_info)); - Block block = tablet_schema.create_block(reader_params.return_columns); + Block block = reader_params.read_schema->create_read_block(); size_t output_rows = 0; bool eof = false; while (!eof && !ExecEnv::GetInstance()->storage_engine().stopped()) { @@ -363,13 +348,12 @@ Status Merger::vertical_compact_one_group( // for segcompaction Status Merger::vertical_compact_one_group( - int64_t tablet_id, ReaderType reader_type, const TabletSchema& tablet_schema, bool is_key, - const std::vector& column_group, RowSourcesBuffer* row_source_buf, - VerticalBlockReader& src_block_reader, segment_v2::SegmentWriter& dst_segment_writer, - Statistics* stats_output, uint64_t* index_size, KeyBoundsPB& key_bounds, - SimpleRowIdConversion* rowid_conversion) { + int64_t tablet_id, ReaderType reader_type, const ReadSchema& read_schema, bool is_key, + RowSourcesBuffer* row_source_buf, VerticalBlockReader& src_block_reader, + segment_v2::SegmentWriter& dst_segment_writer, Statistics* stats_output, + uint64_t* index_size, KeyBoundsPB& key_bounds, SimpleRowIdConversion* rowid_conversion) { // TODO: record_rowids - Block block = tablet_schema.create_block(column_group); + Block block = read_schema.create_read_block(); size_t output_rows = 0; bool eof = false; while (!eof && !ExecEnv::GetInstance()->storage_engine().stopped()) { diff --git a/be/src/storage/merger.h b/be/src/storage/merger.h index 5185c97e6b0dc9..c6758c3d47602b 100644 --- a/be/src/storage/merger.h +++ b/be/src/storage/merger.h @@ -37,6 +37,7 @@ class SegmentWriter; } // namespace segment_v2 class RowSourcesBuffer; +class ReadSchema; class VerticalBlockReader; struct VerticalCompactionContextStats; @@ -90,8 +91,7 @@ class Merger { // for segcompaction static Status vertical_compact_one_group( - int64_t tablet_id, ReaderType reader_type, const TabletSchema& tablet_schema, - bool is_key, const std::vector& column_group, + int64_t tablet_id, ReaderType reader_type, const ReadSchema& read_schema, bool is_key, RowSourcesBuffer* row_source_buf, VerticalBlockReader& src_block_reader, segment_v2::SegmentWriter& dst_segment_writer, Statistics* stats_output, uint64_t* index_size, KeyBoundsPB& key_bounds, SimpleRowIdConversion* rowid_conversion); diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 12ca1d2e6a458f..8b631fd6e9b205 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -457,7 +457,6 @@ struct OlapReaderStatistics { int64_t tablet_reader_init_timer_ns = 0; int64_t tablet_reader_capture_rs_readers_timer_ns = 0; - int64_t tablet_reader_init_return_columns_timer_ns = 0; int64_t tablet_reader_init_keys_param_timer_ns = 0; int64_t tablet_reader_init_orderby_keys_param_timer_ns = 0; int64_t tablet_reader_init_conditions_param_timer_ns = 0; @@ -472,7 +471,7 @@ struct OlapReaderStatistics { int64_t rowset_reader_load_segments_timer_ns = 0; int64_t segment_iterator_init_timer_ns = 0; - int64_t segment_iterator_init_return_column_iterators_timer_ns = 0; + int64_t segment_iterator_init_column_iterators_timer_ns = 0; int64_t segment_iterator_init_index_iterators_timer_ns = 0; int64_t segment_iterator_init_segment_prefetchers_timer_ns = 0; diff --git a/be/src/storage/partial_update_info.cpp b/be/src/storage/partial_update_info.cpp index 1bea6e05cb75f5..e2dcc96cbfd6fb 100644 --- a/be/src/storage/partial_update_info.cpp +++ b/be/src/storage/partial_update_info.cpp @@ -336,7 +336,7 @@ Status FixedReadPlan::read_columns_by_plan( if (block.get_position_by_name(DELETE_SIGN) == -1) { auto del_col_cid = tablet_schema.field_index(DELETE_SIGN); cids_to_read.emplace_back(del_col_cid); - block.swap(tablet_schema.create_block_by_cids(cids_to_read)); + block.swap(tablet_schema.create_storage_block(cids_to_read)); } } bool has_row_column = tablet_schema.has_row_store_for_all_columns(); @@ -429,7 +429,7 @@ Status FixedReadPlan::fill_missing_columns( tablet_schema.sequence_col_idx()) != including_cids.cend()); } - auto old_value_block = tablet_schema.create_block_by_cids(missing_cids); + auto old_value_block = tablet_schema.create_storage_block(missing_cids); CHECK_EQ(missing_cids.size(), old_value_block.columns()); // segment pos to write -> rowid to read in old_value_block @@ -603,7 +603,7 @@ Status FlexibleReadPlan::fill_non_primary_key_columns( // missing_cids are all non sort key columns' cids const auto& non_sort_key_cids = historical_context.partial_update_info->missing_cids; - auto old_value_block = tablet_schema.create_block_by_cids(non_sort_key_cids); + auto old_value_block = tablet_schema.create_storage_block(non_sort_key_cids); CHECK_EQ(non_sort_key_cids.size(), old_value_block.columns()); if (!use_row_store) { @@ -1002,7 +1002,7 @@ Status BlockAggregator::aggregate_for_sequence_column( ->get_data(); const auto* delete_signs = BaseTablet::get_delete_sign_column_data(*block, num_rows); - auto filtered_block = _tablet_schema.create_block(); + auto filtered_block = _tablet_schema.create_storage_block(); MutableBlock output_block = MutableBlock::build_mutable_block(std::move(filtered_block)); int same_key_rows {0}; @@ -1039,8 +1039,8 @@ Status BlockAggregator::fill_sequence_column(Block* block, size_t num_rows, std::vector cids {static_cast(_tablet_schema.sequence_col_idx())}; auto seq_col_unique_id = _tablet_schema.column(_tablet_schema.sequence_col_idx()).unique_id(); - auto seq_col_block = _tablet_schema.create_block_by_cids(cids); - auto tmp_block = _tablet_schema.create_block_by_cids(cids); + auto seq_col_block = _tablet_schema.create_storage_block(cids); + auto tmp_block = _tablet_schema.create_storage_block(cids); std::map read_index; RETURN_IF_ERROR(read_plan.read_columns_by_plan(_tablet_schema, cids, _writer._rsid_to_rowset, seq_col_block, &read_index, false)); diff --git a/be/src/storage/predicate/column_predicate.h b/be/src/storage/predicate/column_predicate.h index 2bcc8359464ce9..d974e2a61af64c 100644 --- a/be/src/storage/predicate/column_predicate.h +++ b/be/src/storage/predicate/column_predicate.h @@ -51,6 +51,7 @@ enum class PredicateType { IS_NOT_NULL = 10, BF = 11, // BloomFilter MATCH = 13, // fulltext match + LIKE = 14, }; template @@ -115,6 +116,9 @@ inline std::string type_to_string(PredicateType type) { case PredicateType::BF: return "BF"; + + case PredicateType::LIKE: + return "LIKE"; default: return ""; }; @@ -382,6 +386,8 @@ class ColumnPredicate : public std::enable_shared_from_this { return "bf"; case PredicateType::MATCH: return "match"; + case PredicateType::LIKE: + return "like"; default: return "unknown"; } diff --git a/be/src/storage/predicate/like_column_predicate.h b/be/src/storage/predicate/like_column_predicate.h index 9cd74f8a9adcb4..d483c2b336b380 100644 --- a/be/src/storage/predicate/like_column_predicate.h +++ b/be/src/storage/predicate/like_column_predicate.h @@ -55,6 +55,7 @@ class LikeColumnPredicate final : public ColumnPredicate { pattern = other.pattern; _state = other._state; _opposite = other._opposite; + THROW_IF_ERROR(_state->search_state.clone(_like_state)); } LikeColumnPredicate(const LikeColumnPredicate& other) = delete; std::shared_ptr clone(uint32_t col_id) const override { @@ -67,7 +68,7 @@ class LikeColumnPredicate final : public ColumnPredicate { return fmt::to_string(debug_string_buffer); } - PredicateType type() const override { return PredicateType::EQ; } + PredicateType type() const override { return PredicateType::LIKE; } void evaluate_vec(const IColumn& column, uint16_t size, bool* flags) const override; void evaluate_and_vec(const IColumn& column, uint16_t size, bool* flags) const override; diff --git a/be/src/storage/row_cursor.cpp b/be/src/storage/row_cursor.cpp index 7194718afc6b5e..b2cabb70fa1c7f 100644 --- a/be/src/storage/row_cursor.cpp +++ b/be/src/storage/row_cursor.cpp @@ -43,9 +43,9 @@ RowCursor::RowCursor(RowCursor&&) noexcept = default; RowCursor& RowCursor::operator=(RowCursor&&) noexcept = default; void RowCursor::_init_schema(TabletSchemaSPtr schema, uint32_t column_count) { - std::vector columns(column_count); - std::iota(columns.begin(), columns.end(), 0); - _schema.reset(new Schema(schema->columns(), columns)); + std::vector columns(schema->columns().begin(), + schema->columns().begin() + column_count); + _schema.reset(new ReadSchema(std::move(columns))); } Status RowCursor::init(TabletSchemaSPtr schema, const OlapTuple& tuple) { @@ -74,10 +74,10 @@ Status RowCursor::init_scan_key(TabletSchemaSPtr schema, std::vector fiel } Status RowCursor::_from_tuple(const OlapTuple& tuple) { - if (tuple.size() != _schema->num_column_ids()) { + if (tuple.size() != _schema->num_read_columns()) { return Status::Error( "column count does not match. tuple_size={}, field_count={}", tuple.size(), - _schema->num_column_ids()); + _schema->num_read_columns()); } _fields.resize(tuple.size()); for (size_t i = 0; i < tuple.size(); ++i) { @@ -88,7 +88,7 @@ Status RowCursor::_from_tuple(const OlapTuple& tuple) { RowCursor RowCursor::clone() const { RowCursor result; - result._schema = std::make_unique(*_schema); + result._schema = std::make_unique(*_schema); result._fields = _fields; return result; } @@ -174,8 +174,7 @@ template void RowCursor::encode_key_with_padding(std::string* buf, size_t num_keys, bool padding_minimal) const { for (uint32_t cid = 0; cid < num_keys; cid++) { - auto* column = _schema->column(cid); - if (column == nullptr) { + if (cid >= _schema->num_read_columns()) { if (padding_minimal) { buf->push_back(KeyConsts::KEY_MINIMAL_MARKER); } else { @@ -194,7 +193,7 @@ void RowCursor::encode_key_with_padding(std::string* buf, size_t num_keys, } buf->push_back(KeyConsts::KEY_NORMAL_MARKER); - _encode_column_value(column, _fields[cid], is_mow, buf); + _encode_column_value(_schema->column(cid), _fields[cid], is_mow, buf); } } diff --git a/be/src/storage/row_cursor.h b/be/src/storage/row_cursor.h index f0f484d3a7018c..94d4eec63ccc71 100644 --- a/be/src/storage/row_cursor.h +++ b/be/src/storage/row_cursor.h @@ -59,7 +59,7 @@ class RowCursor { size_t field_count() const { return _fields.size(); } const TabletColumn* column(uint32_t cid) const { return _schema->column(cid); } - const Schema* schema() const { return _schema.get(); } + const ReadSchema* schema() const { return _schema.get(); } // Returns a deep copy of this RowCursor with the same schema and field values. RowCursor clone() const; @@ -97,7 +97,7 @@ class RowCursor { void _encode_column_value(const TabletColumn* column, const Field& value, bool full_encode, std::string* buf) const; - std::unique_ptr _schema; + std::unique_ptr _schema; std::vector _fields; }; } // namespace doris diff --git a/be/src/storage/rowset/beta_rowset_reader.cpp b/be/src/storage/rowset/beta_rowset_reader.cpp index 43fe44fd7a1851..73d24350abbb31 100644 --- a/be/src/storage/rowset/beta_rowset_reader.cpp +++ b/be/src/storage/rowset/beta_rowset_reader.cpp @@ -25,13 +25,11 @@ #include #include #include -#include #include #include "common/logging.h" #include "common/status.h" #include "core/block/block.h" -#include "core/data_type/data_type_number.h" #include "io/io_common.h" #include "runtime/descriptors.h" #include "runtime/query_context.h" @@ -42,7 +40,6 @@ #include "storage/olap_define.h" #include "storage/predicate/block_column_predicate.h" #include "storage/predicate/column_predicate.h" -#include "storage/predicate/predicate_creator.h" #include "storage/row_cursor.h" #include "storage/rowset/rowset_meta.h" #include "storage/rowset/rowset_reader_context.h" @@ -126,45 +123,20 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context } } - // delete_hanlder is always set, but it maybe not init, so that it will return empty conditions - // or predicates when it is not inited. + const auto& read_schema = _read_context->read_schema; + std::set delete_columns; if (_read_context->delete_handler != nullptr) { _read_context->delete_handler->get_delete_conditions_after_version( _rowset->end_version(), _read_options.delete_condition_predicates.get(), &_read_options.del_predicates_for_zone_map); + _read_options.delete_condition_predicates->get_all_column_ids(delete_columns); } - - std::vector read_columns; - std::set read_columns_set; - std::set delete_columns_set; - for (int i = 0; i < _read_context->return_columns->size(); ++i) { - read_columns.push_back(_read_context->return_columns->at(i)); - read_columns_set.insert(_read_context->return_columns->at(i)); - } - _read_options.delete_condition_predicates->get_all_column_ids(delete_columns_set); - for (auto cid : delete_columns_set) { - if (read_columns_set.find(cid) == read_columns_set.end()) { - read_columns.push_back(cid); - } - } - if (_read_context->tso_predicate_column_id.has_value() && - read_columns_set.find(*_read_context->tso_predicate_column_id) == read_columns_set.end()) { - read_columns.push_back(*_read_context->tso_predicate_column_id); - } - // disable condition cache if you have delete condition or forced pushed tso predicate + // Disable condition cache if you have delete condition. _read_context->condition_cache_digest = - (delete_columns_set.empty() && !_read_context->tso_predicate_column_id.has_value()) - ? _read_context->condition_cache_digest - : 0; + delete_columns.empty() ? _read_context->condition_cache_digest : 0; // create segment iterators - VLOG_NOTICE << "read columns size: " << read_columns.size(); - _input_schema = std::make_shared(_read_context->tablet_schema->columns(), read_columns); + VLOG_NOTICE << "read columns size: " << read_schema->num_read_columns(); _read_options.extra_columns = _read_context->extra_columns; - // output_schema only contains return_columns (excludes extra columns like delete-predicate - // columns and the TSO predicate-only column). - // It is used by merge/union iterators to determine how many columns to copy to the output block. - _output_schema = std::make_shared(_read_context->tablet_schema->columns(), - *(_read_context->return_columns)); if (_read_context->predicates != nullptr) { _read_options.column_predicates.insert(_read_options.column_predicates.end(), _read_context->predicates->begin(), @@ -179,38 +151,6 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context } } - // Forced TSO range pushdown for binlog/snapshot incremental read. Build the (start_tso, - // end_tso] comparison predicates here and inject them directly, so they always reach the - // SegmentIterator for row-level filtering instead of going through the value/key predicate - // split in TabletReader::_init_conditions_param (where they may be dropped). This is a - // correctness requirement for MIN_DELTA, which groups consecutive same-key rows and would - // produce wrong results if out-of-range rows leaked into a group. - if (_read_context->tso_predicate_column_id.has_value() && - (_read_context->start_tso.has_value() || _read_context->end_tso.has_value())) { - ColumnId tso_cid = *_read_context->tso_predicate_column_id; - auto tso_data_type = std::make_shared(); - const std::string& tso_col_name = _read_context->tablet_schema->column(tso_cid).name(); - auto add_tso_predicate = [&](std::shared_ptr pred) { - if (_read_options.col_id_to_predicates.count(pred->column_id()) < 1) { - _read_options.col_id_to_predicates.insert( - {pred->column_id(), AndBlockColumnPredicate::create_shared()}); - } - _read_options.col_id_to_predicates[pred->column_id()]->add_column_predicate( - SingleColumnBlockPredicate::create_unique(pred)); - _read_options.column_predicates.push_back(std::move(pred)); - }; - if (_read_context->start_tso.has_value()) { - add_tso_predicate(create_comparison_predicate( - tso_cid, tso_col_name, tso_data_type, - Field::create_field(*_read_context->start_tso), false)); - } - if (_read_context->end_tso.has_value()) { - add_tso_predicate(create_comparison_predicate( - tso_cid, tso_col_name, tso_data_type, - Field::create_field(*_read_context->end_tso), false)); - } - } - // Take a delete-bitmap for each segment, the bitmap contains all deletes // until the max read version, which is read_context->version.second if (_read_context->delete_bitmap != nullptr) { @@ -253,18 +193,13 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context _read_context->enable_unique_key_merge_on_write; _read_options.record_rowids = _read_context->record_rowids; _read_options.topn_filter_source_node_ids = _read_context->topn_filter_source_node_ids; - _read_options.topn_filter_target_node_id = _read_context->topn_filter_target_node_id; _read_options.read_orderby_key_reverse = _read_context->read_orderby_key_reverse; _read_options.use_insert_order_when_same = _read_context->use_insert_order_when_same; _read_options.read_row_binlog = _read_context->read_row_binlog; - int32_t tso_col_id = _read_context->tablet_schema->binlog_tso_col_idx(); - if (tso_col_id >= 0) { - for (size_t i = 0; i < _read_context->return_columns->size(); ++i) { - if (_read_context->return_columns->at(i) == static_cast(tso_col_id)) { - _read_options.binlog_tso_idx = static_cast(i); - break; - } - } + // Binlog TSO drives the merge tie-break and is therefore part of the exact + // read schema shared by the rowset merge. + if (int32_t tso_ordinal = read_schema->tso_ordinal(); tso_ordinal >= 0) { + _read_options.binlog_tso_idx = tso_ordinal; } _read_options.read_orderby_key_columns = _read_context->read_orderby_key_columns; _read_options.io_ctx.reader_type = _read_context->reader_type; @@ -335,7 +270,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context if (_segment_row_ranges.empty()) { _read_options.row_ranges.clear(); iter = std::make_unique(_rowset, i, should_use_cache, - _input_schema, _read_options); + read_schema, _read_options); } else { DCHECK_EQ(seg_end - seg_start, _segment_row_ranges.size()); auto local_options = _read_options; @@ -345,7 +280,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context local_options.row_ranges.get_digest(local_options.condition_cache_digest); } iter = std::make_unique(_rowset, i, should_use_cache, - _input_schema, local_options); + read_schema, local_options); } if (iter->empty()) { @@ -380,28 +315,22 @@ Status BetaRowsetReader::_init_iterator() { } // merge or union segment iterator if (is_merge_iterator()) { - auto sequence_loc = -1; - if (_read_context->sequence_id_idx != -1) { - for (int loc = 0; loc < _read_context->return_columns->size(); loc++) { - if (_read_context->return_columns->at(loc) == _read_context->sequence_id_idx) { - sequence_loc = loc; - break; - } - } - } + int32_t sequence_loc = _read_context->use_sequence_column_for_merge_order + ? _read_context->read_schema->sequence_ordinal() + : -1; if (_read_options.binlog_tso_idx != -1) { sequence_loc = _read_options.binlog_tso_idx; } _iterator = new_merge_iterator(std::move(iterators), sequence_loc, _read_context->is_unique, _read_context->read_orderby_key_reverse, - _read_context->merged_rows, _output_schema, + _read_context->merged_rows, _read_context->read_schema, _read_options.binlog_tso_idx != -1); } else { if (_read_context->read_orderby_key_reverse) { // reverse iterators to read backward for ORDER BY key DESC std::reverse(iterators.begin(), iterators.end()); } - _iterator = new_union_iterator(std::move(iterators), _output_schema); + _iterator = new_union_iterator(std::move(iterators), _read_context->read_schema); } auto s = _iterator->init(_read_options); @@ -420,7 +349,7 @@ bool BetaRowsetReader::_should_push_down_value_predicates() const { return _rowset->keys_type() == UNIQUE_KEYS && (((_rowset->start_version() == 0 || _rowset->start_version() == 2) && !_rowset->_rowset_meta->is_segments_overlapping() && - _read_context->sequence_id_idx == -1) || + _read_context->read_schema->sequence_ordinal() == -1) || _read_context->enable_unique_key_merge_on_write || _read_context->enable_mor_value_predicate_pushdown); } diff --git a/be/src/storage/rowset/beta_rowset_reader.h b/be/src/storage/rowset/beta_rowset_reader.h index d06ac71c934f4a..c1e2d96224b6c3 100644 --- a/be/src/storage/rowset/beta_rowset_reader.h +++ b/be/src/storage/rowset/beta_rowset_reader.h @@ -37,7 +37,7 @@ namespace doris { class RuntimeProfile; -class Schema; +class ReadSchema; struct RowLocation; struct RowsetReaderContext; @@ -73,6 +73,8 @@ class BetaRowsetReader : public RowsetReader { RowsetSharedPtr rowset() override { return std::dynamic_pointer_cast(_rowset); } + const ReadSchema& read_schema() const override { return *_read_context->read_schema; } + // Return the total number of filtered rows, will be used for validation of schema change int64_t filtered_rows() override { return _stats->rows_del_filtered + _stats->rows_del_by_bitmap + @@ -145,17 +147,6 @@ class BetaRowsetReader : public RowsetReader { std::pair _segment_offsets; std::vector _segment_row_ranges; - // _input_schema: includes return_columns + delete_predicate_columns. - // Used by SegmentIterator internally (iter->schema() returns this). SegmentIterator - // handles the extra delete predicate columns through _current_return_columns and - // _evaluate_short_circuit_predicate(), independent of the block structure. - // e.g. return_columns={c1, c2}, delete_pred on c3 => input_schema={c1, c2, c3} - SchemaSPtr _input_schema; - // _output_schema: includes only return_columns (a subset of input_schema). - // Passed to VMergeIterator/VUnionIterator. block_reset() builds the internal block - // with this schema, and copy_rows() copies exactly these columns to the destination. - // e.g. return_columns={c1, c2} => output_schema={c1, c2} - SchemaSPtr _output_schema; RowsetReaderContext* _read_context = nullptr; BetaRowsetSharedPtr _rowset; diff --git a/be/src/storage/rowset/rowset_reader.h b/be/src/storage/rowset/rowset_reader.h index ee76df2d65e007..559aec4a52b87b 100644 --- a/be/src/storage/rowset/rowset_reader.h +++ b/be/src/storage/rowset/rowset_reader.h @@ -70,6 +70,9 @@ class RowsetReader { virtual RowsetSharedPtr rowset() = 0; + // Exact schema of the caller-visible Block produced by next_batch(). + virtual const ReadSchema& read_schema() const = 0; + virtual int64_t filtered_rows() = 0; virtual uint64_t merged_rows() = 0; diff --git a/be/src/storage/rowset/rowset_reader_context.h b/be/src/storage/rowset/rowset_reader_context.h index d944490fe038af..90f522c99ad17a 100644 --- a/be/src/storage/rowset/rowset_reader_context.h +++ b/be/src/storage/rowset/rowset_reader_context.h @@ -18,7 +18,6 @@ #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H -#include #include #include @@ -31,6 +30,7 @@ #include "storage/olap_common.h" #include "storage/predicate/column_predicate.h" #include "storage/rowid_conversion.h" +#include "storage/schema.h" namespace doris { @@ -45,7 +45,6 @@ struct RowsetReaderContext { Version version {-1, -1}; TabletSchemaSPtr tablet_schema = nullptr; std::vector topn_filter_source_node_ids; - int topn_filter_target_node_id = -1; // whether rowset should return ordered rows. bool need_ordered_result = true; // used for special optimization for query : ORDER BY key DESC LIMIT n @@ -54,18 +53,11 @@ struct RowsetReaderContext { // For example, use lower rowset version / segment id first. bool use_insert_order_when_same = false; bool force_key_ordered_read = false; - // columns for orderby keys + // ordinals (positions in read_schema) of the orderby key columns std::vector* read_orderby_key_columns = nullptr; // limit of rows for read_orderby_key size_t read_orderby_key_limit = 0; - // projection columns: the set of columns rowset reader should return - const std::vector* return_columns = nullptr; - // TSO predicate column that is absent from return_columns but must be read by storage. - std::optional tso_predicate_column_id; - // Binlog/snapshot incremental read TSO range (start_tso, end_tso]. BetaRowsetReader builds - // the tso comparison predicates from this range and forces them onto read options. - std::optional start_tso; - std::optional end_tso; + ReadSchemaSPtr read_schema = nullptr; TPushAggOp::type push_down_agg_type_opt = TPushAggOp::NONE; // column name -> column predicate // adding column_name for predicate to make use of column selectivity @@ -81,7 +73,8 @@ struct RowsetReaderContext { RuntimeState* runtime_state = nullptr; VExprContextSPtrs common_expr_ctxs_push_down; bool use_page_cache = false; - int sequence_id_idx = -1; + // Whether equal-key merge should use the sequence column as its tie-break column. + bool use_sequence_column_for_merge_order = true; int batch_size = 1024; // Effective adaptive batch size byte budget. 0 means disabled internally. size_t preferred_block_size_bytes = 8388608UL; @@ -95,6 +88,7 @@ struct RowsetReaderContext { bool record_rowids = false; RowIdConversion* rowid_conversion = nullptr; bool is_key_column_group = false; + // column unique ids referenced by output/orderby exprs (NOT ordinals) const std::set* output_columns = nullptr; std::set extra_columns; RowsetId rowset_id; diff --git a/be/src/storage/rowset/segcompaction.cpp b/be/src/storage/rowset/segcompaction.cpp index 0606bae3246204..16d8df7e37a031 100644 --- a/be/src/storage/rowset/segcompaction.cpp +++ b/be/src/storage/rowset/segcompaction.cpp @@ -84,9 +84,8 @@ void SegcompactionWorker::init_mem_tracker(const RowsetWriterContext& rowset_wri Status SegcompactionWorker::_get_segcompaction_reader( SegCompactionCandidatesSharedPtr segments, TabletSharedPtr tablet, - std::shared_ptr schema, OlapReaderStatistics* stat, - RowSourcesBuffer& row_sources_buf, bool is_key, std::vector& return_columns, - std::vector& key_group_cluster_key_idxes, + ReadSchemaSPtr read_schema, OlapReaderStatistics* stat, RowSourcesBuffer& row_sources_buf, + bool is_key, std::vector& key_group_cluster_key_idxes, std::unique_ptr* reader) { const auto& ctx = _writer->_context; bool record_rowids = need_convert_delete_bitmap() && is_key; @@ -112,7 +111,7 @@ Status SegcompactionWorker::_get_segcompaction_reader( std::map segment_rows; for (auto& seg_ptr : *segments) { std::unique_ptr iter; - auto s = seg_ptr->new_iterator(schema, read_options, &iter); + auto s = seg_ptr->new_iterator(read_schema, read_options, &iter); if (!s.ok()) { return Status::Error("failed to create iterator[{}]: {}", seg_ptr->id(), s.to_string()); @@ -132,7 +131,7 @@ Status SegcompactionWorker::_get_segcompaction_reader( // no reader_params.version shouldn't break segcompaction reader_params.tablet_schema = ctx.tablet_schema; reader_params.tablet = tablet; - reader_params.return_columns = return_columns; + reader_params.read_schema = std::move(read_schema); reader_params.is_key_column_group = is_key; reader_params.use_page_cache = false; reader_params.record_rowids = record_rowids; @@ -304,12 +303,11 @@ Status SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt writer->clear(); RETURN_IF_ERROR(writer->init(column_ids, is_key)); - auto schema = std::make_shared(ctx.tablet_schema->columns(), column_ids); + auto schema = std::make_shared(ctx.tablet_schema->columns(), column_ids); OlapReaderStatistics reader_stats; std::unique_ptr reader; - auto s = - _get_segcompaction_reader(segments, tablet, schema, &reader_stats, row_sources_buf, - is_key, column_ids, key_group_cluster_key_idxes, &reader); + auto s = _get_segcompaction_reader(segments, tablet, schema, &reader_stats, row_sources_buf, + is_key, key_group_cluster_key_idxes, &reader); if (UNLIKELY(reader == nullptr || !s.ok())) { return Status::Error( "failed to get segcompaction reader. err: {}", s.to_string()); @@ -317,9 +315,9 @@ Status SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt Merger::Statistics merger_stats; RETURN_IF_ERROR(Merger::vertical_compact_one_group( - tablet->tablet_id(), ReaderType::READER_SEGMENT_COMPACTION, *ctx.tablet_schema, - is_key, column_ids, &row_sources_buf, *reader, *writer, &merger_stats, &index_size, - key_bounds, _rowid_conversion.get())); + tablet->tablet_id(), ReaderType::READER_SEGMENT_COMPACTION, *schema, is_key, + &row_sources_buf, *reader, *writer, &merger_stats, &index_size, key_bounds, + _rowid_conversion.get())); total_index_size += index_size; if (is_key) { RETURN_IF_ERROR(row_sources_buf.flush()); diff --git a/be/src/storage/rowset/segcompaction.h b/be/src/storage/rowset/segcompaction.h index b8004002681fcd..9252dbac39c5ff 100644 --- a/be/src/storage/rowset/segcompaction.h +++ b/be/src/storage/rowset/segcompaction.h @@ -29,7 +29,7 @@ #include "storage/tablet/tablet.h" namespace doris { -class Schema; +class ReadSchema; namespace segment_v2 { class SegmentWriter; @@ -79,9 +79,9 @@ class SegcompactionWorker { Status _create_segment_writer_for_segcompaction( std::unique_ptr* writer, uint32_t begin, uint32_t end); Status _get_segcompaction_reader(SegCompactionCandidatesSharedPtr segments, - TabletSharedPtr tablet, std::shared_ptr schema, + TabletSharedPtr tablet, ReadSchemaSPtr read_schema, OlapReaderStatistics* stat, RowSourcesBuffer& row_sources_buf, - bool is_key, std::vector& return_columns, + bool is_key, std::vector& key_group_cluster_key_idxes, std::unique_ptr* reader); std::unique_ptr _create_segcompaction_writer(uint32_t begin, diff --git a/be/src/storage/schema.cpp b/be/src/storage/schema.cpp index bd141a0f3f0e71..fd0069177fe17f 100644 --- a/be/src/storage/schema.cpp +++ b/be/src/storage/schema.cpp @@ -17,90 +17,143 @@ #include "storage/schema.h" -#include - -#include -#include -#include #include #include "common/config.h" -#include "core/column/column_array.h" -#include "core/column/column_decimal.h" +#include "common/logging.h" +#include "core/block/block.h" #include "core/column/column_dictionary.h" -#include "core/column/column_map.h" +#include "core/column/column_nothing.h" #include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_struct.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_factory.hpp" -#include "core/data_type/define_primitive_type.h" -#include "core/types.h" -#include "storage/olap_common.h" -#include "util/trace.h" +#include "storage/binlog.h" namespace doris { -Schema::Schema(const Schema& other) { - _copy_from(other); +ReadSchema::ReadSchema(std::vector columns) + : _read_columns(std::move(columns)), _num_block_columns(_read_columns.size()) { + _init_read_types(); + _init_descriptors(); } -Schema& Schema::operator=(const Schema& other) { - if (this != &other) { - _copy_from(other); +ReadSchema::ReadSchema(const std::vector& columns, + const std::vector& cids) + : _num_block_columns(cids.size()) { + _read_columns.reserve(cids.size()); + for (auto cid : cids) { + _read_columns.emplace_back(columns[cid]); } - return *this; + _init_read_types(); + _init_descriptors(); +} + +ReadSchema::ReadSchema(std::vector columns, std::vector read_types) + : _read_columns(std::move(columns)), + _read_types(std::move(read_types)), + _num_block_columns(_read_types.size()) { + _init_descriptors(); } -void Schema::_copy_from(const Schema& other) { - _col_ids = other._col_ids; - _column_id_to_index = other._column_id_to_index; - _num_key_columns = other._num_key_columns; - _delete_sign_idx = other._delete_sign_idx; - _has_sequence_col = other._has_sequence_col; - _rowid_col_idx = other._rowid_col_idx; - _version_col_idx = other._version_col_idx; - _commit_tso_col_idx = other._commit_tso_col_idx; - _tso_col_idx = other._tso_col_idx; - _lsn_col_idx = other._lsn_col_idx; - _op_col_idx = other._op_col_idx; - - _cols.resize(other._cols.size()); - for (auto cid : _col_ids) { - _cols[cid] = other._cols[cid]; +void ReadSchema::_init_read_types() { + _read_types.reserve(_read_columns.size()); + for (const auto& column : _read_columns) { + auto data_type = column->get_vec_type(); + DORIS_CHECK(data_type != nullptr); + _read_types.emplace_back(std::move(data_type)); } } -void Schema::_init(const std::vector& cols, const std::vector& col_ids, - size_t num_key_columns) { - _col_ids = col_ids; - _num_key_columns = num_key_columns; +void ReadSchema::_init_before_column_ordinals() { + std::unordered_map name_to_ordinal; + name_to_ordinal.reserve(_num_block_columns); + for (ColumnId ordinal = 0; ordinal < _num_block_columns; ++ordinal) { + name_to_ordinal.emplace(_read_columns[ordinal]->name(), ordinal); + } - _cols.resize(cols.size()); + _before_column_ordinals.resize(_num_block_columns); + for (ColumnId ordinal = 0; ordinal < _num_block_columns; ++ordinal) { + auto read_ordinal = static_cast(ordinal); + if (read_ordinal == _tso_ordinal || read_ordinal == _lsn_ordinal || + read_ordinal == _op_ordinal) { + _before_column_ordinals[ordinal] = ordinal; + continue; + } + auto before_name = binlog::build_before_column_name(_read_columns[ordinal]->name()); + auto before = name_to_ordinal.find(before_name); + _before_column_ordinals[ordinal] = + before == name_to_ordinal.end() ? ordinal : before->second; + } +} - std::unordered_set col_id_set(col_ids.begin(), col_ids.end()); - _column_id_to_index.assign(cols.size(), -1); - for (size_t i = 0; i < col_ids.size(); ++i) { - _column_id_to_index[col_ids[i]] = static_cast(i); +Block ReadSchema::create_read_block() const { + Block block; + for (size_t ordinal = 0; ordinal < _num_block_columns; ++ordinal) { + const auto& data_type = _read_types[ordinal]; + DORIS_CHECK(data_type != nullptr); + MutableColumnPtr column; + if (_read_columns[ordinal]->name().starts_with(BeConsts::VIRTUAL_COLUMN_PREFIX)) { + column = ColumnNothing::create(0); + } else { + column = data_type->create_column(); + } + block.insert({std::move(column), data_type, _read_columns[ordinal]->name()}); } + return block; +} - for (int cid = 0; cid < cols.size(); ++cid) { - if (col_id_set.find(cid) == col_id_set.end()) { - continue; +std::string ReadSchema::read_columns_to_string() const { + // Avoid lines that are too long to display in SHOW PROFILE. + constexpr int columns_per_line = 10; + int column_index = 0; + std::string result = "["; + for (auto it = _read_columns.cbegin(); it != _read_columns.cend(); ++it) { + if (it != _read_columns.cbegin()) { + result += ", "; + } + result += (*it)->name(); + if (column_index >= columns_per_line) { + result += "\n"; + column_index = 0; + } else { + ++column_index; } - _cols[cid] = cols[cid]; } + result += "]"; + return result; } -Schema::~Schema() = default; +Status ReadSchema::init_sequence_map(const TabletSchema& tablet_schema) { + if (tablet_schema.has_sequence_col()) { + auto msg = "sequence columns conflict, both seq_col and seq_map are true!"; + LOG(WARNING) << msg; + return Status::InternalError(msg); + } + + _sequence_map.clear(); + for (const auto& [sequence_cid, value_cids] : tablet_schema.seq_col_idx_to_value_cols_idx()) { + std::vector value_ordinals; + for (auto value_cid : value_cids) { + int32_t value_ordinal = ordinal_by_column(tablet_schema.column(value_cid)); + if (value_ordinal >= 0 && static_cast(value_ordinal) < num_block_columns()) { + value_ordinals.emplace_back(value_ordinal); + } + } -DataTypePtr Schema::get_data_type_ptr(const TabletColumn& column) { - return DataTypeFactory::instance().create_data_type(column); + int32_t sequence_ordinal = ordinal_by_column(tablet_schema.column(sequence_cid)); + if (sequence_ordinal < 0 || static_cast(sequence_ordinal) >= num_block_columns()) { + if (value_ordinals.empty()) { + continue; + } + return Status::InvalidArgument( + "Sequence column {} must be present in the read Block schema", + tablet_schema.column(sequence_cid).name()); + } + _sequence_map.emplace(sequence_ordinal, std::move(value_ordinals)); + } + return Status::OK(); } -IColumn::MutablePtr Schema::get_predicate_column_ptr(const DataTypePtr& data_type, - const ReaderType reader_type) { +IColumn::MutablePtr ReadSchema::get_predicate_column_ptr(const DataTypePtr& data_type, + const ReaderType reader_type) { // Low-cardinality dictionary optimization substitutes a ColumnDictI32 for the // canonical string column during query reads. Every other case just materializes // the data type's own canonical column (which already wraps nullable for us). diff --git a/be/src/storage/schema.h b/be/src/storage/schema.h index e35d4c37ac17f0..57f013ec2b6717 100644 --- a/be/src/storage/schema.h +++ b/be/src/storage/schema.h @@ -24,9 +24,11 @@ #include #include #include +#include #include #include "common/consts.h" +#include "common/status.h" #include "core/column/column.h" #include "exprs/aggregate/aggregate_function.h" #include "io/io_common.h" @@ -37,110 +39,181 @@ namespace doris { -// The class is used to represent row's format in memory. Each row contains -// multiple columns, some of which are key-columns (the rest are value-columns). -// NOTE: If both key-columns and value-columns exist, then the key-columns -// must be placed before value-columns. -// -// To compare two rows whose schemas are different, but they are from the same origin -// we store all column schema maybe accessed here. And default access through column id -class Schema; -using SchemaSPtr = std::shared_ptr; -class Schema { +class ReadSchema; +class Block; +using ReadSchemaSPtr = std::shared_ptr; +class ReadSchema { public: - // All the columns of one table may exist in the columns param, but col_ids is only a subset. - Schema(const std::vector& columns, const std::vector& col_ids) { - size_t num_key_columns = 0; - for (int i = 0; i < columns.size(); ++i) { - if (columns[i]->is_key()) { - ++num_key_columns; - } - if (columns[i]->name() == DELETE_SIGN) { - _delete_sign_idx = i; - } - if (columns[i]->name() == BeConsts::ROWID_COL || - columns[i]->name().starts_with(BeConsts::GLOBAL_ROWID_COL)) { - _rowid_col_idx = i; - } - if (columns[i]->name() == VERSION_COL) { - _version_col_idx = i; - } - if (columns[i]->name() == BINLOG_TSO_COL) { - _tso_col_idx = i; - } - if (columns[i]->name() == BINLOG_LSN_COL) { - _lsn_col_idx = i; - } - if (columns[i]->name() == BINLOG_OP_COL) { - _op_col_idx = i; - } - if (columns[i]->name() == COMMIT_TSO_COL) { - _commit_tso_col_idx = i; - } - } - _init(columns, col_ids, num_key_columns); - } + using SequenceMap = std::unordered_map>; - Schema(const Schema&); - Schema& operator=(const Schema& other); + explicit ReadSchema(std::vector columns); - ~Schema(); + explicit ReadSchema(const std::vector& columns, + const std::vector& cids); - static DataTypePtr get_data_type_ptr(const TabletColumn& column); + // Initially every column is a caller-visible FE slot. Storage-only columns + // may be appended later without changing `num_block_columns()`. + explicit ReadSchema(std::vector columns, std::vector read_types); static IColumn::MutablePtr get_predicate_column_ptr(const DataTypePtr& data_type, const ReaderType reader_type); - const std::vector& columns() const { return _cols; } + const std::vector& columns() const { return _read_columns; } - const TabletColumn* column(ColumnId cid) const { return _cols[cid].get(); } + DataTypePtr data_type(size_t ordinal) const { + DCHECK_LT(ordinal, _read_types.size()); + return _read_types[ordinal]; + } + + // Append a storage-only column without extending the caller-visible slot prefix. + ColumnId append_column(TabletColumnPtr column) { + auto data_type = column->get_vec_type(); + auto ordinal = cast_set(_read_columns.size()); + if (column->unique_id() >= 0) { + _uid_to_ordinal.emplace(column->unique_id(), ordinal); + } + _read_columns.emplace_back(std::move(column)); + _read_types.emplace_back(std::move(data_type)); + return ordinal; + } + + // Create caller-visible Blocks from the FE-slot prefix. + Block create_read_block() const; + + std::string read_columns_to_string() const; + + Status init_sequence_map(const TabletSchema& tablet_schema); + + const SequenceMap& sequence_map() const { return _sequence_map; } + + ColumnId before_column_ordinal(ColumnId ordinal) const { + DCHECK_LT(ordinal, _before_column_ordinals.size()); + return _before_column_ordinals[ordinal]; + } + + const TabletColumn* column(size_t ordinal) const { return _read_columns[ordinal].get(); } + + // Resolve by unique id when one exists. Name identity is only for legacy + // columns without a unique id. + int32_t ordinal_by_column(const TabletColumn& column) const { + if (column.unique_id() >= 0) { + return ordinal_by_uid(column.unique_id()); + } + for (uint32_t ordinal = 0; ordinal < _read_columns.size(); ++ordinal) { + if (_read_columns[ordinal]->name() == column.name()) { + return static_cast(ordinal); + } + } + return -1; + } + + // Total columns used inside storage, including appended storage-only columns. + // Use this for per-column state and iteration over the complete ReadSchema. + size_t num_read_columns() const { return _read_columns.size(); } + + // Columns materialized in caller Blocks. They are the ReadSchema prefix before + // appended storage-only columns; use this for Block layout and position bounds. + size_t num_block_columns() const { return _num_block_columns; } size_t num_key_columns() const { return _num_key_columns; } - size_t num_columns() const { return _cols.size(); } - size_t num_column_ids() const { return _col_ids.size(); } - const std::vector& column_ids() const { return _col_ids; } - ColumnId column_id(size_t index) const { return _col_ids[index]; } - int column_index(ColumnId cid) const { return _column_id_to_index[cid]; } - const std::vector& column_id_to_index() const { return _column_id_to_index; } - int32_t delete_sign_idx() const { return _delete_sign_idx; } - bool has_sequence_col() const { return _has_sequence_col; } - int32_t rowid_col_idx() const { return _rowid_col_idx; } - int32_t version_col_idx() const { return _version_col_idx; } - int32_t commit_tso_col_idx() const { return _commit_tso_col_idx; } - int32_t tso_col_idx() const { return _tso_col_idx; } - int32_t lsn_col_idx() const { return _lsn_col_idx; } - int32_t op_col_idx() const { return _op_col_idx; } - // Don't use. - // TODO: memory size of Schema cannot be accurately tracked. - // In some places, temporarily use num_columns() as Schema size. - int64_t mem_size() const { return _mem_size; } + int32_t delete_sign_ordinal() const { return _delete_sign_ordinal; } + int32_t sequence_ordinal() const { return _sequence_ordinal; } + int32_t rowid_ordinal() const { return _rowid_ordinal; } + int32_t version_ordinal() const { return _version_ordinal; } + int32_t tso_ordinal() const { return _tso_ordinal; } + int32_t lsn_ordinal() const { return _lsn_ordinal; } + int32_t op_ordinal() const { return _op_ordinal; } + int32_t commit_tso_ordinal() const { return _commit_tso_ordinal; } + + // -1 if no column with this unique id is present. Columns without a valid + // unique id (e.g. variant extracted subcolumns) are not in this map; + // resolve those with ordinal_by_column(). + int32_t ordinal_by_uid(int32_t unique_id) const { + auto it = _uid_to_ordinal.find(unique_id); + return it == _uid_to_ordinal.end() ? -1 : it->second; + } private: - void _init(const std::vector& cols, const std::vector& col_ids, - size_t num_key_columns); - - void _copy_from(const Schema& other); - - // NOTE: The ColumnId here represents the sequential index number (starting from 0) of - // a column in current row, not the unique id-identifier of each column - std::vector _col_ids; - // NOTE: _cols[cid] can only be accessed when the cid is - // contained in _col_ids - std::vector _cols; - // Tablet column id -> slot index in this Schema. - std::vector _column_id_to_index; - - size_t _num_key_columns; - int32_t _delete_sign_idx = -1; - bool _has_sequence_col = false; - int32_t _rowid_col_idx = -1; - int32_t _version_col_idx = -1; - int32_t _commit_tso_col_idx = -1; - int32_t _tso_col_idx = -1; - int32_t _lsn_col_idx = -1; - int32_t _op_col_idx = -1; - int64_t _mem_size = 0; + void _init_read_types(); + void _init_before_column_ordinals(); + + void _init_descriptors() { + DORIS_CHECK_LE(_num_block_columns, _read_columns.size()); + DORIS_CHECK_EQ(_read_columns.size(), _read_types.size()); + _num_key_columns = 0; + _delete_sign_ordinal = -1; + _sequence_ordinal = -1; + _rowid_ordinal = -1; + _version_ordinal = -1; + _tso_ordinal = -1; + _lsn_ordinal = -1; + _op_ordinal = -1; + _commit_tso_ordinal = -1; + _before_column_ordinals.clear(); + _uid_to_ordinal.clear(); + for (uint32_t i = 0; i < _read_columns.size(); ++i) { + const auto& col = *_read_columns[i]; + if (col.unique_id() >= 0) { + _uid_to_ordinal.emplace(col.unique_id(), i); + } + } + for (uint32_t i = 0; i < _num_block_columns; ++i) { + const auto& col = *_read_columns[i]; + if (col.is_key()) { + ++_num_key_columns; + } + if (col.name() == DELETE_SIGN) { + _delete_sign_ordinal = i; + } + if (col.name() == SEQUENCE_COL) { + _sequence_ordinal = i; + } + if (col.name() == BeConsts::ROWID_COL || + col.name().starts_with(BeConsts::GLOBAL_ROWID_COL)) { + _rowid_ordinal = i; + } + if (col.name() == VERSION_COL) { + _version_ordinal = i; + } + if (col.name() == BINLOG_TSO_COL) { + _tso_ordinal = i; + } + if (col.name() == BINLOG_LSN_COL) { + _lsn_ordinal = i; + } + if (col.name() == BINLOG_OP_COL) { + _op_ordinal = i; + } + if (col.name() == COMMIT_TSO_COL) { + _commit_tso_ordinal = i; + } + } + if (_op_ordinal >= 0) { + _init_before_column_ordinals(); + } + } + + // The first `_num_block_columns` entries are the requested read columns. Any + // remaining entries are dropped columns appended for historical delete predicates. + std::vector _read_columns; + // Types aligned by ordinal with `_read_columns`. + std::vector _read_types; + // Boundary between the columns materialized in Blocks and the appended dropped columns. + size_t _num_block_columns = 0; + + size_t _num_key_columns = 0; + int32_t _delete_sign_ordinal = -1; + int32_t _sequence_ordinal = -1; + int32_t _rowid_ordinal = -1; + int32_t _version_ordinal = -1; + int32_t _tso_ordinal = -1; + int32_t _lsn_ordinal = -1; + int32_t _op_ordinal = -1; + int32_t _commit_tso_ordinal = -1; + std::unordered_map _uid_to_ordinal; + SequenceMap _sequence_map; + std::vector _before_column_ordinals; }; } // namespace doris diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index b705d75a4da205..8d23519e421cf9 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -160,7 +160,7 @@ class MultiBlockMerger { // The block version is incremental. std::stable_sort(row_refs.begin(), row_refs.end(), _cmp); - auto finalized_block = _tablet->tablet_schema()->create_block(); + auto finalized_block = _tablet->tablet_schema()->create_storage_block(); int columns = finalized_block.columns(); *merged_rows += rows; @@ -578,17 +578,11 @@ Status VSchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader RowsetWriter* rowset_writer, BaseTabletSPtr new_tablet, TabletSchemaSPtr base_tablet_schema, TabletSchemaSPtr new_tablet_schema) { + const auto& source_block_schema = rowset_reader->read_schema(); bool eof = false; do { - auto new_block = Block::create_unique(new_tablet_schema->create_block()); - // create_block() skips dropped columns (from light-weight schema change). - // Dropped columns are only needed for delete predicate evaluation, which - // SegmentIterator handles internally — it creates temporary columns for - // predicate columns not present in the block (via `i >= block->columns()` - // guard in _init_current_block). If dropped columns were included here, - // the block would have more columns than VMergeIterator's output_schema - // expects, causing DCHECK failures in copy_rows. - auto ref_block = Block::create_unique(base_tablet_schema->create_block()); + auto new_block = Block::create_unique(new_tablet_schema->create_storage_block()); + auto ref_block = Block::create_unique(source_block_schema.create_read_block()); Status st = next_batch(rowset_reader, ref_block.get(), _row_same_bit); if (!st) { @@ -625,6 +619,7 @@ Status VBaseSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset BaseTabletSPtr new_tablet, TabletSchemaSPtr base_tablet_schema, TabletSchemaSPtr new_tablet_schema) { + const auto& source_block_schema = rowset_reader->read_schema(); // for internal sorting std::vector> blocks; @@ -653,18 +648,11 @@ Status VBaseSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset return Status::OK(); }; - auto new_block = Block::create_unique(new_tablet_schema->create_block()); + auto new_block = Block::create_unique(new_tablet_schema->create_storage_block()); bool eof = false; do { - // create_block() skips dropped columns (from light-weight schema change). - // Dropped columns are only needed for delete predicate evaluation, which - // SegmentIterator handles internally — it creates temporary columns for - // predicate columns not present in the block (via `i >= block->columns()` - // guard in _init_current_block). If dropped columns were included here, - // the block would have more columns than VMergeIterator's output_schema - // expects, causing DCHECK failures in copy_rows. - auto ref_block = Block::create_unique(base_tablet_schema->create_block()); + auto ref_block = Block::create_unique(source_block_schema.create_read_block()); Status st = next_batch(rowset_reader, ref_block.get(), _row_same_bit); if (!st) { if (st.is()) { @@ -700,7 +688,7 @@ Status VBaseSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset _mem_tracker->consume(new_block->allocated_bytes()); // move unique ptr - blocks.push_back(Block::create_unique(new_tablet_schema->create_block())); + blocks.push_back(Block::create_unique(new_tablet_schema->create_storage_block())); swap(blocks.back(), new_block); } while (!eof); @@ -938,39 +926,17 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques std::vector rs_splits; // delete handlers for new tablet DeleteHandler delete_handler; - std::vector return_columns; - - // Use tablet schema directly from base tablet, they are the newest schema, not contain - // dropped column during light weight schema change. - // But the tablet schema in base tablet maybe not the latest from FE, so that if fe pass through - // a tablet schema, then use request schema. - // - // return_columns does NOT include dropped columns. It is computed here BEFORE - // merge_dropped_columns() appends dropped columns to _base_tablet_schema below. - // This means return_columns only covers the original (non-dropped) columns. - // - // This is important because: - // - BetaRowsetReader builds _output_schema from return_columns, which determines the - // number of columns in ref_block (via create_block() which also skips dropped cols). - // - VMergeIterator's copy_rows iterates over _output_schema columns, so ref_block - // must match _output_schema exactly. - // - Dropped columns are only needed for delete predicate evaluation, and SegmentIterator - // handles them internally (creates temporary columns for predicate columns not present - // in the block via `i >= block->columns()` guard in _init_current_block). - // - // Example: table has columns [k1, v1, v2], then DROP COLUMN v1, then - // DELETE FROM t WHERE v1 = 'x' was issued before the drop. - // - _base_tablet_schema after merge_dropped_columns: [k1, v2, v1(DROPPED)] - // - return_columns (computed before merge): [0, 1] → [k1, v2] - // - _output_schema / ref_block columns: [k1, v2] (2 columns) - // - SegmentIterator reads v1 internally for delete predicate, but does not - // output it to ref_block. copy_rows only iterates 2 columns — no OOB access. + + // Use the visible schema from the base tablet. If FE supplies a schema in + // the request, prefer it because the tablet schema may not be the latest. + // Historical delete columns are appended after this schema-change output + // prefix and are not returned in Blocks. size_t num_cols = request.columns.empty() ? _base_tablet_schema->num_columns() : request.columns.size(); - return_columns.resize(num_cols); - for (int i = 0; i < num_cols; ++i) { - return_columns[i] = i; - } + DORIS_CHECK_LE(num_cols, _base_tablet_schema->columns().size()); + std::vector read_columns(_base_tablet_schema->columns().begin(), + _base_tablet_schema->columns().begin() + num_cols); + ReadSchemaSPtr read_schema = std::make_shared(std::move(read_columns)); std::vector cluster_key_idxes; DBUG_EXECUTE_IF("SchemaChangeJob::_do_process_alter_tablet.block", DBUG_BLOCK); @@ -1068,33 +1034,35 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques if (!rs_meta->has_delete_predicate() || rs_meta->start_version() > end_version) { continue; } - _base_tablet_schema->merge_dropped_columns(*rs_meta->tablet_schema()); del_preds.push_back(rs_meta); } - res = delete_handler.init(_base_tablet_schema, del_preds, end_version); + std::vector dropped_columns; + res = delete_handler.init(del_preds, end_version, read_schema, dropped_columns); if (!res) { LOG(WARNING) << "init delete handler failed. base_tablet=" << _base_tablet->tablet_id() << ", end_version=" << end_version; break; } + for (auto& column : dropped_columns) { + read_schema->append_column(std::make_shared(std::move(column))); + } reader_context.reader_type = ReaderType::READER_ALTER_TABLE; reader_context.tablet_schema = _base_tablet_schema; reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; - reader_context.return_columns = &return_columns; - reader_context.sequence_id_idx = reader_context.tablet_schema->sequence_col_idx(); + reader_context.read_schema = read_schema; reader_context.is_unique = _base_tablet->keys_type() == UNIQUE_KEYS; reader_context.batch_size = ALTER_TABLE_BATCH_SIZE; reader_context.delete_bitmap = _base_tablet->tablet_meta()->delete_bitmap_ptr(); reader_context.version = Version(0, end_version); if (!_base_tablet_schema->cluster_key_uids().empty()) { for (const auto& uid : _base_tablet_schema->cluster_key_uids()) { - cluster_key_idxes.emplace_back(_base_tablet_schema->field_index(uid)); + cluster_key_idxes.emplace_back(read_schema->ordinal_by_uid(uid)); } reader_context.read_orderby_key_columns = &cluster_key_idxes; reader_context.is_unique = false; - reader_context.sequence_id_idx = -1; + reader_context.use_sequence_column_for_merge_order = false; } for (auto& rs_split : rs_splits) { res = rs_split.rs_reader->init(&reader_context); diff --git a/be/src/storage/segment/empty_segment_iterator.cpp b/be/src/storage/segment/empty_segment_iterator.cpp index 22facd9842bec1..9a15b6fdbd5c09 100644 --- a/be/src/storage/segment/empty_segment_iterator.cpp +++ b/be/src/storage/segment/empty_segment_iterator.cpp @@ -22,7 +22,7 @@ class Block; namespace segment_v2 { -EmptySegmentIterator::EmptySegmentIterator(const doris::Schema& schema) : _schema(schema) {} +EmptySegmentIterator::EmptySegmentIterator(const doris::ReadSchema& schema) : _schema(schema) {} Status EmptySegmentIterator::next_batch(Block* block) { return Status::EndOfFile("no more data in segment"); diff --git a/be/src/storage/segment/empty_segment_iterator.h b/be/src/storage/segment/empty_segment_iterator.h index 40fba543d6896f..e20ffeb1294e16 100644 --- a/be/src/storage/segment/empty_segment_iterator.h +++ b/be/src/storage/segment/empty_segment_iterator.h @@ -28,15 +28,15 @@ namespace segment_v2 { class EmptySegmentIterator : public RowwiseIterator { public: - explicit EmptySegmentIterator(const Schema& schema); + explicit EmptySegmentIterator(const ReadSchema& schema); ~EmptySegmentIterator() override {} Status init(const StorageReadOptions& opts) override { return Status::OK(); } - const Schema& schema() const override { return _schema; } + const ReadSchema& schema() const override { return _schema; } Status next_batch(Block* block) override; bool empty() const override { return true; } private: - const Schema& _schema; + const ReadSchema& _schema; }; } // namespace segment_v2 diff --git a/be/src/storage/segment/historical_row_retriever.cpp b/be/src/storage/segment/historical_row_retriever.cpp index 8def79ed1e1418..e150397876650b 100644 --- a/be/src/storage/segment/historical_row_retriever.cpp +++ b/be/src/storage/segment/historical_row_retriever.cpp @@ -176,7 +176,7 @@ Status PrimaryKeyModelRowRetriever::build_before_block(Block* before_block, } // Create block to hold historical values for value columns. - Block old_value_block = tablet_schema->create_block_by_cids(value_cids); + Block old_value_block = tablet_schema->create_storage_block(value_cids); CHECK_EQ(value_cids.size(), old_value_block.columns()); // key: logical row index in current batch; value: index in old_value_block diff --git a/be/src/storage/segment/lazy_init_segment_iterator.cpp b/be/src/storage/segment/lazy_init_segment_iterator.cpp index 8e61384556e9ea..2ad058e5ef4287 100644 --- a/be/src/storage/segment/lazy_init_segment_iterator.cpp +++ b/be/src/storage/segment/lazy_init_segment_iterator.cpp @@ -22,7 +22,7 @@ namespace doris::segment_v2 { LazyInitSegmentIterator::LazyInitSegmentIterator(BetaRowsetSharedPtr rowset, int64_t segment_id, - bool should_use_cache, SchemaSPtr schema, + bool should_use_cache, ReadSchemaSPtr schema, const StorageReadOptions& opts) : _rowset(std::move(rowset)), _segment_id(segment_id), diff --git a/be/src/storage/segment/lazy_init_segment_iterator.h b/be/src/storage/segment/lazy_init_segment_iterator.h index 147480e0e85886..ca14de49a43ede 100644 --- a/be/src/storage/segment/lazy_init_segment_iterator.h +++ b/be/src/storage/segment/lazy_init_segment_iterator.h @@ -31,7 +31,7 @@ namespace doris::segment_v2 { class LazyInitSegmentIterator : public RowwiseIterator { public: LazyInitSegmentIterator(BetaRowsetSharedPtr rowset, int64_t segment_id, bool should_use_cache, - SchemaSPtr schema, const StorageReadOptions& opts); + ReadSchemaSPtr schema, const StorageReadOptions& opts); ~LazyInitSegmentIterator() override = default; @@ -46,7 +46,7 @@ class LazyInitSegmentIterator : public RowwiseIterator { return _inner_iterator->next_batch(block); } - const Schema& schema() const override { return *_schema; } + const ReadSchema& schema() const override { return *_schema; } Status current_block_row_locations(std::vector* locations) override { return _inner_iterator->current_block_row_locations(locations); @@ -63,7 +63,7 @@ class LazyInitSegmentIterator : public RowwiseIterator { BetaRowsetSharedPtr _rowset; int64_t _segment_id {-1}; bool _should_use_cache {false}; - SchemaSPtr _schema = nullptr; + ReadSchemaSPtr _schema = nullptr; StorageReadOptions _read_options; RowwiseIteratorUPtr _inner_iterator; }; diff --git a/be/src/storage/segment/row_binlog_segment_writer.cpp b/be/src/storage/segment/row_binlog_segment_writer.cpp index 807330b94960d2..68fcd11575be4e 100644 --- a/be/src/storage/segment/row_binlog_segment_writer.cpp +++ b/be/src/storage/segment/row_binlog_segment_writer.cpp @@ -184,7 +184,7 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, } // use full_block to save entrie row data - Block full_block = source_schema->create_block(); + Block full_block = source_schema->create_storage_block(); RETURN_IF_ERROR(_source_data_writer->prepare_by_source_block(block, row_pos, num_rows, partial_cids, &full_block)); @@ -331,7 +331,7 @@ Status RowBinlogSegmentWriter::_append_direct_block(const Block* block, size_t r Status RowBinlogSegmentWriter::_fill_binlog_columns(size_t num_rows, const std::vector& op_types) { std::vector binlog_cids = {_binlog_tso_col_id, _binlog_lsn_col_id, _binlog_op_col_id}; - Block binlog_prefix_block = _tablet_schema->create_block_by_cids(binlog_cids); + Block binlog_prefix_block = _tablet_schema->create_storage_block(binlog_cids); { auto binlog_prefix_columns_guard = binlog_prefix_block.mutate_columns_scoped(); auto& binlog_prefix_columns = binlog_prefix_columns_guard.mutable_columns(); @@ -424,7 +424,7 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) { before_cids.emplace_back(cid); } - Block before_block = _tablet_schema->create_block_by_cids(before_cids); + Block before_block = _tablet_schema->create_storage_block(before_cids); // Compatibility path: only fill empty BEFORE values. if (_fill_empty_before_value) { diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index 9e58a76e0a16e1..83182f6cb63c18 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -89,11 +89,9 @@ namespace doris::segment_v2 { -class InvertedIndexIterator; - namespace { -Status build_segment_zonemap_context(Segment* segment, const Schema& schema, +Status build_segment_zonemap_context(Segment* segment, const ReadSchema& schema, const StorageReadOptions& read_options, const VExprContextSPtrs& conjuncts, ZoneMapEvalContext* ctx) { DORIS_CHECK(segment != nullptr); @@ -113,17 +111,15 @@ Status build_segment_zonemap_context(Segment* segment, const Schema& schema, root->collect_slot_column_ids(slot_indexes); } for (const int slot_index : slot_indexes) { - if (slot_index < 0 || cast_set(slot_index) >= schema.num_column_ids()) { - continue; - } - const auto column_id = schema.column_id(cast_set(slot_index)); - const auto* tablet_column = schema.column(column_id); - DORIS_CHECK(tablet_column != nullptr); + const auto ordinal = cast_set(slot_index); + DORIS_CHECK_LT(ordinal, schema.num_block_columns()); + const auto* tablet_column = schema.column(ordinal); if (!segment->can_apply_predicate_safely( - column_id, schema, read_options.target_cast_type_for_variants, read_options)) { + slot_index, schema, read_options.target_cast_type_for_variants, read_options)) { continue; } - auto data_type = segment->get_data_type_of(*tablet_column, read_options); + auto data_type = + segment->get_data_type_of(*tablet_column, schema.data_type(ordinal), read_options); if (data_type == nullptr) { continue; } @@ -368,7 +364,7 @@ Status Segment::_open_index_file_reader() { return Status::OK(); } -bool Segment::is_tso_placeholder_col(int cid, const Schema& schema, +bool Segment::is_tso_placeholder_col(int cid, const ReadSchema& schema, const StorageReadOptions& read_options) const { if (read_options.version.first != read_options.version.second) { return false; @@ -376,11 +372,11 @@ bool Segment::is_tso_placeholder_col(int cid, const Schema& schema, if (!read_options.read_row_binlog) { return false; } - // tso_col_idx() is -1 for non-binlog schemas, so this returns false there. - return cid == schema.tso_col_idx(); + // tso_ordinal() is -1 for non-binlog schemas, so this returns false there. + return cid == schema.tso_ordinal(); } -Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options, +Status Segment::new_iterator(ReadSchemaSPtr schema, const StorageReadOptions& read_options, std::unique_ptr* iter) { if (read_options.runtime_state != nullptr) { _be_exec_version = read_options.runtime_state->be_exec_version(); @@ -390,12 +386,9 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o read_options.stats->total_segment_number++; // trying to prune the current segment by segment-level zone map for (const auto& entry : read_options.col_id_to_predicates) { + // col_id_to_predicates is keyed by read-schema ordinal. int32_t column_id = entry.first; - // schema change - if (_tablet_schema->num_columns() <= column_id) { - continue; - } - const TabletColumn& col = read_options.tablet_schema->column(column_id); + const TabletColumn& col = *schema->column(column_id); std::shared_ptr reader; // __DORIS_COMMIT_TSO_COL__ on a single-version segment stores a 0 placeholder on disk // (replaced with the rowset's real commit_tso at read time). Its on-disk zonemap [0,0] @@ -403,7 +396,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o // commit_tso to prune against the real value instead. std::optional const_value; if (read_options.version.first == read_options.version.second && - column_id == schema->commit_tso_col_idx() && read_options.commit_tso.end_tso() != -1) { + column_id == schema->commit_tso_ordinal() && read_options.commit_tso.end_tso() != -1) { const_value = Field::create_field(read_options.commit_tso.end_tso()); } Status st = get_column_reader(col, &reader, read_options.stats, &read_options.io_ctx, @@ -455,8 +448,6 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o } } - // Segment-level expr-zonemap runs before SegmentIterator can rebind storage expressions to - // the reader schema. Only apply it when scan tuple slot ordinals already match this schema. if (expr_zonemap::is_expr_zonemap_filter_enabled(read_options.runtime_state) && !read_options.common_expr_ctxs_push_down.empty()) { ZoneMapEvalContext ctx; @@ -493,9 +484,13 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o auto pruned = false; for (auto& it : _column_reader_cache->get_available_readers(false)) { const auto uid = it.first; - const auto column_id = read_options.tablet_schema->field_index(uid); + // Column readers are keyed by UID; predicates use read schema ordinals. + const auto ordinal = schema->ordinal_by_uid(uid); + if (ordinal < 0) { + continue; + } bool tmp_pruned = false; - RETURN_IF_ERROR(it.second->prune_predicates_by_zone_map(pruned_predicates, column_id, + RETURN_IF_ERROR(it.second->prune_predicates_by_zone_map(pruned_predicates, ordinal, &tmp_pruned)); pruned |= tmp_pruned; } @@ -519,7 +514,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o // Key columns may still be required by key range seeks even if the segment zone // map proves their predicates always true. Only mark non-key columns as safe for // the no-need-read path. - if (!read_options.tablet_schema->column(pred_cid).is_key() && + if (!schema->column(pred_cid)->is_key() && !options_with_pruned_predicates.col_id_to_predicates.contains(pred_cid)) { options_with_pruned_predicates.zonemap_always_true_pred_cols.insert(pred_cid); } @@ -779,23 +774,23 @@ Status Segment::healthy_status() { } } -// Return the storage datatype of related column to field. -DataTypePtr Segment::get_data_type_of(const TabletColumn& column, +DataTypePtr Segment::get_data_type_of(const TabletColumn& read_column, const DataTypePtr& read_type, const StorageReadOptions& read_options) { - const PathInDataPtr path = column.path_info_ptr(); + const PathInDataPtr path = read_column.path_info_ptr(); // none variant column if (path == nullptr || path->empty()) { - return DataTypeFactory::instance().create_data_type(column); + return read_type; } // Path exists, proceed with variant logic. PathInData relative_path = path->copy_pop_front(); - int32_t unique_id = column.unique_id() >= 0 ? column.unique_id() : column.parent_unique_id(); + int32_t unique_id = + read_column.unique_id() >= 0 ? read_column.unique_id() : read_column.parent_unique_id(); // If this uid does not exist in segment meta, fallback to schema type. if (!_column_meta_accessor->has_column_uid(unique_id)) { - return DataTypeFactory::instance().create_data_type(column); + return DataTypeFactory::instance().create_data_type(read_column); } std::shared_ptr v_reader; @@ -809,7 +804,7 @@ DataTypePtr Segment::get_data_type_of(const TabletColumn& column, auto* variant_reader = static_cast(v_reader.get()); // Delegate type inference for variant paths to VariantColumnReader. DataTypePtr type; - THROW_IF_ERROR(variant_reader->infer_data_type_for_path(&type, column, read_options, + THROW_IF_ERROR(variant_reader->infer_data_type_for_path(&type, read_column, read_options, _column_reader_cache.get())); return type; } @@ -1259,7 +1254,7 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto make_nullable(slot->type()), path.get_path(), variant_util::ExtraInfo {.parent_unique_id = slot->col_unique_id(), .path_info = path}); - auto storage_type = get_data_type_of(column, storage_read_options); + auto storage_type = get_data_type_of(column, slot->type(), storage_read_options); DCHECK(storage_type != nullptr); MutableColumnPtr file_storage_column = storage_type->create_column(); diff --git a/be/src/storage/segment/segment.h b/be/src/storage/segment/segment.h index 2c0b05a3516e2f..ef1792ccdcf9d6 100644 --- a/be/src/storage/segment/segment.h +++ b/be/src/storage/segment/segment.h @@ -51,7 +51,7 @@ namespace doris { class IDataType; class ShortKeyIndexDecoder; -class Schema; +class ReadSchema; class StorageReadOptions; class PrimaryKeyIndexReader; class RowwiseIterator; @@ -110,7 +110,7 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd int64_t get_metadata_size() const override; void update_metadata_size(); - Status new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options, + Status new_iterator(ReadSchemaSPtr schema, const StorageReadOptions& read_options, std::unique_ptr* iter); static Status new_default_iterator(const TabletColumn& tablet_column, @@ -180,21 +180,21 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd // another method `get_metadata_size` not include the column reader, only the segment object itself. int64_t meta_mem_usage() const { return _meta_mem_usage; } - // Get the inner file column's data type. - // When `read_options` is provided, the decision (e.g. flat-leaf vs hierarchical) can depend - // on the reader type and tablet schema; when it is nullptr, we treat it as a query reader. - // nullptr will be returned if storage type does not contain such column. - std::shared_ptr get_data_type_of(const TabletColumn& column, + // Variant paths use segment metadata; other columns use `read_type`. + std::shared_ptr get_data_type_of(const TabletColumn& read_column, + const DataTypePtr& read_type, const StorageReadOptions& read_options); // If column in segment is the same type in schema, then it is safe to apply predicate. + // `ordinal` is a read-schema ordinal (the coordinate ColumnPredicate::column_id() carries). bool can_apply_predicate_safely( - int cid, const Schema& schema, + int ordinal, const ReadSchema& schema, const std::map& target_cast_type_for_variants, const StorageReadOptions& read_options) { - const TabletColumn* col = schema.column(cid); - DCHECK(col != nullptr) << "Column not found in schema for cid=" << cid; - DataTypePtr storage_column_type = get_data_type_of(*col, read_options); + const TabletColumn* col = schema.column(ordinal); + DCHECK(col != nullptr) << "Column not found in schema for ordinal=" << ordinal; + DataTypePtr storage_column_type = + get_data_type_of(*col, schema.data_type(ordinal), read_options); if (storage_column_type == nullptr || col->type() != FieldType::OLAP_FIELD_TYPE_VARIANT || !target_cast_type_for_variants.contains(col->name())) { // Default column iterator or not variant column @@ -212,7 +212,7 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd // (SegmentIterator::_update_tso_col_if_needed). Its zonemap reflects the placeholder, so // it must NOT drive zonemap pruning. Mirrors the guards of _update_tso_col_if_needed. // Returns false for range (compaction) segments whose on-disk value is real. - bool is_tso_placeholder_col(int cid, const Schema& schema, + bool is_tso_placeholder_col(int cid, const ReadSchema& schema, const StorageReadOptions& read_options) const; const TabletSchemaSPtr& tablet_schema() const { return _tablet_schema; } diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 14a5fab34c582a..645939466a25f3 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -28,8 +28,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -42,7 +42,6 @@ #include "common/exception.h" #include "common/logging.h" #include "common/metrics/doris_metrics.h" -#include "common/object_pool.h" #include "common/status.h" #include "core/assert_cast.h" #include "core/block/column_with_type_and_name.h" @@ -63,7 +62,6 @@ #include "core/types.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/function/array/function_array_index.h" -#include "exprs/runtime_filter_expr.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "exprs/virtual_slot_ref.h" @@ -99,7 +97,6 @@ #include "storage/olap_common.h" #include "storage/predicate/bloom_filter_predicate.h" #include "storage/predicate/column_predicate.h" -#include "storage/predicate/like_column_predicate.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" #include "storage/segment/column_reader_cache.h" @@ -356,16 +353,15 @@ class SegmentIterator::BackwardBitmapRangeIterator : public SegmentIterator::Bit uint32_t _rowid_left; }; -SegmentIterator::SegmentIterator(std::shared_ptr segment, SchemaSPtr schema) +SegmentIterator::SegmentIterator(std::shared_ptr segment, ReadSchemaSPtr schema) : _segment(std::move(segment)), _schema(schema), - _column_iterators(_schema->num_columns()), - _index_iterators(_schema->num_columns()), + _column_iterators(_schema->num_read_columns()), + _index_iterators(_schema->num_read_columns()), _cur_rowid(0), - _lazy_materialization_read(false), + _column_states(_schema->num_read_columns()), _lazy_inited(false), - _inited(false), - _pool(new ObjectPool) {} + _inited(false) {} Status SegmentIterator::init(const StorageReadOptions& opts) { auto status = _init_impl(opts); @@ -380,26 +376,20 @@ std::unique_ptr SegmentIterator::_make_block_size_pr return nullptr; } - // Collect per-column raw byte metadata from the segment footer for the columns - // this iterator will actually output (defined by _schema, which is built from - // _opts.return_columns). + // Collect per-column raw byte metadata from the segment footer for the visible Block + // columns. Delete-predicate suffix columns do not contribute to the output byte budget. uint32_t seg_rows = _segment->num_rows(); uint64_t total_raw_bytes = 0; double metadata_hint_bytes_per_row = 0.0; if (seg_rows > 0) { - const auto& ts = _segment->tablet_schema(); - if (ts) { - for (ColumnId cid : _schema->column_ids()) { - if (static_cast(cid) < ts->num_columns()) { - int32_t uid = ts->column(cid).unique_id(); - uint64_t raw_bytes = _segment->column_raw_data_bytes(uid); - if (uid >= 0 && raw_bytes > 0) { - total_raw_bytes += raw_bytes; - } - } + for (size_t ordinal = 0; ordinal < _schema->num_block_columns(); ++ordinal) { + const auto& col = _schema->columns()[ordinal]; + int32_t uid = col->unique_id(); + if (uid >= 0) { + total_raw_bytes += _segment->column_raw_data_bytes(uid); } - metadata_hint_bytes_per_row = total_raw_bytes / static_cast(seg_rows); } + metadata_hint_bytes_per_row = total_raw_bytes / static_cast(seg_rows); } return std::make_unique( @@ -425,6 +415,7 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { } _col_predicates.emplace_back(predicate); } + _init_column_states(); _tablet_id = opts.tablet_id; // Read options will not change, so that just resize here _block_rowids.resize(_opts.block_row_max); @@ -433,10 +424,6 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { _initial_block_row_max = _opts.block_row_max; _block_size_predictor = _make_block_size_predictor(); - if (_schema->rowid_col_idx() > 0) { - _record_rowids = true; - } - _virtual_column_exprs = _opts.virtual_column_exprs; _score_runtime = _opts.score_runtime; _ann_topn_runtime = _opts.ann_topn_runtime; @@ -446,47 +433,39 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { _opts.runtime_state->enable_prune_nested_column(); if (opts.output_columns != nullptr) { - _output_columns = *(opts.output_columns); + _output_column_uids = *(opts.output_columns); } - _storage_name_and_type.resize(_schema->columns().size()); + _storage_name_and_type.resize(_schema->num_read_columns()); auto storage_format = _opts.tablet_schema->get_inverted_index_storage_format(); - for (int i = 0; i < _schema->columns().size(); ++i) { + for (size_t i = 0; i < _schema->num_read_columns(); ++i) { const TabletColumn* col = _schema->column(i); - if (col) { - auto storage_type = _segment->get_data_type_of(*col, _opts); - if (storage_type == nullptr) { - storage_type = - DataTypeFactory::instance().create_data_type(*col, col->is_nullable()); - } - // Currently, when writing a lucene index, the field of the document is column_name, and the column name is - // bound to the index field. Since version 1.2, the data file storage has been changed from column_name to - // column_unique_id, allowing the column name to be changed. Due to current limitations, previous inverted - // index data cannot be used after Doris changes the column name. Column names also support Unicode - // characters, which may cause other problems with indexing in non-ASCII characters. - // After consideration, it was decided to change the field name from column_name to column_unique_id in - // format V2, while format V1 continues to use column_name. - std::string field_name; - if (storage_format == InvertedIndexStorageFormatPB::V1) { - field_name = col->name(); + auto storage_type = _segment->get_data_type_of(*col, _schema->data_type(i), _opts); + // Currently, when writing a lucene index, the field of the document is column_name, and the column name is + // bound to the index field. Since version 1.2, the data file storage has been changed from column_name to + // column_unique_id, allowing the column name to be changed. Due to current limitations, previous inverted + // index data cannot be used after Doris changes the column name. Column names also support Unicode + // characters, which may cause other problems with indexing in non-ASCII characters. + // After consideration, it was decided to change the field name from column_name to column_unique_id in + // format V2, while format V1 continues to use column_name. + std::string field_name; + if (storage_format == InvertedIndexStorageFormatPB::V1) { + field_name = col->name(); + } else { + if (col->is_extracted_column()) { + // variant sub col + // field_name format: parent_unique_id.sub_col_name + field_name = std::to_string(col->parent_unique_id()) + "." + col->name(); } else { - if (col->is_extracted_column()) { - // variant sub col - // field_name format: parent_unique_id.sub_col_name - field_name = std::to_string(col->parent_unique_id()) + "." + col->name(); - } else { - field_name = std::to_string(col->unique_id()); - } - } - _storage_name_and_type[i] = std::make_pair(field_name, storage_type); - if (int32_t uid = - col->is_extracted_column() ? col->parent_unique_id() : col->unique_id(); - !_variant_sparse_column_cache.contains(uid)) { - DCHECK(uid >= 0); - _variant_sparse_column_cache.emplace(uid, - std::make_unique()); + field_name = std::to_string(col->unique_id()); } } + _storage_name_and_type[i] = std::make_pair(field_name, storage_type); + if (int32_t uid = col->is_extracted_column() ? col->parent_unique_id() : col->unique_id(); + !_variant_sparse_column_cache.contains(uid)) { + DCHECK(uid >= 0); + _variant_sparse_column_cache.emplace(uid, std::make_unique()); + } } RETURN_IF_ERROR(init_iterators()); @@ -499,6 +478,47 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { return Status::OK(); } +void SegmentIterator::_init_column_states() { + std::set> delete_predicates; + _opts.delete_condition_predicates->get_all_column_predicate(delete_predicates); + for (const auto& predicate : delete_predicates) { + auto cid = predicate->column_id(); + _column_states[cid].is_delete_predicate = true; + } + + _rebuild_row_predicate_columns(); +} + +void SegmentIterator::_rebuild_row_predicate_columns() { + for (auto& state : _column_states) { + state.is_row_predicate = false; + } + for (const auto& predicate : _col_predicates) { + _column_states[predicate->column_id()].is_row_predicate = true; + } +} + +void SegmentIterator::_extract_common_expr_columns(const VExprSPtr& expr) { + if (expr->is_slot_ref()) { + const auto ordinal = + cast_set(assert_cast(expr.get())->column_id()); + DORIS_CHECK_LT(ordinal, _schema->num_block_columns()); + _column_states[ordinal].is_common_expr = true; + _is_need_expr_eval = true; + return; + } + if (expr->is_virtual_slot_ref()) { + const auto& virtual_expr = + assert_cast(expr.get())->get_virtual_column_expr(); + DORIS_CHECK(virtual_expr != nullptr); + _extract_common_expr_columns(virtual_expr); + return; + } + for (const auto& child : expr->children()) { + _extract_common_expr_columns(child); + } +} + void SegmentIterator::_initialize_predicate_results() { // Initialize from _col_predicates for (auto pred : _col_predicates) { @@ -510,7 +530,7 @@ void SegmentIterator::_initialize_predicate_results() { } Status SegmentIterator::init_iterators() { - RETURN_IF_ERROR(_init_return_column_iterators()); + RETURN_IF_ERROR(_init_column_iterators()); RETURN_IF_ERROR(_init_index_iterators()); return Status::OK(); } @@ -562,51 +582,15 @@ Status SegmentIterator::_lazy_init(Block* block) { // cause heap-buffer-overflow if a later prediction is larger. auto nrows_reserve_limit = std::min(_row_bitmap.cardinality(), uint64_t(_initial_block_row_max)); - if (_lazy_materialization_read || _opts.record_rowids || _is_need_expr_eval) { - _block_rowids.resize(_initial_block_row_max); - } - _current_return_columns.resize(_schema->columns().size()); + _current_columns.resize(_schema->num_read_columns()); - for (size_t i = 0; i < _schema->column_ids().size(); i++) { - ColumnId cid = _schema->column_ids()[i]; - const auto* column_desc = _schema->column(cid); - if (_is_pred_column[cid]) { - auto storage_column_type = _storage_name_and_type[cid].second; + for (size_t i = 0; i < _schema->num_read_columns(); i++) { + if (_column_states[i].is_predicate()) { RETURN_IF_CATCH_EXCEPTION( - // Here, cid will not go out of bounds - // because the size of _current_return_columns equals _schema->tablet_columns().size() - _current_return_columns[cid] = Schema::get_predicate_column_ptr( - storage_column_type, _opts.io_ctx.reader_type)); - _current_return_columns[cid]->set_rowset_segment_id( - {_segment->rowset_id(), _segment->id()}); - _current_return_columns[cid]->reserve(nrows_reserve_limit); - } else if (i >= block->columns()) { - // This column needs to be scanned, but doesn't need to be returned upward. (delete sign) - // if i >= block->columns means the column and not the pred_column means `column i` is - // a delete condition column. but the column is not effective in the segment. so we just - // create a column to hold the data. - // a. origin data -> b. delete condition -> c. new load data - // the segment of c do not effective delete condition, but it still need read the column - // to match the schema. - // TODO: skip read the not effective delete column to speed up segment read. - _current_return_columns[cid] = Schema::get_data_type_ptr(*column_desc)->create_column(); - _current_return_columns[cid]->reserve(nrows_reserve_limit); - } - } - - // Additional deleted filter condition will be materialized column be at the end of the block, - // after _output_column_by_sel_idx will be erase, we not need to filter it, - // so erase it from _columns_to_filter in the first next_batch. - // Eg: - // `delete from table where a = 10;` - // `select b from table;` - // a column only effective in segment iterator, the block from query engine only contain the b column, - // so no need to filter a column by expr. - for (auto it = _columns_to_filter.begin(); it != _columns_to_filter.end();) { - if (*it >= block->columns()) { - it = _columns_to_filter.erase(it); - } else { - ++it; + _current_columns[i] = ReadSchema::get_predicate_column_ptr( + _storage_name_and_type[i].second, _opts.io_ctx.reader_type)); + _current_columns[i]->set_rowset_segment_id({_segment->rowset_id(), _segment->id()}); + _current_columns[i]->reserve(nrows_reserve_limit); } } @@ -637,10 +621,11 @@ void SegmentIterator::_init_segment_prefetchers() { "[verbose] SegmentIterator _init_segment_prefetchers, is_query={}, " "enable_prefetch={}, " "_row_bitmap.isEmpty()={}, row_bitmap.cardinality()={}, tablet={}, rowset={}, " - "segment={}, predicate_column_ids={}, common_expr_column_ids={}", + "segment={}, predicate_ordinals={}, common_expr_ordinals={}, output_ordinals={}", is_query, enable_prefetch, _row_bitmap.isEmpty(), _row_bitmap.cardinality(), _opts.tablet_id, _opts.rowset_id.to_string(), segment_id(), - fmt::join(_predicate_column_ids, ","), fmt::join(_common_expr_column_ids, ",")); + fmt::join(_predicate_ordinals, ","), fmt::join(_common_expr_ordinals, ","), + fmt::join(_output_ordinals, ",")); if (enable_prefetch && !_row_bitmap.isEmpty()) { int window_size = 1 + (is_query ? config::query_segment_file_cache_prefetch_block_size @@ -651,7 +636,7 @@ void SegmentIterator::_init_segment_prefetchers() { !_column_iterators.empty()) { // ensure init_iterators has been called SegmentPrefetcherConfig prefetch_config(window_size, config::file_cache_each_block_size); - for (auto cid : _schema->column_ids()) { + for (uint32_t cid = 0; cid < _schema->num_read_columns(); ++cid) { auto& column_iter = _column_iterators[cid]; if (column_iter == nullptr) { continue; @@ -686,7 +671,7 @@ void SegmentIterator::_init_segment_prefetchers() { auto* column_iter = _column_iterators[cid].get(); if (column_iter != nullptr) { ScopedColumnIteratorReadPhase scoped_read_phase { - column_iter, _support_lazy_read_pruned_columns.contains(cid) + column_iter, _has_lazy_pruned_children(cid) ? ColumnIterator::ReadPhase::PREDICATE : ColumnIterator::ReadPhase::NORMAL}; column_iter->collect_prefetchers(prefetchers, init_method); @@ -751,63 +736,56 @@ Status SegmentIterator::_get_row_ranges_by_keys() { // Set up environment for the following seek. Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_range) { - std::vector key_columns; - std::set column_set; - if (key_range.lower_key != nullptr) { - for (auto cid : key_range.lower_key->schema()->column_ids()) { - column_set.emplace(cid); - key_columns.emplace_back(key_range.lower_key->column(cid)); - } - } - if (key_range.upper_key != nullptr) { - for (auto cid : key_range.upper_key->schema()->column_ids()) { - if (column_set.count(cid) == 0) { - key_columns.emplace_back(key_range.upper_key->column(cid)); - column_set.emplace(cid); - } - } + // Both bound keys carry a dense prefix of the tablet key columns, + const doris::ReadSchema* key_schema = + key_range.lower_key != nullptr ? key_range.lower_key->schema() : nullptr; + if (key_range.upper_key != nullptr && + (key_schema == nullptr || + key_range.upper_key->schema()->num_read_columns() > key_schema->num_read_columns())) { + key_schema = key_range.upper_key->schema(); + } + if (key_schema == nullptr) { + return Status::OK(); } if (!_seek_schema) { - std::vector cols; - cols.reserve(key_columns.size()); - for (const TabletColumn* col : key_columns) { - cols.emplace_back(std::make_shared(*col)); - } - std::vector column_ids(cols.size()); - std::iota(column_ids.begin(), column_ids.end(), 0); - _seek_schema = std::make_unique(cols, column_ids); + _seek_schema = std::make_unique(*key_schema); } // todo(wb) need refactor here, when using pk to search, _seek_block is useless - if (_seek_block.size() == 0) { - _seek_block.resize(_seek_schema->num_column_ids()); - int i = 0; - for (auto cid : _seek_schema->column_ids()) { - auto column_desc = _seek_schema->column(cid); - _seek_block[i] = Schema::get_data_type_ptr(*column_desc)->create_column(); - i++; + if (_seek_block.empty()) { + _seek_block.resize(_seek_schema->num_read_columns()); + for (size_t i = 0; i < _seek_schema->num_read_columns(); ++i) { + _seek_block[i] = _seek_schema->data_type(i)->create_column(); } } - // create used column iterator - for (auto cid : _seek_schema->column_ids()) { - if (_column_iterators[cid] == nullptr) { - // TODO: Do we need this? - if (_virtual_column_exprs.contains(cid)) { - _column_iterators[cid] = std::make_unique(); - continue; - } - - RETURN_IF_ERROR(_segment->new_column_iterator(_opts.tablet_schema->column(cid), - &_column_iterators[cid], &_opts, - &_variant_sparse_column_cache)); - ColumnIteratorOptions iter_opts { - .use_page_cache = _opts.use_page_cache, - .file_reader = _file_reader.get(), - .stats = _opts.stats, - .io_ctx = _opts.io_ctx, - }; - RETURN_IF_ERROR(_column_iterators[cid]->init(iter_opts)); + // The seek key schema covers the leading key columns of the TABLET schema; + // a key column may not be part of the read schema at all. Reuse the shared + // iterator when it is, otherwise the seek path owns a dedicated one. + if (_seek_column_iterators.empty()) { + _seek_column_iterators.resize(_seek_schema->num_read_columns(), nullptr); + } + for (size_t i = 0; i < _seek_schema->num_read_columns(); ++i) { + if (_seek_column_iterators[i] != nullptr) { + continue; + } + const TabletColumn* col = _seek_schema->column(i); + int32_t read_ordinal = _schema->ordinal_by_column(*col); + if (read_ordinal >= 0 && _column_iterators[read_ordinal] != nullptr) { + _seek_column_iterators[i] = _column_iterators[read_ordinal].get(); + continue; } + std::unique_ptr iter; + RETURN_IF_ERROR( + _segment->new_column_iterator(*col, &iter, &_opts, &_variant_sparse_column_cache)); + ColumnIteratorOptions iter_opts { + .use_page_cache = _opts.use_page_cache, + .file_reader = _file_reader.get(), + .stats = _opts.stats, + .io_ctx = _opts.io_ctx, + }; + RETURN_IF_ERROR(iter->init(iter_opts)); + _seek_column_iterators[i] = iter.get(); + _owned_seek_column_iterators.emplace_back(std::move(iter)); } return Status::OK(); @@ -849,10 +827,10 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { _opts.condition_cache_digest = _common_expr_ctxs_push_down.empty() ? 0 : _opts.condition_cache_digest; _opts.stats->rows_inverted_index_filtered += (input_rows - _row_bitmap.cardinality()); - for (auto cid : _schema->column_ids()) { + for (uint32_t cid = 0; cid < _schema->num_read_columns(); ++cid) { bool result_true = _check_all_conditions_passed_inverted_index_for_column(cid); if (result_true) { - _need_read_data_indices[cid] = false; + _column_states[cid].need_read_data = false; } } } @@ -916,21 +894,20 @@ Status SegmentIterator::_apply_ann_topn_predicate() { } VLOG_DEBUG << fmt::format("Try apply ann topn: {}", _ann_topn_runtime->debug_string()); - size_t src_col_idx = _ann_topn_runtime->get_src_column_idx(); - // AnnTopNRuntime keeps VSlotRef::column_id(), which is the scan schema ordinal. - ColumnId src_cid = _schema->column_id(src_col_idx); + // AnnTopNRuntime keeps VSlotRef::column_id(), which is the read-schema ordinal. + ColumnId src_cid = cast_set(_ann_topn_runtime->get_src_column_idx()); IndexIterator* ann_index_iterator = _index_iterators[src_cid].get(); bool has_ann_index = _column_has_ann_index(src_cid); bool has_common_expr_push_down = !_common_expr_ctxs_push_down.empty(); - bool has_column_predicate = std::any_of(_is_pred_column.begin(), _is_pred_column.end(), - [](bool is_pred) { return is_pred; }); + bool has_column_predicate = std::ranges::any_of( + _column_states, [](const auto& state) { return state.is_predicate(); }); if (!has_ann_index || has_common_expr_push_down || has_column_predicate) { VLOG_DEBUG << fmt::format( "Ann topn can not be evaluated by ann index, has_ann_index: {}, " "has_common_expr_push_down: {}, has_column_predicate: {}", has_ann_index, has_common_expr_push_down, has_column_predicate); // Disable index-only scan on ann indexed column. - _need_read_data_indices[src_cid] = true; + _column_states[src_cid].need_read_data = true; _opts.stats->ann_fall_back_brute_force_cnt += 1; return Status::OK(); } @@ -944,7 +921,7 @@ Status SegmentIterator::_apply_ann_topn_predicate() { VLOG_DEBUG << fmt::format( "Asc topn for inner product can not be evaluated by ann index"); // Disable index-only scan on ann indexed column. - _need_read_data_indices[src_cid] = true; + _column_states[src_cid].need_read_data = true; _opts.stats->ann_fall_back_brute_force_cnt += 1; return Status::OK(); } @@ -952,7 +929,7 @@ Status SegmentIterator::_apply_ann_topn_predicate() { if (!_ann_topn_runtime->is_asc()) { VLOG_DEBUG << fmt::format("Desc topn for l2/cosine can not be evaluated by ann index"); // Disable index-only scan on ann indexed column. - _need_read_data_indices[src_cid] = true; + _column_states[src_cid].need_read_data = true; _opts.stats->ann_fall_back_brute_force_cnt += 1; return Status::OK(); } @@ -966,7 +943,7 @@ Status SegmentIterator::_apply_ann_topn_predicate() { metric_to_string(_ann_topn_runtime->get_metric_type()), metric_to_string(ann_index_reader->get_metric_type())); // Disable index-only scan on ann indexed column. - _need_read_data_indices[src_cid] = true; + _column_states[src_cid].need_read_data = true; _opts.stats->ann_fall_back_brute_force_cnt += 1; return Status::OK(); } @@ -982,7 +959,7 @@ Status SegmentIterator::_apply_ann_topn_predicate() { pre_size, rows_of_segment, user_params.ann_index_candidate_rows_threshold, user_params.ann_index_candidate_rows_percent_threshold); // Disable index-only scan on ann indexed column. - _need_read_data_indices[src_cid] = true; + _column_states[src_cid].need_read_data = true; _opts.stats->ann_fall_back_brute_force_cnt += 1; _opts.stats->ann_topn_fallback_by_small_candidate_cnt += 1; _opts.stats->ann_topn_fallback_small_candidate_rows += pre_size; @@ -997,7 +974,7 @@ Status SegmentIterator::_apply_ann_topn_predicate() { dynamic_cast(ann_index_iterator); if (ann_index_iterator_casted == nullptr) { VLOG_DEBUG << "Failed to cast index iterator to AnnIndexIterator, fallback to brute force"; - _need_read_data_indices[src_cid] = true; + _column_states[src_cid].need_read_data = true; _opts.stats->ann_fall_back_brute_force_cnt += 1; return Status::OK(); } @@ -1007,7 +984,7 @@ Status SegmentIterator::_apply_ann_topn_predicate() { SCOPED_TIMER(&(ann_index_stats.load_index_costs_ns)); if (!ann_index_iterator_casted->try_load_index()) { VLOG_DEBUG << "Failed to load ANN index, fallback to brute force search"; - _need_read_data_indices[src_cid] = true; + _column_states[src_cid].need_read_data = true; _opts.stats->ann_fall_back_brute_force_cnt += 1; return Status::OK(); } @@ -1044,7 +1021,7 @@ Status SegmentIterator::_apply_ann_topn_predicate() { _opts.stats->ann_index_topn_search_cnt += 1; _opts.stats->ann_index_cache_hits += ann_index_stats.topn_cache_hits.value(); const size_t dst_col_idx = _ann_topn_runtime->get_dest_column_idx(); - ColumnIterator* column_iter = _column_iterators[_schema->column_id(dst_col_idx)].get(); + ColumnIterator* column_iter = _column_iterators[dst_col_idx].get(); DCHECK(column_iter != nullptr); VirtualColumnIterator* virtual_column_iter = dynamic_cast(column_iter); DCHECK(virtual_column_iter != nullptr); @@ -1054,7 +1031,7 @@ Status SegmentIterator::_apply_ann_topn_predicate() { // reference count of result_column should be 1, so move will not issue any data copy. virtual_column_iter->prepare_materialization(std::move(result_column), result_row_ids); - _need_read_data_indices[src_cid] = false; + _column_states[src_cid].need_read_data = false; VLOG_DEBUG << fmt::format( "Enable ANN index-only scan for src column cid {} (skip reading data pages)", src_cid); @@ -1181,43 +1158,6 @@ Status SegmentIterator::_get_row_ranges_from_conditions(RowRanges* condition_row return Status::OK(); } -bool SegmentIterator::_is_literal_node(const TExprNodeType::type& node_type) { - switch (node_type) { - case TExprNodeType::BOOL_LITERAL: - case TExprNodeType::INT_LITERAL: - case TExprNodeType::LARGE_INT_LITERAL: - case TExprNodeType::FLOAT_LITERAL: - case TExprNodeType::DECIMAL_LITERAL: - case TExprNodeType::STRING_LITERAL: - case TExprNodeType::DATE_LITERAL: - case TExprNodeType::TIMEV2_LITERAL: - return true; - default: - return false; - } -} - -Status SegmentIterator::_extract_common_expr_columns(const VExprSPtr& expr) { - auto& children = expr->children(); - for (int i = 0; i < children.size(); ++i) { - RETURN_IF_ERROR(_extract_common_expr_columns(children[i])); - } - - auto node_type = expr->node_type(); - if (node_type == TExprNodeType::SLOT_REF) { - auto slot_expr = std::dynamic_pointer_cast(expr); - auto cid = _schema->column_id(slot_expr->column_id()); - _is_common_expr_column[cid] = true; - _common_expr_columns.insert(cid); - } else if (node_type == TExprNodeType::VIRTUAL_SLOT_REF) { - std::shared_ptr virtual_slot_ref = - std::dynamic_pointer_cast(expr); - RETURN_IF_ERROR(_extract_common_expr_columns(virtual_slot_ref->get_virtual_column_expr())); - } - - return Status::OK(); -} - bool SegmentIterator::_check_apply_by_inverted_index(std::shared_ptr pred) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) { return false; @@ -1245,11 +1185,6 @@ bool SegmentIterator::_check_apply_by_inverted_index(std::shared_ptr(pred.get()) != nullptr) { - return false; - } - bool handle_by_fulltext = _column_has_fulltext_index(pred_column_id); if (handle_by_fulltext) { // when predicate is leafNode of andNode, @@ -1308,9 +1243,9 @@ Status SegmentIterator::_apply_index_expr() { size_t origin_rows = _row_bitmap.cardinality(); bool ann_range_search_executed = false; RETURN_IF_ERROR(expr_ctx->evaluate_ann_range_search( - _index_iterators, _schema->column_ids(), _column_iterators, - _common_expr_to_slotref_map, num_rows(), _row_bitmap, ann_index_stats, - enable_ann_index_result_cache, &ann_range_search_executed)); + _index_iterators, _column_iterators, _common_expr_to_slotref_map, num_rows(), + _row_bitmap, ann_index_stats, enable_ann_index_result_cache, + &ann_range_search_executed)); if (ann_range_search_executed) { _opts.stats->ann_index_range_search_cnt++; } @@ -1384,7 +1319,7 @@ bool SegmentIterator::_column_has_fulltext_index(int32_t cid) { } inline bool SegmentIterator::_inverted_index_not_support_pred_type(const PredicateType& type) { - return type == PredicateType::BF; + return type == PredicateType::BF || type == PredicateType::LIKE; } Status SegmentIterator::_apply_inverted_index_on_column_predicate( @@ -1447,14 +1382,14 @@ bool SegmentIterator::_need_read_data(ColumnId cid) { if (_has_delete_predicate(cid)) { return true; } - if (_output_columns.count(-1)) { - // if _output_columns contains -1, it means that the light + if (_output_column_uids.count(-1)) { + // if _output_column_uids contains -1, it means that the light // weight schema change may not be enabled or other reasons // caused the column unique_id not be set, to prevent errors // occurring, return true here that column data needs to be read return true; } - const auto& column = _opts.tablet_schema->column(cid); + const auto& column = *_schema->column(cid); // Different subcolumns may share the same parent_unique_id, so we choose to abandon this optimization. if (column.is_extracted_column() && _opts.push_down_agg_type_opt != TPushAggOp::COUNT_ON_INDEX) { @@ -1467,19 +1402,17 @@ bool SegmentIterator::_need_read_data(ColumnId cid) { // A column can skip data reads when its predicates have already been fully resolved. // zonemap_always_true_pred_cols is produced only for non-key columns because key columns // must remain readable for short-key range seeks. - const bool used_by_common_expr = - cid < _is_common_expr_column.size() && _is_common_expr_column[cid]; + const bool used_by_common_expr = _column_states[cid].is_common_expr; const bool zonemap_always_true_filter_column = _opts.zonemap_always_true_pred_cols.contains(cid); DCHECK(!zonemap_always_true_filter_column || !column.is_key()); const bool no_need_read_filter_column = - (_need_read_data_indices.contains(cid) && !_need_read_data_indices[cid]) || + !_column_states[cid].need_read_data || (zonemap_always_true_filter_column && !used_by_common_expr); - if ((no_need_read_filter_column && !_output_columns.contains(unique_id)) || - (no_need_read_filter_column && _output_columns.count(unique_id) == 1 && + if ((no_need_read_filter_column && !_output_column_uids.contains(unique_id)) || + (no_need_read_filter_column && _output_column_uids.count(unique_id) == 1 && _opts.push_down_agg_type_opt == TPushAggOp::COUNT_ON_INDEX)) { - VLOG_DEBUG << "SegmentIterator no need read data for column: " - << _opts.tablet_schema->column_by_uid(unique_id).name(); + VLOG_DEBUG << "SegmentIterator no need read data for column: " << column.name(); return false; } return true; @@ -1487,18 +1420,13 @@ bool SegmentIterator::_need_read_data(ColumnId cid) { Status SegmentIterator::_apply_inverted_index() { std::vector> remaining_predicates; - std::set> no_need_to_pass_column_predicate_set; for (auto pred : _col_predicates) { - if (no_need_to_pass_column_predicate_set.count(pred) > 0) { - continue; - } else { - bool continue_apply = true; - RETURN_IF_ERROR(_apply_inverted_index_on_column_predicate(pred, remaining_predicates, - &continue_apply)); - if (!continue_apply) { - break; - } + bool continue_apply = true; + RETURN_IF_ERROR(_apply_inverted_index_on_column_predicate(pred, remaining_predicates, + &continue_apply)); + if (!continue_apply) { + break; } } @@ -1548,13 +1476,16 @@ bool SegmentIterator::_check_all_conditions_passed_inverted_index_for_column(Col return default_return; } -Status SegmentIterator::_init_return_column_iterators() { - SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_return_column_iterators_timer_ns); +Status SegmentIterator::_init_column_iterators() { + SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_column_iterators_timer_ns); if (_cur_rowid >= num_rows()) { return Status::OK(); } - for (auto cid : _schema->column_ids()) { + for (uint32_t cid = 0; cid < _schema->num_read_columns(); ++cid) { + if (!_is_active_read_column(cid)) { + continue; + } if (_schema->column(cid)->name() == BeConsts::ROWID_COL) { _column_iterators[cid].reset( new RowIdColumnIterator(_opts.tablet_id, _opts.rowset_id, _segment->id())); @@ -1575,28 +1506,15 @@ Status SegmentIterator::_init_return_column_iterators() { continue; } - std::set del_cond_id_set; - _opts.delete_condition_predicates->get_all_column_ids(del_cond_id_set); - std::vector tmp_is_pred_column; - tmp_is_pred_column.resize(_schema->columns().size(), false); - for (auto predicate : _col_predicates) { - auto p_cid = predicate->column_id(); - tmp_is_pred_column[p_cid] = true; - } - // handle delete_condition - for (auto d_cid : del_cond_id_set) { - tmp_is_pred_column[d_cid] = true; - } - if (_column_iterators[cid] == nullptr) { - RETURN_IF_ERROR(_segment->new_column_iterator(_opts.tablet_schema->column(cid), + RETURN_IF_ERROR(_segment->new_column_iterator(*_schema->column(cid), &_column_iterators[cid], &_opts, &_variant_sparse_column_cache)); ColumnIteratorOptions iter_opts { .use_page_cache = _opts.use_page_cache, // If the col is predicate column, then should read the last page to check // if the column is full dict encoding - .is_predicate_column = tmp_is_pred_column[cid], + .is_predicate_column = _column_states[cid].is_predicate(), .file_reader = _file_reader.get(), .stats = _opts.stats, .io_ctx = _opts.io_ctx, @@ -1638,12 +1556,13 @@ Status SegmentIterator::_init_index_iterators() { } // Inverted index iterators - for (auto cid : _schema->column_ids()) { + for (ColumnId cid = 0; cid < _schema->num_read_columns(); ++cid) { // Use segment’s own index_meta, for compatibility with future indexing needs to default to lowercase. if (_index_iterators[cid] == nullptr) { - // Scan-time Variant path placeholders retain the Variant storage type. Use their - // parent unique id and path to locate the extracted column's inverted-index metadata. - const auto& column = _opts.tablet_schema->column(cid); + // Scan-time Variant path placeholders in the read schema retain the Variant storage + // type. Use their parent unique id and path to locate the extracted column's + // inverted-index metadata. + const auto& column = *_schema->column(cid); std::vector inverted_indexs; // Keep shared_ptr alive to prevent use-after-free when accessing raw pointers TabletIndexes inverted_indexs_holder; @@ -1717,9 +1636,9 @@ Status SegmentIterator::_init_index_iterators() { } // Ann index iterators - for (auto cid : _schema->column_ids()) { + for (ColumnId cid = 0; cid < _schema->num_read_columns(); ++cid) { if (_index_iterators[cid] == nullptr) { - const auto& column = _opts.tablet_schema->column(cid); + const auto& column = *_schema->column(cid); const auto* index_meta = _segment->_tablet_schema->ann_index(column); if (index_meta) { RETURN_IF_ERROR(_segment->new_index_iterator(column, index_meta, _opts, @@ -1762,7 +1681,7 @@ Status SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool key.encode_key_with_padding(&index_key, _segment->_tablet_schema->num_short_key_columns(), is_include); - const auto& key_col_ids = key.schema()->column_ids(); + const size_t num_key_cols = key.schema()->num_read_columns(); ssize_t start_block_id = 0; auto start_iter = sk_index_decoder->lower_bound(index_key); @@ -1791,7 +1710,7 @@ Status SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool while (start < end) { rowid_t mid = (start + end) / 2; RETURN_IF_ERROR(_seek_and_peek(mid)); - int cmp = _compare_short_key_with_seek_block(key, key_col_ids); + int cmp = _compare_short_key_with_seek_block(key, num_key_cols); if (cmp > 0) { start = mid + 1; } else if (cmp == 0) { @@ -1881,30 +1800,25 @@ Status SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool return Status::OK(); } -// seek to the row and load that row to _key_cursor +// seek to the row and load that row to _seek_block Status SegmentIterator::_seek_and_peek(rowid_t rowid) { { _opts.stats->block_init_seek_num += 1; SCOPED_RAW_TIMER(&_opts.stats->block_init_seek_ns); - RETURN_IF_ERROR(_seek_columns(_seek_schema->column_ids(), rowid)); + for (size_t i = 0; i < _seek_schema->num_read_columns(); ++i) { + RETURN_IF_ERROR(_seek_column_iterators[i]->seek_to_ordinal(rowid)); + } } - size_t num_rows = 1; //note(wb) reset _seek_block for memory reuse // it is easier to use row based memory layout for clear memory - for (int i = 0; i < _seek_block.size(); i++) { + for (size_t i = 0; i < _seek_schema->num_read_columns(); ++i) { _seek_block[i]->clear(); - } - RETURN_IF_ERROR(_read_columns(_seek_schema->column_ids(), _seek_block, num_rows)); - return Status::OK(); -} - -Status SegmentIterator::_seek_columns(const std::vector& column_ids, rowid_t pos) { - for (auto cid : column_ids) { - if (!_need_read_data(cid)) { - continue; + size_t num_rows = 1; + RETURN_IF_ERROR(_seek_column_iterators[i]->next_batch(&num_rows, _seek_block[i])); + if (num_rows != 1) { + return Status::Error("nrows(1) != rows_read({})", num_rows); } - RETURN_IF_ERROR(_column_iterators[cid]->seek_to_ordinal(pos)); } return Status::OK(); } @@ -1940,185 +1854,76 @@ Status SegmentIterator::_seek_columns(const std::vector& column_ids, r // todo(wb) need a UT here Status SegmentIterator::_vec_init_lazy_materialization() { - _is_pred_column.resize(_schema->columns().size(), false); - - // including short/vec/delete pred - std::set pred_column_ids; - _lazy_materialization_read = false; - - std::set del_cond_id_set; - _opts.delete_condition_predicates->get_all_column_ids(del_cond_id_set); - - std::set> delete_predicate_set {}; - _opts.delete_condition_predicates->get_all_column_predicate(delete_predicate_set); - for (auto predicate : delete_predicate_set) { - if (PredicateTypeTraits::is_range(predicate->type())) { - _delete_range_column_ids.push_back(predicate->column_id()); - } else if (PredicateTypeTraits::is_bloom_filter(predicate->type())) { - _delete_bloom_filter_column_ids.push_back(predicate->column_id()); - } - } + // Inverted-index evaluation may have removed fully evaluated predicates + // from _col_predicates. Rebuild only the row-predicate part; delete + // predicate membership is stable for the lifetime of this iterator. + _rebuild_row_predicate_columns(); // Step1: extract columns that can be lazy materialization - if (!_col_predicates.empty() || !del_cond_id_set.empty()) { - std::set short_cir_pred_col_id_set; // using set for distinct cid - std::set vec_pred_col_id_set; - - for (auto predicate : _col_predicates) { - auto cid = predicate->column_id(); - _is_pred_column[cid] = true; - pred_column_ids.insert(cid); - - // check pred using short eval or vec eval - if (_can_evaluated_by_vectorized(predicate)) { - vec_pred_col_id_set.insert(cid); - _pre_eval_block_predicate.push_back(predicate); - } else { - short_cir_pred_col_id_set.insert(cid); - _short_cir_eval_predicate.push_back(predicate); - } - if (predicate->is_runtime_filter()) { - _filter_info_id.push_back(predicate); - } - } - - // handle delete_condition - if (!del_cond_id_set.empty()) { - short_cir_pred_col_id_set.insert(del_cond_id_set.begin(), del_cond_id_set.end()); - pred_column_ids.insert(del_cond_id_set.begin(), del_cond_id_set.end()); - - for (auto cid : del_cond_id_set) { - _is_pred_column[cid] = true; - } + for (const auto& predicate : _col_predicates) { + // check pred using short eval or vec eval + if (_can_evaluated_by_vectorized(predicate)) { + _pre_eval_block_predicate.push_back(predicate); + } else { + _short_cir_eval_predicate.push_back(predicate); } - - _vec_pred_column_ids.assign(vec_pred_col_id_set.cbegin(), vec_pred_col_id_set.cend()); - _short_cir_pred_column_ids.assign(short_cir_pred_col_id_set.cbegin(), - short_cir_pred_col_id_set.cend()); } - if (!_vec_pred_column_ids.empty()) { - _is_need_vec_eval = true; - } - if (!_short_cir_pred_column_ids.empty()) { - _is_need_short_eval = true; - } + _is_need_vec_eval = !_pre_eval_block_predicate.empty(); + _is_need_short_eval = !_short_cir_eval_predicate.empty() || + _opts.delete_condition_predicates->num_of_column_predicate() > 0; // Step2: extract columns that can execute expr context - _is_common_expr_column.resize(_schema->columns().size(), false); if (!_common_expr_ctxs_push_down.empty()) { for (const auto& expr_ctx : _common_expr_ctxs_push_down) { - RETURN_IF_ERROR(_extract_common_expr_columns(expr_ctx->root())); - } - if (!_common_expr_columns.empty()) { - _is_need_expr_eval = true; - for (auto cid : _schema->column_ids()) { - // pred column also needs to be filtered by expr, exclude additional delete condition column. - // if delete condition column not in the block, no filter is needed - // and will be removed from _columns_to_filter in the first next_batch. - if (_is_common_expr_column[cid] || _is_pred_column[cid]) { - auto loc = _schema->column_index(cid); - _columns_to_filter.push_back(loc); - - const auto field_type = _schema->column(cid)->type(); - if (_is_common_expr_column[cid] && _enable_prune_nested_column && - (field_type == FieldType::OLAP_FIELD_TYPE_STRUCT || - field_type == FieldType::OLAP_FIELD_TYPE_ARRAY || - field_type == FieldType::OLAP_FIELD_TYPE_MAP)) { - DCHECK(_column_iterators[cid]); - if (_column_iterators[cid]->read_requirement() == - ColumnIterator::ReadRequirement::PREDICATE && - _column_iterators[cid]->has_lazy_read_target()) { - // Only split lazy recovery for complex common expr columns that have - // both predicate-only and non-predicate nested targets. The two requirement - // checks already imply that nested-column pruning happened: without an - // explicit predicate sub-path the parent would not be - // PREDICATE, and without a pruned non-predicate child there - // would be no lazy target to recover after filtering. - _support_lazy_read_pruned_columns.emplace(cid); - } + _extract_common_expr_columns(expr_ctx->root()); + } + if (_is_need_expr_eval) { + for (uint32_t cid = 0; cid < _schema->num_block_columns(); ++cid) { + const auto field_type = _schema->column(cid)->type(); + if (_column_states[cid].is_common_expr && _enable_prune_nested_column && + (field_type == FieldType::OLAP_FIELD_TYPE_STRUCT || + field_type == FieldType::OLAP_FIELD_TYPE_ARRAY || + field_type == FieldType::OLAP_FIELD_TYPE_MAP)) { + DCHECK(_column_iterators[cid]); + if (_column_iterators[cid]->read_requirement() == + ColumnIterator::ReadRequirement::PREDICATE && + _column_iterators[cid]->has_lazy_read_target()) { + // Only split lazy recovery for complex common expr columns that have + // both predicate-only and non-predicate nested targets. The two requirement + // checks already imply that nested-column pruning happened: without an + // explicit predicate sub-path the parent would not be + // PREDICATE, and without a pruned non-predicate child there + // would be no lazy target to recover after filtering. + _lazy_pruned_ordinals.push_back(cid); } } } - - for (const auto& entry : _virtual_column_exprs) { - _columns_to_filter.push_back(_schema->column_index(entry.first)); - } - } - } - - // Step 3: fill non predicate columns and second read column - // if _schema columns size equal to pred_column_ids size, lazy_materialization_read is false, - // all columns are lazy materialization columns without non predicte column. - // If common expr pushdown exists, and expr column is not contained in lazy materialization columns, - // add to second read column, which will be read after lazy materialization - if (_schema->column_ids().size() > pred_column_ids.size()) { - // pred_column_ids maybe empty, so that could not set _lazy_materialization_read = true here - // has to check there is at least one predicate column - for (auto cid : _schema->column_ids()) { - if (!_is_pred_column[cid]) { - if (_is_need_vec_eval || _is_need_short_eval) { - _lazy_materialization_read = true; - } - if (_is_common_expr_column[cid]) { - _common_expr_column_ids.push_back(cid); - } else { - _non_predicate_columns.push_back(cid); - } - } } } - // Step 4: fill first read columns - if (_lazy_materialization_read) { - // insert pred cid to first_read_columns - for (auto cid : pred_column_ids) { - _predicate_column_ids.push_back(cid); - } - } else if (!_is_need_vec_eval && !_is_need_short_eval && !_is_need_expr_eval) { - for (int i = 0; i < _schema->num_column_ids(); i++) { - auto cid = _schema->column_id(i); - _predicate_column_ids.push_back(cid); + // Step 3: assign every active column to its earliest materialization role. + for (uint32_t cid = 0; cid < _schema->num_read_columns(); ++cid) { + if (!_is_active_read_column(cid)) { + continue; } - } else { - if (_is_need_vec_eval || _is_need_short_eval) { - // TODO To refactor, because we suppose lazy materialization is better performance. - // pred exits, but we can eliminate lazy materialization - // insert pred/non-pred cid to first read columns - std::set pred_id_set; - pred_id_set.insert(_short_cir_pred_column_ids.begin(), - _short_cir_pred_column_ids.end()); - pred_id_set.insert(_vec_pred_column_ids.begin(), _vec_pred_column_ids.end()); - - DCHECK(_common_expr_column_ids.empty()); - // _non_predicate_column_ids must be empty. Otherwise _lazy_materialization_read must not false. - for (int i = 0; i < _schema->num_column_ids(); i++) { - auto cid = _schema->column_id(i); - if (pred_id_set.find(cid) != pred_id_set.end()) { - _predicate_column_ids.push_back(cid); - } - } - } else if (_is_need_expr_eval) { - DCHECK(!_is_need_vec_eval && !_is_need_short_eval); - for (auto cid : _common_expr_columns) { - _predicate_column_ids.push_back(cid); - } + if (_column_states[cid].is_predicate()) { + _predicate_ordinals.push_back(cid); + } else if (_column_states[cid].is_common_expr) { + _common_expr_ordinals.push_back(cid); + } else { + _output_ordinals.push_back(cid); } } VLOG_DEBUG << fmt::format( "Laze materialization init end. " - "lazy_materialization_read: {}, " "_col_predicates size: {}, " - "_cols_read_by_column_predicate: [{}], " - "_non_predicate_columns: [{}], " - "_cols_read_by_common_expr: [{}], " - "columns_to_filter: [{}], " - "schema_column_id_to_index: [{}]", - _lazy_materialization_read, _col_predicates.size(), - fmt::join(_predicate_column_ids, ","), fmt::join(_non_predicate_columns, ","), - fmt::join(_common_expr_column_ids, ","), fmt::join(_columns_to_filter, ","), - fmt::join(_schema->column_id_to_index(), ",")); + "_predicate_ordinals: [{}], " + "_common_expr_ordinals: [{}], " + "_output_ordinals: [{}]", + _col_predicates.size(), fmt::join(_predicate_ordinals, ","), + fmt::join(_common_expr_ordinals, ","), fmt::join(_output_ordinals, ",")); return Status::OK(); } @@ -2136,7 +1941,8 @@ bool SegmentIterator::_can_evaluated_by_vectorized(std::shared_ptr& column_ids, - MutableColumns& column_block, size_t nrows) { - for (auto cid : column_ids) { - auto& column = column_block[cid]; - size_t rows_read = nrows; - if (_prune_column(cid, column, rows_read)) { - continue; - } - RETURN_IF_ERROR(_column_iterators[cid]->next_batch(&rows_read, column)); - if (nrows != rows_read) { - return Status::Error("nrows({}) != rows_read({})", nrows, - rows_read); - } - } - return Status::OK(); + return !_virtual_column_exprs.contains(cid) && !_column_states[cid].is_predicate() && + !_column_states[cid].is_common_expr; } Status SegmentIterator::_init_current_block(Block* block, std::vector& current_columns, uint32_t nrows_read_limit) { - block->clear_column_data(_schema->num_column_ids()); + block->clear_column_data(cast_set(_schema->num_block_columns())); - for (size_t i = 0; i < _schema->num_column_ids(); i++) { - auto cid = _schema->column_id(i); - const auto* column_desc = _schema->column(cid); + for (ColumnId i = 0; i < _schema->num_read_columns(); i++) { + if (!_is_active_read_column(i)) { + continue; + } + const auto* column_desc = _schema->column(i); + + const auto& file_column_type = _storage_name_and_type[i].second; + const auto& expected_type = _schema->data_type(i); + if (_column_states[i].is_predicate()) { + if (current_columns[i].get() == nullptr) { + return Status::InternalError("SegmentIterator meet invalid column, id={}, name={}", + i, column_desc->name()); + } + current_columns[i]->clear(); + continue; + } - auto file_column_type = _storage_name_and_type[cid].second; - auto expected_type = Schema::get_data_type_ptr(*column_desc); - if (!_is_pred_column[cid] && !file_column_type->equals(*expected_type)) { - // The storage layer type is different from schema needed type, so we use storage - // type to read columns instead of schema type for safety + DCHECK_LT(i, _schema->num_block_columns()); + if (!file_column_type->equals(*expected_type)) { + // The column iterator writes a different type from the read-schema type, so + // materialize into an intermediate column and convert it after reading. VLOG_DEBUG << fmt::format( - "Recreate column with expected type {}, file column type {}, col_name {}, " + "Recreate column with expected type {}, materialization type {}, col_name {}, " "col_path {}", block->get_by_position(i).type->get_name(), file_column_type->get_name(), column_desc->name(), @@ -2228,22 +2027,11 @@ Status SegmentIterator::_init_current_block(Block* block, ? "" : column_desc->path_info_ptr()->get_path()); // TODO reuse - current_columns[cid] = file_column_type->create_column(); - current_columns[cid]->reserve(nrows_read_limit); + current_columns[i] = file_column_type->create_column(); + current_columns[i]->reserve(nrows_read_limit); } else { - // the column in block must clear() here to insert new data - if (_is_pred_column[cid] || - i >= block->columns()) { //todo(wb) maybe we can release it after output block - if (current_columns[cid].get() == nullptr) { - return Status::InternalError( - "SegmentIterator meet invalid column, id={}, name={}", cid, - _schema->column(cid)->name()); - } - current_columns[cid]->clear(); - } else { // non-predicate column - current_columns[cid] = std::move(*block->get_by_position(i).column).mutate(); - current_columns[cid]->reserve(nrows_read_limit); - } + current_columns[i] = std::move(*block->get_by_position(i).column).mutate(); + current_columns[i]->reserve(nrows_read_limit); } } @@ -2256,50 +2044,30 @@ Status SegmentIterator::_init_current_block(Block* block, return Status::OK(); } -Status SegmentIterator::_output_non_pred_columns(Block* block) { +Status SegmentIterator::_output_columns_to_block(Block* block) { SCOPED_RAW_TIMER(&_opts.stats->output_col_ns); - VLOG_DEBUG << fmt::format( - "Output non-predicate columns, _non_predicate_columns: [{}], " - "schema_column_id_to_index: [{}]", - fmt::join(_non_predicate_columns, ","), fmt::join(_schema->column_id_to_index(), ",")); - RETURN_IF_ERROR(_convert_to_expected_type(_non_predicate_columns)); - for (auto cid : _non_predicate_columns) { - auto loc = _schema->column_index(cid); - // Whether a delete predicate column gets output depends on how the caller builds - // the block passed to next_batch(). Both calling paths now build the block with - // only the output schema (return_columns), so delete predicate columns are skipped: - // - // 1) VMergeIterator path: block_reset() builds _block using the output schema - // (return_columns only), e.g. block has 2 columns {c1, c2}. - // Here loc=2 for delete predicate c3, block->columns()=2, so loc < block->columns() - // is false, and c3 is skipped. - // - // 2) VUnionIterator path: the caller's block is built with only return_columns - // (output schema), e.g. block has 2 columns {c1, c2}. - // Here loc=2 for c3, block->columns()=2, so loc < block->columns() is false, - // and c3 is skipped — same behavior as the VMergeIterator path. - if (loc < block->columns()) { - bool column_in_block_is_nothing = check_and_get_column( - block->get_by_position(loc).column.get()); - bool column_is_normal = !_virtual_column_exprs.contains(cid); - bool return_column_is_nothing = - check_and_get_column(_current_return_columns[cid].get()); - VLOG_DEBUG << fmt::format( - "Cid {} loc {}, column_in_block_is_nothing {}, column_is_normal {}, " - "return_column_is_nothing {}", - cid, loc, column_in_block_is_nothing, column_is_normal, - return_column_is_nothing); - - if (column_in_block_is_nothing || column_is_normal) { - block->replace_by_position(loc, std::move(_current_return_columns[cid])); - VLOG_DEBUG << fmt::format( - "Output non-predicate column, cid: {}, loc: {}, col_name: {}, rows {}", cid, - loc, _schema->column(cid)->name(), - block->get_by_position(loc).column->size()); - } - // Means virtual column in block has been materialized(maybe by common expr). - // so do nothing here. + VLOG_DEBUG << fmt::format("Output columns, ordinals: [{}]", fmt::join(_output_ordinals, ",")); + RETURN_IF_ERROR(_convert_to_expected_type(_output_ordinals)); + for (auto cid : _output_ordinals) { + DCHECK_LT(cid, _schema->num_block_columns()); + bool column_in_block_is_nothing = + check_and_get_column(block->get_by_position(cid).column.get()); + bool column_is_normal = !_virtual_column_exprs.contains(cid); + bool current_column_is_nothing = + check_and_get_column(_current_columns[cid].get()); + VLOG_DEBUG << fmt::format( + "Cid {}, column_in_block_is_nothing {}, column_is_normal {}, " + "current_column_is_nothing {}", + cid, column_in_block_is_nothing, column_is_normal, current_column_is_nothing); + + if (column_in_block_is_nothing || column_is_normal) { + block->replace_by_position(cid, std::move(_current_columns[cid])); + VLOG_DEBUG << fmt::format("Output non-predicate column, cid: {}, col_name: {}, rows {}", + cid, _schema->column(cid)->name(), + block->get_by_position(cid).column->size()); } + // Means virtual column in block has been materialized(maybe by common expr). + // so do nothing here. } return Status::OK(); } @@ -2314,7 +2082,7 @@ Status SegmentIterator::_output_non_pred_columns(Block* block) { * 1. Reads a batch of rowids (up to the specified limit), and checks if they are continuous. * Continuous here means that the rowids form an unbroken sequence (e.g., 1, 2, 3, 4...). * - * 2. For each column that needs to be read (identified by _predicate_column_ids): + * 2. For each column in read_ordinals: * - If the rowids are continuous, the function uses seek_to_ordinal and next_batch * for efficient reading. * - If the rowids are not continuous, the function processes them in smaller batches @@ -2325,7 +2093,8 @@ Status SegmentIterator::_output_non_pred_columns(Block* block) { * This approach optimizes reading performance by leveraging batch processing for continuous * rowid sequences and handling discontinuities gracefully in smaller chunks. */ -Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16_t& nrows_read) { +Status SegmentIterator::_read_columns_by_index(const std::vector& read_ordinals, + uint32_t nrows_read_limit, uint16_t& nrows_read) { SCOPED_RAW_TIMER(&_opts.stats->predicate_column_read_ns); nrows_read = (uint16_t)_range_iter->read_batch_rowids(_block_rowids.data(), nrows_read_limit); @@ -2333,17 +2102,16 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16 (_block_rowids[nrows_read - 1] - _block_rowids[0] == nrows_read - 1); VLOG_DEBUG << fmt::format( "nrows_read from range iterator: {}, is_continus {}, " - "_cols_read_by_column_predicate " - "[{}]", - nrows_read, is_continuous, fmt::join(_predicate_column_ids, ",")); + "read_ordinals: [{}]", + nrows_read, is_continuous, fmt::join(read_ordinals, ",")); LOG_IF(INFO, config::enable_segment_prefetch_verbose_log) << fmt::format( "[verbose] SegmentIterator::_read_columns_by_index read {} rowids, continuous: {}, " "rowids: [{}...{}]", nrows_read, is_continuous, nrows_read > 0 ? _block_rowids[0] : 0, nrows_read > 0 ? _block_rowids[nrows_read - 1] : 0); - for (auto cid : _predicate_column_ids) { - auto& column = _current_return_columns[cid]; + for (auto cid : read_ordinals) { + auto& column = _current_columns[cid]; VLOG_DEBUG << fmt::format("Reading column {}, col_name {}", cid, _schema->column(cid)->name()); if (!_virtual_column_exprs.contains(cid)) { @@ -2356,7 +2124,7 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16 continue; } DBUG_EXECUTE_IF("segment_iterator._read_columns_by_index", { - auto col_name = _opts.tablet_schema->column(cid).name(); + auto col_name = _schema->column(cid)->name(); auto debug_col_name = DebugPoints::instance()->get_debug_param_or_default( "segment_iterator._read_columns_by_index", "column_name", ""); @@ -2373,9 +2141,8 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16 auto* column_iter = _column_iterators[cid].get(); ScopedColumnIteratorReadPhase scoped_read_phase { - column_iter, _support_lazy_read_pruned_columns.contains(cid) - ? ColumnIterator::ReadPhase::PREDICATE - : ColumnIterator::ReadPhase::NORMAL}; + column_iter, _has_lazy_pruned_children(cid) ? ColumnIterator::ReadPhase::PREDICATE + : ColumnIterator::ReadPhase::NORMAL}; if (is_continuous) { size_t rows_read = nrows_read; @@ -2426,7 +2193,7 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16 return Status::OK(); } -void SegmentIterator::_replace_version_col_if_needed(const std::vector& column_ids, +void SegmentIterator::_replace_version_col_if_needed(const std::vector& ordinals, size_t num_rows) { // Only the rowset with single version need to replace the version column. // Doris can't determine the version before publish_version finished, so @@ -2436,23 +2203,24 @@ void SegmentIterator::_replace_version_col_if_needed(const std::vector if (_opts.version.first != _opts.version.second) { return; } - int32_t version_idx = _schema->version_col_idx(); - if (std::ranges::find(column_ids, version_idx) == column_ids.end()) { + int32_t version_ordinal = _schema->version_ordinal(); + if (version_ordinal < 0 || std::ranges::find(ordinals, version_ordinal) == ordinals.end()) { return; } - const auto* column_desc = _schema->column(version_idx); - auto column = Schema::get_data_type_ptr(*column_desc)->create_column(); - DCHECK(_schema->column(version_idx)->type() == FieldType::OLAP_FIELD_TYPE_BIGINT); + const auto* column_desc = _schema->column(version_ordinal); + auto column = _schema->data_type(version_ordinal)->create_column(); + DCHECK(column_desc->type() == FieldType::OLAP_FIELD_TYPE_BIGINT); auto* col_ptr = assert_cast(column.get()); for (size_t j = 0; j < num_rows; j++) { col_ptr->insert_value(_opts.version.second); } - _current_return_columns[version_idx] = std::move(column); - VLOG_DEBUG << "replaced version column in segment iterator, version_col_idx:" << version_idx; + _current_columns[version_ordinal] = std::move(column); + VLOG_DEBUG << "replaced version column in segment iterator, version_ordinal:" + << version_ordinal; } -void SegmentIterator::_update_tso_col_if_needed(const std::vector& column_ids, +void SegmentIterator::_update_tso_col_if_needed(const std::vector& ordinals, size_t num_rows) { // use physical time part of commit tso to replace timestamp col if (_opts.version.first != _opts.version.second) { @@ -2463,19 +2231,19 @@ void SegmentIterator::_update_tso_col_if_needed(const std::vector& col return; } - int32_t tso_col_idx = _schema->tso_col_idx(); - if (tso_col_idx < 0 || std::ranges::find(column_ids, tso_col_idx) == column_ids.end()) { + int32_t tso_ordinal = _schema->tso_ordinal(); + if (tso_ordinal < 0 || std::ranges::find(ordinals, tso_ordinal) == ordinals.end()) { return; } DCHECK_EQ(_opts.commit_tso.start_tso(), _opts.commit_tso.end_tso()); Int64 commit_tso = _opts.commit_tso.end_tso() == -1 ? 0 : _opts.commit_tso.end_tso(); - if (_is_pred_column[tso_col_idx]) { + if (_column_states[tso_ordinal].is_predicate()) { // Nullable predicate column is represented as ColumnNullable(predicate_col) - if (auto* tso_nullable = check_and_get_column( - _current_return_columns[tso_col_idx].get())) { - _current_return_columns[tso_col_idx]->clear(); + if (auto* tso_nullable = + check_and_get_column(_current_columns[tso_ordinal].get())) { + _current_columns[tso_ordinal]->clear(); auto value = commit_tso; for (size_t j = 0; j < num_rows; j++) { tso_nullable->get_nested_column_ptr()->insert_data( @@ -2485,7 +2253,7 @@ void SegmentIterator::_update_tso_col_if_needed(const std::vector& col return; } - auto* tso_column = assert_cast(_current_return_columns[tso_col_idx].get()); + auto* tso_column = assert_cast(_current_columns[tso_ordinal].get()); tso_column->clear(); auto value = commit_tso; for (size_t j = 0; j < num_rows; j++) { @@ -2494,8 +2262,8 @@ void SegmentIterator::_update_tso_col_if_needed(const std::vector& col return; } - const auto* column_desc = _schema->column(tso_col_idx); - auto column = Schema::get_data_type_ptr(*column_desc)->create_column(); + const auto* column_desc = _schema->column(tso_ordinal); + auto column = _schema->data_type(tso_ordinal)->create_column(); DCHECK(column_desc->type() == FieldType::OLAP_FIELD_TYPE_BIGINT); if (auto* tso_nullable = check_and_get_column(column.get())) { @@ -2510,7 +2278,7 @@ void SegmentIterator::_update_tso_col_if_needed(const std::vector& col col_ptr->insert_value(commit_tso); } } - _current_return_columns[tso_col_idx] = std::move(column); + _current_columns[tso_ordinal] = std::move(column); } uint16_t SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, @@ -2544,7 +2312,7 @@ uint16_t SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_ continue; } auto column_id = pred->column_id(); - auto& column = _current_return_columns[column_id]; + auto& column = _current_columns[column_id]; if (is_first) { pred->evaluate_vec(*column, original_size, (bool*)_ret_flags.data()); is_first = false; @@ -2599,7 +2367,7 @@ uint16_t SegmentIterator::_evaluate_short_circuit_predicate(uint16_t* vec_sel_ro uint16_t original_size = selected_size; for (auto predicate : _short_cir_eval_predicate) { auto column_id = predicate->column_id(); - auto& short_cir_column = _current_return_columns[column_id]; + auto& short_cir_column = _current_columns[column_id]; selected_size = predicate->evaluate(*short_cir_column, vec_sel_rowid_idx, selected_size); } @@ -2608,8 +2376,8 @@ uint16_t SegmentIterator::_evaluate_short_circuit_predicate(uint16_t* vec_sel_ro // evaluate delete condition original_size = selected_size; - selected_size = _opts.delete_condition_predicates->evaluate(_current_return_columns, - vec_sel_rowid_idx, selected_size); + selected_size = _opts.delete_condition_predicates->evaluate(_current_columns, vec_sel_rowid_idx, + selected_size); _opts.stats->rows_vec_del_cond_filtered += original_size - selected_size; return selected_size; } @@ -2661,7 +2429,7 @@ Status SegmentIterator::_apply_read_limit_to_selected_rows(Block* block, uint16_ return Status::OK(); } -Status SegmentIterator::_read_columns_by_rowids(std::vector& read_column_ids, +Status SegmentIterator::_read_columns_by_rowids(const std::vector& read_ordinals, std::vector& rowid_vector, uint16_t* sel_rowid_idx, size_t select_size, MutableColumns* mutable_columns, @@ -2683,7 +2451,7 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector& read_colu } } - for (auto cid : read_column_ids) { + for (auto cid : read_ordinals) { auto& colunm = (*mutable_columns)[cid]; if (_no_need_read_key_data(cid, colunm, select_size)) { continue; @@ -2698,34 +2466,34 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector& read_colu if (debug_col_name.empty()) { return Status::Error("does not need to read data"); } - auto col_name = _opts.tablet_schema->column(cid).name(); + auto col_name = _schema->column(cid)->name(); if (debug_col_name.find(col_name) != std::string::npos) { return Status::Error("does not need to read data, {}", debug_col_name); } }) - if (_current_return_columns[cid].get() == nullptr) { + if (_current_columns[cid].get() == nullptr) { return Status::InternalError( - "SegmentIterator meet invalid column, return columns size {}, cid {}", - _current_return_columns.size(), cid); + "SegmentIterator met invalid current column, schema size {}, ordinal {}", + _current_columns.size(), cid); } auto* column_iter = _column_iterators[cid].get(); ScopedColumnIteratorReadPhase scoped_read_phase { - column_iter, read_for_predicate && _support_lazy_read_pruned_columns.contains(cid) + column_iter, read_for_predicate && _has_lazy_pruned_children(cid) ? ColumnIterator::ReadPhase::PREDICATE : ColumnIterator::ReadPhase::NORMAL}; - RETURN_IF_ERROR(column_iter->read_by_rowids(rowids.data(), select_size, - _current_return_columns[cid])); + RETURN_IF_ERROR( + column_iter->read_by_rowids(rowids.data(), select_size, _current_columns[cid])); } return Status::OK(); } Status SegmentIterator::_read_lazy_pruned_columns(Block* block) { - if (_support_lazy_read_pruned_columns.empty()) { + if (_lazy_pruned_ordinals.empty()) { return Status::OK(); } @@ -2735,9 +2503,9 @@ Status SegmentIterator::_read_lazy_pruned_columns(Block* block) { rowids[i] = _block_rowids[_sel_rowid_idx[i]]; } - for (auto cid : _support_lazy_read_pruned_columns) { - auto loc = _schema->column_index(cid); - auto column = IColumn::mutate(std::move(block->get_by_position(loc).column)); + for (auto cid : _lazy_pruned_ordinals) { + DCHECK_LT(cid, _schema->num_block_columns()); + auto column = IColumn::mutate(std::move(block->get_by_position(cid).column)); auto* column_iter = _column_iterators[cid].get(); ScopedColumnIteratorReadPhase scoped_read_phase {column_iter, ColumnIterator::ReadPhase::LAZY}; @@ -2745,7 +2513,7 @@ Status SegmentIterator::_read_lazy_pruned_columns(Block* block) { RETURN_IF_ERROR(column_iter->read_by_rowids(rowids.data(), _selected_size, column)); } column_iter->finalize_lazy_phase(column); - block->get_by_position(loc).column = std::move(column); + block->get_by_position(cid).column = std::move(column); } return Status::OK(); } @@ -2783,9 +2551,9 @@ Status SegmentIterator::next_batch(Block* block) { // Since we have a type check at the caller. // So a replacement of nothing column with real column is needed. for (const auto& [cid, expr_ctx] : _virtual_column_exprs) { - auto idx = _schema->column_index(cid); + DCHECK_LT(cid, _schema->num_block_columns()); auto type = expr_ctx->root()->data_type(); - block->replace_by_position(idx, type->create_column()); + block->replace_by_position(cid, type->create_column()); } if (_opts.condition_cache_digest && !_find_condition_cache) { @@ -2839,28 +2607,32 @@ Status SegmentIterator::next_batch(Block* block) { return status; } -Status SegmentIterator::_convert_to_expected_type(const std::vector& col_ids) { - for (ColumnId i : col_ids) { - if (!_current_return_columns[i] || _converted_column_ids[i] || _is_pred_column[i]) { - continue; - } - const TabletColumn* column_desc = _schema->column(i); - DataTypePtr expected_type = Schema::get_data_type_ptr(*column_desc); - DataTypePtr file_column_type = _storage_name_and_type[i].second; - if (!file_column_type->equals(*expected_type)) { - ColumnPtr expected; - ColumnPtr original = _current_return_columns[i]->assert_mutable()->get_ptr(); - RETURN_IF_ERROR(variant_util::cast_column({original, file_column_type, ""}, - expected_type, &expected)); - _current_return_columns[i] = expected->assert_mutable(); - _converted_column_ids[i] = true; - VLOG_DEBUG << fmt::format("Convert {} fom file column type {} to {}, num_rows {}", - column_desc->path_info_ptr() == nullptr - ? "" - : column_desc->path_info_ptr()->get_path(), - file_column_type->get_name(), expected_type->get_name(), - _current_return_columns[i]->size()); - } +Status SegmentIterator::_convert_column_to_expected_type(ColumnId column_id) { + if (!_current_columns[column_id] || _column_states[column_id].is_predicate()) { + return Status::OK(); + } + const TabletColumn* column_desc = _schema->column(column_id); + const auto& expected_type = _schema->data_type(column_id); + const auto& file_column_type = _storage_name_and_type[column_id].second; + if (!file_column_type->equals(*expected_type)) { + ColumnPtr expected; + ColumnPtr original = _current_columns[column_id]->assert_mutable()->get_ptr(); + RETURN_IF_ERROR(variant_util::cast_column({original, file_column_type, ""}, expected_type, + &expected)); + _current_columns[column_id] = expected->assert_mutable(); + VLOG_DEBUG << fmt::format("Convert {} from materialization type {} to {}, num_rows {}", + column_desc->path_info_ptr() == nullptr + ? "" + : column_desc->path_info_ptr()->get_path(), + file_column_type->get_name(), expected_type->get_name(), + _current_columns[column_id]->size()); + } + return Status::OK(); +} + +Status SegmentIterator::_convert_to_expected_type(const std::vector& ordinals) { + for (ColumnId ordinal : ordinals) { + RETURN_IF_ERROR(_convert_column_to_expected_type(ordinal)); } return Status::OK(); } @@ -2914,13 +2686,19 @@ Status SegmentIterator::_next_batch_internal(Block* block) { } }) - RETURN_IF_ERROR(_init_current_block(block, _current_return_columns, nrows_read_limit)); - _converted_column_ids.assign(_schema->columns().size(), false); + RETURN_IF_ERROR(_init_current_block(block, _current_columns, nrows_read_limit)); + + const bool need_predicate_eval = _is_need_vec_eval || _is_need_short_eval; + const bool has_residual_eval = need_predicate_eval || _is_need_expr_eval; + const auto& initial_read_ordinals = + need_predicate_eval ? _predicate_ordinals + : (_is_need_expr_eval ? _common_expr_ordinals : _output_ordinals); _selected_size = 0; - RETURN_IF_ERROR(_read_columns_by_index(nrows_read_limit, _selected_size)); - _replace_version_col_if_needed(_predicate_column_ids, _selected_size); - _update_tso_col_if_needed(_predicate_column_ids, _selected_size); + RETURN_IF_ERROR( + _read_columns_by_index(initial_read_ordinals, nrows_read_limit, _selected_size)); + _replace_version_col_if_needed(initial_read_ordinals, _selected_size); + _update_tso_col_if_needed(initial_read_ordinals, _selected_size); _opts.stats->blocks_load += 1; _opts.stats->raw_rows_read += _selected_size; @@ -2929,10 +2707,10 @@ Status SegmentIterator::_next_batch_internal(Block* block) { return _process_eof(block); } - if (_is_need_vec_eval || _is_need_short_eval || _is_need_expr_eval) { + if (has_residual_eval) { _sel_rowid_idx.resize(_selected_size); - if (_is_need_vec_eval || _is_need_short_eval) { + if (need_predicate_eval) { _convert_dict_code_for_predicate_if_necessary(); // step 1: evaluate vectorization predicate @@ -2949,38 +2727,33 @@ Status SegmentIterator::_next_batch_internal(Block* block) { _selected_size); if (_selected_size > 0) { // step 3.1: output short circuit and predicate column - // when lazy materialization enables, _predicate_column_ids = distinct(_short_cir_pred_column_ids + _vec_pred_column_ids) - // see _vec_init_lazy_materialization - // todo(wb) need to tell input columnids from output columnids - RETURN_IF_ERROR(_output_column_by_sel_idx(block, _predicate_column_ids, + RETURN_IF_ERROR(_output_column_by_sel_idx(block, _predicate_ordinals, _sel_rowid_idx.data(), _selected_size)); // step 3.2: read remaining expr column and evaluate it. if (_is_need_expr_eval) { // The predicate column contains the remaining expr column, no need second read. - if (_common_expr_column_ids.size() > 0) { + if (!_common_expr_ordinals.empty()) { SCOPED_RAW_TIMER(&_opts.stats->non_predicate_read_ns); RETURN_IF_ERROR(_read_columns_by_rowids( - _common_expr_column_ids, _block_rowids, _sel_rowid_idx.data(), - _selected_size, &_current_return_columns, false, true)); - _replace_version_col_if_needed(_common_expr_column_ids, _selected_size); - _update_tso_col_if_needed(_common_expr_column_ids, _selected_size); - RETURN_IF_ERROR(_process_columns(_common_expr_column_ids, block)); + _common_expr_ordinals, _block_rowids, _sel_rowid_idx.data(), + _selected_size, &_current_columns, false, true)); + _replace_version_col_if_needed(_common_expr_ordinals, _selected_size); + _update_tso_col_if_needed(_common_expr_ordinals, _selected_size); + RETURN_IF_ERROR(_process_columns(_common_expr_ordinals, block)); } - DCHECK(block->columns() > _schema->column_index(*_common_expr_columns.begin())); RETURN_IF_ERROR( _process_common_expr(_sel_rowid_idx.data(), _selected_size, block)); } } else { _fill_column_nothing(); if (_is_need_expr_eval) { - RETURN_IF_ERROR(_process_columns(_common_expr_column_ids, block)); + RETURN_IF_ERROR(_process_columns(_common_expr_ordinals, block)); } } } else if (_is_need_expr_eval) { - DCHECK(!_predicate_column_ids.empty()); - RETURN_IF_ERROR(_process_columns(_predicate_column_ids, block)); + RETURN_IF_ERROR(_process_columns(_common_expr_ordinals, block)); // first read all rows are insert block, initialize sel_rowid_idx to all rows. for (uint16_t i = 0; i < _selected_size; ++i) { _sel_rowid_idx[i] = i; @@ -2990,15 +2763,15 @@ Status SegmentIterator::_next_batch_internal(Block* block) { RETURN_IF_ERROR(_apply_read_limit_to_selected_rows(block, _selected_size)); - // step4: read non_predicate column + // step4: read deferred output columns if (_selected_size > 0) { - if (!_non_predicate_columns.empty()) { + if (!_output_ordinals.empty()) { RETURN_IF_ERROR(_read_columns_by_rowids( - _non_predicate_columns, _block_rowids, _sel_rowid_idx.data(), - _selected_size, &_current_return_columns, - _opts.condition_cache_digest && !_find_condition_cache, false)); - _replace_version_col_if_needed(_non_predicate_columns, _selected_size); - _update_tso_col_if_needed(_non_predicate_columns, _selected_size); + _output_ordinals, _block_rowids, _sel_rowid_idx.data(), _selected_size, + &_current_columns, _opts.condition_cache_digest && !_find_condition_cache, + false)); + _replace_version_col_if_needed(_output_ordinals, _selected_size); + _update_tso_col_if_needed(_output_ordinals, _selected_size); } else { if (_opts.condition_cache_digest && !_find_condition_cache) { auto& condition_cache = *_condition_cache; @@ -3014,7 +2787,7 @@ Status SegmentIterator::_next_batch_internal(Block* block) { } // step5: output columns - RETURN_IF_ERROR(_output_non_pred_columns(block)); + RETURN_IF_ERROR(_output_columns_to_block(block)); // Convert inverted index bitmaps to result columns for virtual column exprs // (e.g., MATCH projections). This must run before _materialization_of_virtual_column // so that fast_execute() can find the pre-computed result columns. @@ -3035,24 +2808,24 @@ Status SegmentIterator::_next_batch_internal(Block* block) { return _check_output_block(block); } -Status SegmentIterator::_process_columns(const std::vector& column_ids, Block* block) { - RETURN_IF_ERROR(_convert_to_expected_type(column_ids)); - for (auto cid : column_ids) { - auto loc = _schema->column_index(cid); - block->replace_by_position(loc, std::move(_current_return_columns[cid])); +Status SegmentIterator::_process_columns(const std::vector& ordinals, Block* block) { + RETURN_IF_ERROR(_convert_to_expected_type(ordinals)); + for (auto cid : ordinals) { + DCHECK_LT(cid, _schema->num_block_columns()); + block->replace_by_position(cid, std::move(_current_columns[cid])); } return Status::OK(); } void SegmentIterator::_fill_column_nothing() { - // If column_predicate filters out all rows, the corresponding column in _current_return_columns[cid] must be a ColumnNothing. - // Because: - // 1. Before each batch, _init_return_columns is called to initialize _current_return_columns, and virtual columns in _current_return_columns are initialized as ColumnNothing. - // 2. When select_size == 0, the read method of VirtualColumnIterator will definitely not be called, so the corresponding Column remains a ColumnNothing + // If column predicates filter out all rows, each virtual column in + // _current_columns remains a ColumnNothing because its iterator is not read. + // _init_current_block initializes virtual columns as ColumnNothing. When + // select_size == 0, VirtualColumnIterator is not called, so they stay so. for (const auto& [cid, expr_ctx] : _virtual_column_exprs) { [[maybe_unused]] const auto* nothing_col = - assert_cast(_current_return_columns[cid].get()); - _current_return_columns[cid] = expr_ctx->root()->data_type()->create_column(); + assert_cast(_current_columns[cid].get()); + _current_columns[cid] = expr_ctx->root()->data_type()->create_column(); } } @@ -3065,18 +2838,19 @@ Status SegmentIterator::_check_output_block(Block* block) { return Status::InternalError( "Column in idx {} is null, block columns {}, normal_columns {}, " "virtual_columns {}", - idx, block->columns(), _schema->num_column_ids(), _virtual_column_exprs.size()); + idx, block->columns(), _schema->num_block_columns(), + _virtual_column_exprs.size()); } else if (check_and_get_column(entry.column.get())) { if (rows > 0) { - std::vector virtual_column_ids; + std::vector virtual_column_ordinals; for (const auto& pair : _virtual_column_exprs) { - virtual_column_ids.push_back(pair.first); + virtual_column_ordinals.push_back(pair.first); } return Status::InternalError( "Column in idx {} is nothing, block columns {}, normal_columns {}, " - "virtual_column_ids [{}]", - idx, block->columns(), _schema->num_column_ids(), - fmt::join(virtual_column_ids, ",")); + "virtual_column_ordinals [{}]", + idx, block->columns(), _schema->num_block_columns(), + fmt::join(virtual_column_ordinals, ",")); } } else if (entry.column->size() != rows) { return Status::InternalError( @@ -3092,12 +2866,16 @@ Status SegmentIterator::_check_output_block(Block* block) { } Status SegmentIterator::_process_eof(Block* block) { - // Convert all columns in _current_return_columns to schema column - RETURN_IF_ERROR(_convert_to_expected_type(_schema->column_ids())); - for (int i = 0; i < block->columns(); i++) { - auto cid = _schema->column_id(i); - if (!_is_pred_column[cid]) { - block->replace_by_position(i, std::move(_current_return_columns[cid])); + DCHECK_EQ(block->columns(), _schema->num_block_columns()); + // Convert all current columns to the exact read-schema types. + for (ColumnId column_id = 0; column_id < _schema->num_read_columns(); ++column_id) { + if (_is_active_read_column(column_id)) { + RETURN_IF_ERROR(_convert_column_to_expected_type(column_id)); + } + } + for (size_t i = 0; i < block->columns(); i++) { + if (!_column_states[i].is_predicate()) { + block->replace_by_position(i, std::move(_current_columns[i])); } } block->clear_column_data(); @@ -3140,7 +2918,18 @@ Status SegmentIterator::_execute_common_expr(uint16_t* sel_rowid_idx, uint16_t& break; } } - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(block, _columns_to_filter, filter)); + + std::vector materialized_block_ordinals; + materialized_block_ordinals.reserve(_predicate_ordinals.size() + _common_expr_ordinals.size()); + for (auto ordinal : _predicate_ordinals) { + if (ordinal < _schema->num_block_columns()) { + materialized_block_ordinals.push_back(ordinal); + } + } + materialized_block_ordinals.insert(materialized_block_ordinals.end(), + _common_expr_ordinals.begin(), _common_expr_ordinals.end()); + RETURN_IF_CATCH_EXCEPTION( + Block::filter_block_internal(block, materialized_block_ordinals, filter)); selected_size = _evaluate_common_expr_filter(sel_rowid_idx, selected_size, filter); _opts.stats->rows_expr_cond_filtered += original_size - selected_size; @@ -3249,31 +3038,25 @@ void SegmentIterator::_output_index_result_column(const VExprContextSPtrs& expr_ } void SegmentIterator::_convert_dict_code_for_predicate_if_necessary() { - for (auto predicate : _short_cir_eval_predicate) { - _convert_dict_code_for_predicate_if_necessary_impl(predicate); - } - - for (auto predicate : _pre_eval_block_predicate) { - _convert_dict_code_for_predicate_if_necessary_impl(predicate); - } - - for (auto column_id : _delete_range_column_ids) { - _current_return_columns[column_id].get()->convert_dict_codes_if_necessary(); + for (const auto& predicate : _col_predicates) { + _convert_dict_code_for_predicate_if_necessary_impl(*predicate); } - for (auto column_id : _delete_bloom_filter_column_ids) { - _current_return_columns[column_id].get()->initialize_hash_values_for_runtime_filter(); + std::set> delete_predicates; + _opts.delete_condition_predicates->get_all_column_predicate(delete_predicates); + for (const auto& predicate : delete_predicates) { + _convert_dict_code_for_predicate_if_necessary_impl(*predicate); } } void SegmentIterator::_convert_dict_code_for_predicate_if_necessary_impl( - std::shared_ptr predicate) { - auto& column = _current_return_columns[predicate->column_id()]; + const ColumnPredicate& predicate) { + auto& column = _current_columns[predicate.column_id()]; auto* col_ptr = column.get(); - if (PredicateTypeTraits::is_range(predicate->type())) { + if (PredicateTypeTraits::is_range(predicate.type())) { col_ptr->convert_dict_codes_if_necessary(); - } else if (PredicateTypeTraits::is_bloom_filter(predicate->type())) { + } else if (PredicateTypeTraits::is_bloom_filter(predicate.type())) { col_ptr->initialize_hash_values_for_runtime_filter(); } } @@ -3303,8 +3086,8 @@ Status SegmentIterator::_construct_compound_expr_context() { .io_ctx = _opts.io_ctx, }; auto inverted_index_context = std::make_shared( - _schema->column_ids(), _index_iterators, _storage_name_and_type, - _common_expr_index_exec_status, _score_runtime, _segment.get(), iter_opts); + _index_iterators, _storage_name_and_type, _common_expr_index_exec_status, + _score_runtime, _segment.get(), iter_opts); inverted_index_context->set_index_query_context(_index_query_context); for (const auto& expr_ctx : _opts.common_expr_ctxs_push_down) { VExprContextSPtr context; @@ -3355,18 +3138,13 @@ Status SegmentIterator::_apply_expr_zonemap_to_row_ranges(const VExprContextSPtr .io_ctx = _opts.io_ctx, }; for (const auto& [slot_index, slot_conjuncts] : ctxs_by_slot) { - if (cast_set(slot_index) >= _schema->num_column_ids()) { - continue; - } - const auto cid = _schema->column_id(cast_set(slot_index)); + DORIS_CHECK_LT(cast_set(slot_index), _schema->num_block_columns()); + const auto cid = cast_set(slot_index); if (!_segment->can_apply_predicate_safely(cid, *_schema, _opts.target_cast_type_for_variants, _opts)) { continue; } const auto* tablet_column = _schema->column(cid); - if (tablet_column == nullptr) { - continue; - } std::shared_ptr reader; Status st = _segment->get_column_reader(*tablet_column, &reader, _opts.stats, &_opts.io_ctx); @@ -3382,10 +3160,7 @@ Status SegmentIterator::_apply_expr_zonemap_to_row_ranges(const VExprContextSPtr if (page_zone_maps == nullptr || page_zone_maps->empty()) { continue; } - auto data_type = _segment->get_data_type_of(*tablet_column, _opts); - if (data_type == nullptr) { - continue; - } + auto data_type = _segment->get_data_type_of(*tablet_column, _schema->data_type(cid), _opts); RowRanges column_ranges; ZoneMapEvalStats page_stats; @@ -3455,7 +3230,7 @@ void SegmentIterator::_calculate_common_expr_index_exec_status() { for (const auto& vir_child : vir_node->children()) { if (vir_child->is_slot_ref()) { auto* inner_slot_ref = assert_cast(vir_child.get()); - auto cid = _schema->column_id(inner_slot_ref->column_id()); + auto cid = cast_set(inner_slot_ref->column_id()); _common_expr_index_exec_status[cid][expr.get()] = false; _common_expr_to_slotref_map[root_expr_ctx.get()] [inner_slot_ref->column_id()] = @@ -3473,7 +3248,7 @@ void SegmentIterator::_calculate_common_expr_index_exec_status() { auto expr_without_cast = VExpr::expr_without_cast(child); if (expr_without_cast->is_slot_ref() && expr->op() != TExprOpcode::CAST) { auto* column_slot_ref = assert_cast(expr_without_cast.get()); - auto cid = _schema->column_id(column_slot_ref->column_id()); + auto cid = cast_set(column_slot_ref->column_id()); _common_expr_index_exec_status[cid][expr.get()] = false; _common_expr_to_slotref_map[root_expr_ctx.get()][column_slot_ref->column_id()] = expr.get(); @@ -3506,7 +3281,7 @@ bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& col return false; } - if (!_opts.tablet_schema->column(cid).is_key()) { + if (!_schema->column(cid)->is_key()) { return false; } @@ -3522,10 +3297,16 @@ bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& col return true; } -bool SegmentIterator::_has_delete_predicate(ColumnId cid) { - std::set delete_columns_set; - _opts.delete_condition_predicates->get_all_column_ids(delete_columns_set); - return delete_columns_set.contains(cid); +bool SegmentIterator::_has_delete_predicate(ColumnId cid) const { + return _column_states[cid].is_delete_predicate; +} + +bool SegmentIterator::_is_active_read_column(ColumnId cid) const { + return cid < _schema->num_block_columns() || _column_states[cid].is_predicate(); +} + +bool SegmentIterator::_has_lazy_pruned_children(ColumnId cid) const { + return std::ranges::binary_search(_lazy_pruned_ordinals, cid); } bool SegmentIterator::_can_opt_limit_reads() { @@ -3544,15 +3325,17 @@ bool SegmentIterator::_can_opt_limit_reads() { return false; } - bool all_true = std::ranges::all_of(_schema->column_ids(), [this](auto cid) { - if (cid == _opts.tablet_schema->delete_sign_idx()) { - return true; - } - if (_check_all_conditions_passed_inverted_index_for_column(cid, true)) { - return true; - } - return false; - }); + bool all_true = std::ranges::all_of( + std::views::iota(uint32_t {0}, static_cast(_schema->num_read_columns())), + [this](uint32_t cid) { + if (cast_set(cid) == _schema->delete_sign_ordinal()) { + return true; + } + if (_check_all_conditions_passed_inverted_index_for_column(cid, true)) { + return true; + } + return false; + }); DBUG_EXECUTE_IF("segment_iterator.topn_opt_1", { LOG(INFO) << "col_predicates: " << _col_predicates.size() << ", all_true: " << all_true; @@ -3570,8 +3353,8 @@ bool SegmentIterator::_can_opt_limit_reads() { // Before get next batch. make sure all virtual columns in block has type ColumnNothing. void SegmentIterator::_init_virtual_columns(Block* block) { for (const auto& [cid, expr_ctx] : _virtual_column_exprs) { - auto idx = _schema->column_index(cid); - auto& col_with_type_and_name = block->get_by_position(idx); + DCHECK_LT(cid, _schema->num_block_columns()); + auto& col_with_type_and_name = block->get_by_position(cid); col_with_type_and_name.column = ColumnNothing::create(0); col_with_type_and_name.type = expr_ctx->root()->data_type(); } @@ -3582,8 +3365,8 @@ Status SegmentIterator::_materialization_of_virtual_column(Block* block) { // So materialize virtual column in advance to avoid errors. if (_selected_size == 0) { for (const auto& [cid, expr_ctx] : _virtual_column_exprs) { - auto idx = _schema->column_index(cid); - auto& col_with_type_and_name = block->get_by_position(idx); + DCHECK_LT(cid, _schema->num_block_columns()); + auto& col_with_type_and_name = block->get_by_position(cid); col_with_type_and_name.column = expr_ctx->root()->data_type()->create_column(); col_with_type_and_name.type = expr_ctx->root()->data_type(); } @@ -3595,8 +3378,9 @@ Status SegmentIterator::_materialization_of_virtual_column(Block* block) { for (const auto& cid_and_expr : _virtual_column_exprs) { auto cid = cid_and_expr.first; + DCHECK_LT(cid, _schema->num_block_columns()); auto column_expr = cid_and_expr.second; - auto materialized_pos = _schema->column_index(cid); + auto materialized_pos = cid; auto& column = block->get_by_position(materialized_pos).column; if (check_and_get_column(column.get())) { VLOG_DEBUG << fmt::format("Virtual column is doing materialization, cid {}, col idx {}", @@ -3638,7 +3422,7 @@ void SegmentIterator::_prepare_score_column_materialization() { result_row_ids, filter); } const size_t dst_col_idx = _score_runtime->get_dest_column_idx(); - auto* column_iter = _column_iterators[_schema->column_id(dst_col_idx)].get(); + auto* column_iter = _column_iterators[dst_col_idx].get(); auto* virtual_column_iter = dynamic_cast(column_iter); virtual_column_iter->prepare_materialization( std::move(result_column), diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index b67361d53ffa7a..4d6b1d2c312aef 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -59,9 +58,6 @@ namespace doris { -class ObjectPool; -class MatchPredicate; - class VExpr; class VExprContext; struct RowLocation; @@ -69,50 +65,14 @@ struct RowLocation; namespace segment_v2 { class ColumnIterator; -class InvertedIndexIterator; class RowRanges; class IndexIterator; -struct ColumnPredicateInfo { - ColumnPredicateInfo() = default; - - std::string debug_string() const { - std::stringstream ss; - ss << "column_name=" << column_name << ", query_op=" << query_op - << ", query_value=" << boost::join(query_values, ","); - return ss.str(); - } - - bool is_empty() const { - return column_name.empty() && query_values.empty() && query_op.empty(); - } - - bool is_equal(const ColumnPredicateInfo& column_pred_info) const { - if (column_pred_info.column_name != column_name) { - return false; - } - - if (column_pred_info.query_values != query_values) { - return false; - } - - if (column_pred_info.query_op != query_op) { - return false; - } - - return true; - } - - std::string column_name; - // use set to ensure the consistent order of predicate_result_sign generated by inlist. - std::set query_values; - std::string query_op; - int32_t column_id; -}; - class SegmentIterator : public RowwiseIterator { public: - SegmentIterator(std::shared_ptr segment, SchemaSPtr schema); + // Within SegmentIterator, ColumnId means an ordinal in the read schema. + // Storage UIDs and caller-visible Block positions are named explicitly. + SegmentIterator(std::shared_ptr segment, ReadSchemaSPtr schema); ~SegmentIterator() override; [[nodiscard]] Status init_iterators(); @@ -125,12 +85,8 @@ class SegmentIterator : public RowwiseIterator { [[nodiscard]] Status current_block_row_locations( std::vector* block_row_locations) override; - const Schema& schema() const override { return *_schema; } - Segment& segment() { return *_segment; } - StorageReadOptions& storage_read_options() { return _opts; } + const ReadSchema& schema() const override { return *_schema; } uint64_t data_id() const override { return _segment->id(); } - RowsetId rowset_id() const { return _segment->rowset_id(); } - int64_t tablet_id() const { return _tablet_id; } void update_profile(RuntimeProfile* profile) override { _update_profile(profile, _short_cir_eval_predicate, "ShortCircuitPredicates"); @@ -168,7 +124,7 @@ class SegmentIterator : public RowwiseIterator { [[nodiscard]] Status _lazy_init(Block* block); [[nodiscard]] Status _init_impl(const StorageReadOptions& opts); - [[nodiscard]] Status _init_return_column_iterators(); + [[nodiscard]] Status _init_column_iterators(); [[nodiscard]] Status _init_index_iterators(); // calculate row ranges that fall into requested key ranges using short key index @@ -203,29 +159,26 @@ class SegmentIterator : public RowwiseIterator { bool _column_has_ann_index(int32_t cid); bool _downgrade_without_index(Status res, bool need_remaining = false); inline bool _inverted_index_not_support_pred_type(const PredicateType& type); - bool _is_literal_node(const TExprNodeType::type& node_type); + void _init_column_states(); + void _rebuild_row_predicate_columns(); + void _extract_common_expr_columns(const VExprSPtr& expr); Status _vec_init_lazy_materialization(); uint32_t segment_id() const { return _segment->id(); } uint32_t num_rows() const { return _segment->num_rows(); } - [[nodiscard]] Status _seek_columns(const std::vector& column_ids, rowid_t pos); - // read `nrows` of columns specified by `column_ids` into `block` at `row_offset`. - // for vectorization implementation - [[nodiscard]] Status _read_columns(const std::vector& column_ids, - MutableColumns& column_block, size_t nrows); - [[nodiscard]] Status _read_columns_by_index(uint32_t nrows_read_limit, uint16_t& nrows_read); - void _replace_version_col_if_needed(const std::vector& column_ids, size_t num_rows); - void _update_tso_col_if_needed(const std::vector& column_ids, size_t num_rows); + [[nodiscard]] Status _read_columns_by_index(const std::vector& read_ordinals, + uint32_t nrows_read_limit, uint16_t& nrows_read); + void _replace_version_col_if_needed(const std::vector& ordinals, size_t num_rows); + void _update_tso_col_if_needed(const std::vector& ordinals, size_t num_rows); Status _init_current_block(Block* block, std::vector& non_pred_vector, uint32_t nrows_read_limit); uint16_t _evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, uint16_t selected_size); uint16_t _evaluate_short_circuit_predicate(uint16_t* sel_rowid_idx, uint16_t selected_size); Status _apply_read_limit_to_selected_rows(Block* block, uint16_t& selected_size); - void _collect_runtime_filter_predicate(); - Status _output_non_pred_columns(Block* block); - [[nodiscard]] Status _read_columns_by_rowids(std::vector& read_column_ids, + Status _output_columns_to_block(Block* block); + [[nodiscard]] Status _read_columns_by_rowids(const std::vector& read_ordinals, std::vector& rowid_vector, uint16_t* sel_rowid_idx, size_t select_size, MutableColumns* mutable_columns, @@ -238,35 +191,27 @@ class SegmentIterator : public RowwiseIterator { size_t batch_size); template - [[nodiscard]] Status _output_column_by_sel_idx(Block* block, const Container& column_ids, + [[nodiscard]] Status _output_column_by_sel_idx(Block* block, const Container& ordinals, uint16_t* sel_rowid_idx, uint16_t select_size) { SCOPED_RAW_TIMER(&_opts.stats->output_col_ns); - for (auto cid : column_ids) { - int block_cid = _schema->column_index(cid); - // Only the additional deleted filter condition need to materialize column be at the end of the block - // We should not to materialize the column of query engine do not need. So here just return OK. - // Eg: - // `delete from table where a = 10;` - // `select b from table;` - // a column only effective in segment iterator, the block from query engine only contain the b column. - // so the `block_cid >= data.size()` is true - if (block_cid >= block->columns()) { + for (auto ordinal : ordinals) { + if (ordinal >= _schema->num_block_columns()) { continue; } - DataTypePtr storage_type = _segment->get_data_type_of(*_schema->column(cid), _opts); - if (storage_type && !storage_type->equals(*block->get_by_position(block_cid).type)) { + const auto& file_column_type = _storage_name_and_type[ordinal].second; + if (!file_column_type->equals(*block->get_by_position(ordinal).type)) { // Do additional cast - MutableColumnPtr tmp = storage_type->create_column(); - RETURN_IF_ERROR(copy_column_data_by_selector(_current_return_columns[cid].get(), - tmp, sel_rowid_idx, select_size, + MutableColumnPtr tmp = file_column_type->create_column(); + RETURN_IF_ERROR(copy_column_data_by_selector(_current_columns[ordinal].get(), tmp, + sel_rowid_idx, select_size, _opts.block_row_max)); - RETURN_IF_ERROR(variant_util::cast_column( - {tmp->get_ptr(), storage_type, ""}, block->get_by_position(block_cid).type, - &block->get_by_position(block_cid).column)); + RETURN_IF_ERROR(variant_util::cast_column({tmp->get_ptr(), file_column_type, ""}, + block->get_by_position(ordinal).type, + &block->get_by_position(ordinal).column)); } else { MutableColumnPtr output_column = - block->get_by_position(block_cid).column->assert_mutable(); - RETURN_IF_ERROR(copy_column_data_by_selector(_current_return_columns[cid].get(), + block->get_by_position(ordinal).column->assert_mutable(); + RETURN_IF_ERROR(copy_column_data_by_selector(_current_columns[ordinal].get(), output_column, sel_rowid_idx, select_size, _opts.block_row_max)); } @@ -276,7 +221,6 @@ class SegmentIterator : public RowwiseIterator { bool _can_evaluated_by_vectorized(std::shared_ptr predicate); - [[nodiscard]] Status _extract_common_expr_columns(const VExprSPtr& expr); [[nodiscard]] Status _execute_common_expr(uint16_t* sel_rowid_idx, uint16_t& selected_size, Block* block); Status _process_common_expr(uint16_t* sel_rowid_idx, uint16_t& selected_size, Block* block); @@ -287,8 +231,7 @@ class SegmentIterator : public RowwiseIterator { // Dictionary column should do something to initial. void _convert_dict_code_for_predicate_if_necessary(); - void _convert_dict_code_for_predicate_if_necessary_impl( - std::shared_ptr predicate); + void _convert_dict_code_for_predicate_if_necessary_impl(const ColumnPredicate& predicate); bool _check_apply_by_inverted_index(std::shared_ptr pred); @@ -300,10 +243,11 @@ class SegmentIterator : public RowwiseIterator { Status _construct_compound_expr_context(); - int _compare_short_key_with_seek_block(const RowCursor& key, - const std::vector& col_ids) { - for (auto cid : col_ids) { - auto ord = key.field(cid) <=> (*_seek_block[cid])[0]; + // Both the key cursor and _seek_block lay out the leading tablet key + // columns densely, so position i addresses the same column in both. + int _compare_short_key_with_seek_block(const RowCursor& key, size_t num_key_cols) { + for (uint32_t i = 0; i < num_key_cols; ++i) { + auto ord = key.field(i) <=> (*_seek_block[i])[0]; if (ord != std::strong_ordering::equal) { return ord == std::strong_ordering::less ? -1 : 1; } @@ -311,11 +255,14 @@ class SegmentIterator : public RowwiseIterator { return 0; } - Status _convert_to_expected_type(const std::vector& col_ids); + Status _convert_column_to_expected_type(ColumnId column_id); + Status _convert_to_expected_type(const std::vector& ordinals); bool _no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, size_t nrows_read); - bool _has_delete_predicate(ColumnId cid); + bool _is_active_read_column(ColumnId cid) const; + bool _has_delete_predicate(ColumnId cid) const; + bool _has_lazy_pruned_children(ColumnId cid) const; bool _can_skip_reading_extra_column(ColumnId cid); bool _can_opt_limit_reads(); @@ -330,7 +277,7 @@ class SegmentIterator : public RowwiseIterator { void _fill_column_nothing(); - Status _process_columns(const std::vector& column_ids, Block* block); + Status _process_columns(const std::vector& ordinals, Block* block); // Initialize virtual columns in the block, set all virtual columns in the block to ColumnNothing void _init_virtual_columns(Block* block); @@ -345,10 +292,23 @@ class SegmentIterator : public RowwiseIterator { class BitmapRangeIterator; class BackwardBitmapRangeIterator; + struct ColumnReadState { + bool is_delete_predicate = false; + // Mirrors the mutable _col_predicates list: initially all safe row + // predicates, then only residual predicates after index evaluation. + bool is_row_predicate = false; + bool is_common_expr = false; + // Index evaluation sets this to false when it fully supplies the column result. + // _need_read_data() applies the remaining read constraints. + bool need_read_data = true; + + bool is_predicate() const { return is_delete_predicate || is_row_predicate; } + }; + std::shared_ptr _segment; // read schema from scanner - SchemaSPtr _schema; - // storage type schema related to _schema, since column in segment may be different with type in _schema + ReadSchemaSPtr _schema; + // Inverted-index field name and storage/materialization type for each ReadSchema column. std::vector _storage_name_and_type; // vector idx -> column iterarator std::vector> _column_iterators; @@ -361,11 +321,6 @@ class SegmentIterator : public RowwiseIterator { rowid_t _cur_rowid; // members related to lazy materialization read // -------------------------------------------- - // whether lazy materialization read should be used. - bool _lazy_materialization_read; - // columns to read after predicate evaluation and remaining expr execute - std::vector _non_predicate_columns; - std::set _common_expr_columns; // remember the rowids we've read for the current row block. // could be a local variable of next_batch(), kept here to reuse vector memory std::vector _block_rowids; @@ -373,33 +328,21 @@ class SegmentIterator : public RowwiseIterator { bool _is_need_short_eval = false; bool _is_need_expr_eval = false; - std::set _support_lazy_read_pruned_columns; bool _enable_prune_nested_column = false; - // fields for vectorization execution - std::vector - _vec_pred_column_ids; // keep columnId of columns for vectorized predicate evaluation - std::vector - _short_cir_pred_column_ids; // keep columnId of columns for short circuit predicate evaluation - std::vector _is_pred_column; // columns hold _init segmentIter - std::map _need_read_data_indices; - std::vector _is_common_expr_column; - MutableColumns _current_return_columns; + // Per-column state indexed by read schema ordinal. Ordered column lists + // below are execution plans rather than additional column membership sets. + std::vector _column_states; + // Columns of the current batch, indexed by read schema ordinal. + MutableColumns _current_columns; std::vector> _pre_eval_block_predicate; std::vector> _short_cir_eval_predicate; - std::vector _delete_range_column_ids; - std::vector _delete_bloom_filter_column_ids; - // when lazy materialization is enabled, segmentIter need to read data at least twice - // first, read predicate columns by various index - // second, read non-predicate columns - // so we need a field to stand for columns first time to read - std::vector _predicate_column_ids; - std::vector _common_expr_column_ids; - // Block slot indexes to filter after common expr evaluation. This is not - // tablet column ids because Block::filter_block_internal filters by block - // position. - std::vector _columns_to_filter; - std::vector _converted_column_ids; + // Ordered, disjoint column-role lists built once and reused by every batch. + std::vector _predicate_ordinals; + std::vector _common_expr_ordinals; + std::vector _output_ordinals; + // Sparse, ordered execution list for recovering pruned nested children. + std::vector _lazy_pruned_ordinals; // the actual init process is delayed to the first call to next_batch() bool _lazy_inited; @@ -418,14 +361,18 @@ class SegmentIterator : public RowwiseIterator { // make a copy of `_opts.column_predicates` in order to make local changes std::vector> _col_predicates; VExprContextSPtrs _common_expr_ctxs_push_down; - std::set _not_apply_index_pred; - // row schema of the key to seek // only used in `_get_row_ranges_by_keys` - std::unique_ptr _seek_schema; + std::unique_ptr _seek_schema; // used to binary search the rowid for a given key // only used in `_get_row_ranges_by_keys` MutableColumns _seek_block; + // Per-seek-schema-ordinal column iterators for the short-key seek path. + // Points into _column_iterators when the key column is also read, otherwise + // into _owned_seek_column_iterators (a seek key column may not be part of + // the read schema at all). + std::vector _seek_column_iterators; + std::vector> _owned_seek_column_iterators; io::FileReaderSPtr _file_reader; @@ -437,13 +384,9 @@ class SegmentIterator : public RowwiseIterator { // _opts.read_limit to compute the remaining per-batch budget. size_t _rows_returned = 0; - std::unique_ptr _pool; - - // used to collect filter information. - std::vector> _filter_info_id; - bool _record_rowids = false; int64_t _tablet_id = 0; - std::set _output_columns; + // Stable column UIDs requested by the caller; -1 disables data-page skipping. + std::set _output_column_uids; std::vector _ret_flags; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 0f780986f1a119..c67b3fd06cdb8b 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -449,7 +449,7 @@ Status SegmentWriter::append_block_with_partial_content(const Block* block, size const auto& including_cids = _opts.rowset_ctx->partial_update_info->update_cids; // create full block and fill with input columns - auto full_block = _tablet_schema->create_block(); + auto full_block = _tablet_schema->create_storage_block(); size_t input_id = 0; for (auto i : including_cids) { const auto& input_column = block->get_by_position(input_id++); diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index 5676e6b248bebb..748fd42870217a 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -504,7 +504,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da RETURN_IF_ERROR(_partial_update_preconditions_check(data.row_pos, false)); // create full block and fill with input columns - full_block = _tablet_schema->create_block(); + full_block = _tablet_schema->create_storage_block(); const auto& including_cids = _opts.rowset_ctx->partial_update_info->update_cids; size_t input_id = 0; for (auto i : including_cids) { @@ -696,7 +696,7 @@ Status VerticalSegmentWriter::_append_block_with_flexible_partial_content(RowsIn DCHECK(data.block->columns() == _tablet_schema->num_columns()); // create full block and fill with sort key columns - full_block = _tablet_schema->create_block(); + full_block = _tablet_schema->create_storage_block(); // Use _num_rows_written instead of creating column writer 0, since all column writers // should have the same row count, which equals _num_rows_written. @@ -875,7 +875,7 @@ Status VerticalSegmentWriter::_generate_encoded_default_seq_value(const TabletSc const PartialUpdateInfo& info, std::string* encoded_value) { const auto& seq_column = tablet_schema.column(tablet_schema.sequence_col_idx()); - auto block = tablet_schema.create_block_by_cids( + auto block = tablet_schema.create_storage_block( {cast_set(tablet_schema.sequence_col_idx())}); if (seq_column.has_default_value()) { auto idx = tablet_schema.sequence_col_idx() - tablet_schema.num_key_columns(); diff --git a/be/src/storage/tablet/base_tablet.cpp b/be/src/storage/tablet/base_tablet.cpp index aab47367c7d399..288110c30d00d1 100644 --- a/be/src/storage/tablet/base_tablet.cpp +++ b/be/src/storage/tablet/base_tablet.cpp @@ -648,7 +648,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, std::map rsid_to_rowset; rsid_to_rowset[rowset_id] = rowset; - Block block = rowset_schema->create_block(); + Block block = rowset_schema->create_storage_block(); Block ordered_block = block.clone_empty(); uint32_t pos = 0; @@ -861,7 +861,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, auto row_binlog_schema = row_binlog_rowset->tablet_schema(); std::vector lsn_cids = { static_cast(row_binlog_schema->binlog_lsn_col_idx())}; - lsn_block = row_binlog_schema->create_block_by_cids(lsn_cids); + lsn_block = row_binlog_schema->create_storage_block(lsn_cids); std::map rsid_to_row_binlog { {row_binlog_rowset->rowset_id(), row_binlog_rowset}}; std::map read_index; @@ -1071,8 +1071,8 @@ Status BaseTablet::generate_new_block_for_partial_update( auto& full_mutable_columns = full_mutable_columns_guard.mutable_columns(); const auto& missing_cids = partial_update_info->missing_cids; const auto& update_cids = partial_update_info->update_cids; - auto old_block = rowset_schema->create_block_by_cids(missing_cids); - auto update_block = rowset_schema->create_block_by_cids(update_cids); + auto old_block = rowset_schema->create_storage_block(missing_cids); + auto update_block = rowset_schema->create_storage_block(update_cids); bool have_input_seq_column = false; if (rowset_schema->has_sequence_col()) { @@ -1238,8 +1238,8 @@ Status BaseTablet::generate_new_block_for_flexible_partial_update( const auto& non_sort_key_cids = partial_update_info->missing_cids; std::vector all_cids(rowset_schema->num_columns()); std::iota(all_cids.begin(), all_cids.end(), 0); - auto old_block = rowset_schema->create_block_by_cids(non_sort_key_cids); - auto update_block = rowset_schema->create_block_by_cids(all_cids); + auto old_block = rowset_schema->create_storage_block(non_sort_key_cids); + auto update_block = rowset_schema->create_storage_block(all_cids); // rowid in the final block(start from 0, increase continuously) -> rowid to read in update_block std::map read_index_update; diff --git a/be/src/storage/tablet/tablet_reader.cpp b/be/src/storage/tablet/tablet_reader.cpp index a1ee0aee7c970b..2e4d92ebd007f3 100644 --- a/be/src/storage/tablet/tablet_reader.cpp +++ b/be/src/storage/tablet/tablet_reader.cpp @@ -21,26 +21,16 @@ #include #include -#include -#include -#include #include -#include #include -#include -#include -#include #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" #include "common/exception.h" #include "common/logging.h" #include "common/status.h" -#include "core/arena.h" #include "core/block/block.h" -#include "exec/common/variant_util.h" #include "exprs/bloom_filter_func.h" -#include "exprs/create_predicate_function.h" #include "exprs/hybrid_set.h" #include "runtime/query_context.h" #include "runtime/runtime_predicate.h" @@ -52,7 +42,6 @@ #include "storage/olap_define.h" #include "storage/predicate/block_column_predicate.h" #include "storage/predicate/column_predicate.h" -#include "storage/predicate/like_column_predicate.h" #include "storage/predicate/predicate_creator.h" #include "storage/row_cursor.h" #include "storage/schema.h" @@ -82,17 +71,17 @@ Status TabletReader::init(const ReaderParams& read_params) { } void TabletReader::remove_delete_columns_from_access_paths( - const DeleteHandler& delete_handler, const TabletSchema& tablet_schema, + const DeleteHandler& delete_handler, const ReadSchema& read_schema, std::map& all_access_paths) { auto delete_predicates = AndBlockColumnPredicate::create_shared(); std::unordered_map>> del_predicates_for_zone_map; delete_handler.get_delete_conditions_after_version(0, delete_predicates.get(), &del_predicates_for_zone_map); - std::set delete_column_ids; - delete_predicates->get_all_column_ids(delete_column_ids); - for (auto cid : delete_column_ids) { - all_access_paths.erase(tablet_schema.column(cid).unique_id()); + std::set delete_column_ordinals; + delete_predicates->get_all_column_ids(delete_column_ordinals); + for (auto ordinal : delete_column_ordinals) { + all_access_paths.erase(read_schema.column(ordinal)->unique_id()); } } @@ -172,16 +161,12 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { _reader_context.tablet_schema = _tablet_schema; _reader_context.need_ordered_result = need_ordered_result; _reader_context.topn_filter_source_node_ids = read_params.topn_filter_source_node_ids; - _reader_context.topn_filter_target_node_id = read_params.topn_filter_target_node_id; _reader_context.read_orderby_key_reverse = read_params.read_orderby_key_reverse; _reader_context.use_insert_order_when_same = read_params.use_insert_order_when_same || read_params.read_row_binlog; _reader_context.force_key_ordered_read = read_params.force_key_ordered_read; _reader_context.read_orderby_key_limit = read_params.read_orderby_key_limit; - _reader_context.return_columns = &_return_columns; - _reader_context.tso_predicate_column_id = read_params.tso_predicate_column_id; - _reader_context.start_tso = read_params.start_tso; - _reader_context.end_tso = read_params.end_tso; + _reader_context.read_schema = _read_schema; _reader_context.read_orderby_key_columns = !_orderby_key_columns.empty() ? &_orderby_key_columns : nullptr; _reader_context.predicates = &_col_predicates; @@ -193,7 +178,6 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { _reader_context.delete_handler = &_delete_handler; _reader_context.stats = &_stats; _reader_context.use_page_cache = read_params.use_page_cache; - _reader_context.sequence_id_idx = _sequence_col_idx; _reader_context.is_unique = tablet()->keys_type() == UNIQUE_KEYS; _reader_context.merged_rows = &_merged_rows; _reader_context.delete_bitmap = read_params.delete_bitmap; @@ -222,7 +206,7 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { // mark them meta-only (OFFSET/NULL), whose content-less read makes the delete predicate // match nothing and leak deleted rows. if (!_delete_handler.empty() && !_reader_context.all_access_paths.empty()) { - remove_delete_columns_from_access_paths(_delete_handler, *_tablet_schema, + remove_delete_columns_from_access_paths(_delete_handler, *_read_schema, _reader_context.all_access_paths); } @@ -232,20 +216,6 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { return Status::OK(); } -TabletColumn TabletReader::materialize_column(const TabletColumn& orig) { - if (!orig.is_variant_type()) { - return orig; - } - TabletColumn column_with_cast_type = orig; - auto cast_type = _reader_context.target_cast_type_for_variants.at(orig.name()); - return variant_util::get_column_by_type(cast_type, orig.name(), - { - .unique_id = orig.unique_id(), - .parent_unique_id = orig.parent_unique_id(), - .path_info = *orig.path_info_ptr(), - }); -} - Status TabletReader::_init_params(const ReaderParams& read_params) { read_params.check_validation(); @@ -254,22 +224,17 @@ Status TabletReader::_init_params(const ReaderParams& read_params) { _reader_type = read_params.reader_type; _tablet = read_params.tablet; _tablet_schema = read_params.tablet_schema; + _read_schema = read_params.read_schema; _reader_context.runtime_state = read_params.runtime_state; _reader_context.target_cast_type_for_variants = read_params.target_cast_type_for_variants; - RETURN_IF_ERROR(_init_conditions_param(read_params)); - Status res = _init_delete_condition(read_params); if (!res.ok()) { LOG(WARNING) << "fail to init delete param. res = " << res; return res; } - res = _init_return_columns(read_params); - if (!res.ok()) { - LOG(WARNING) << "fail to init return columns. res = " << res; - return res; - } + RETURN_IF_ERROR(_init_column_predicates(read_params)); res = _init_keys_param(read_params); if (!res.ok()) { @@ -281,78 +246,9 @@ Status TabletReader::_init_params(const ReaderParams& read_params) { LOG(WARNING) << "fail to init orderby keys param. res=" << res; return res; } - if (_tablet_schema->has_sequence_col()) { - auto sequence_col_idx = _tablet_schema->sequence_col_idx(); - DCHECK_NE(sequence_col_idx, -1); - for (auto col : _return_columns) { - // query has sequence col - if (col == sequence_col_idx) { - _sequence_col_idx = sequence_col_idx; - break; - } - } - } - return res; } -Status TabletReader::_init_return_columns(const ReaderParams& read_params) { - SCOPED_RAW_TIMER(&_stats.tablet_reader_init_return_columns_timer_ns); - if (read_params.reader_type == ReaderType::READER_QUERY) { - _return_columns = read_params.return_columns; - _tablet_columns_convert_to_null_set = read_params.tablet_columns_convert_to_null_set; - for (auto id : read_params.return_columns) { - if (_tablet_schema->column(id).is_key()) { - _key_cids.push_back(id); - } else { - _value_cids.push_back(id); - } - } - } else if (read_params.return_columns.empty()) { - for (uint32_t i = 0; i < _tablet_schema->num_columns(); ++i) { - _return_columns.push_back(i); - if (_tablet_schema->column(i).is_key()) { - _key_cids.push_back(i); - } else { - _value_cids.push_back(i); - } - } - VLOG_NOTICE << "return column is empty, using full column as default."; - } else if ((read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION || - read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION || - read_params.reader_type == ReaderType::READER_BASE_COMPACTION || - read_params.reader_type == ReaderType::READER_FULL_COMPACTION || - read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || - read_params.reader_type == ReaderType::READER_ALTER_TABLE) && - !read_params.return_columns.empty()) { - _return_columns = read_params.return_columns; - for (auto id : read_params.return_columns) { - if (_tablet_schema->column(id).is_key()) { - _key_cids.push_back(id); - } else { - _value_cids.push_back(id); - } - } - } else if (read_params.reader_type == ReaderType::READER_CHECKSUM) { - _return_columns = read_params.return_columns; - for (auto id : read_params.return_columns) { - if (_tablet_schema->column(id).is_key()) { - _key_cids.push_back(id); - } else { - _value_cids.push_back(id); - } - } - } else { - return Status::Error( - "fail to init return columns. reader_type={}, return_columns_size={}", - int(read_params.reader_type), read_params.return_columns.size()); - } - - std::sort(_key_cids.begin(), _key_cids.end(), std::greater<>()); - - return Status::OK(); -} - Status TabletReader::_init_keys_param(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_keys_param_timer_ns); if (read_params.start_key.empty()) { @@ -424,31 +320,29 @@ Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) { _tablet_schema->cluster_key_uids().size()); } for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) { - auto cid = _tablet_schema->cluster_key_uids()[i]; - auto index = _tablet_schema->field_index(cid); + auto uid = _tablet_schema->cluster_key_uids()[i]; + auto index = _tablet_schema->field_index(uid); if (index < 0) { return Status::Error( "could not find cluster key column with unique_id=" + - std::to_string(cid) + + std::to_string(uid) + " in tablet schema, tablet_id=" + std::to_string(_tablet->tablet_id())); } - for (uint32_t idx = 0; idx < _return_columns.size(); idx++) { - if (_return_columns[idx] == index) { - _orderby_key_columns.push_back(idx); - break; - } + int32_t ordinal = _read_schema->ordinal_by_column(_tablet_schema->column(index)); + if (ordinal < 0) { + break; // size check below reports the error } + _orderby_key_columns.push_back(ordinal); } } else { - // find index in vector _return_columns - // for the read_orderby_key_num_prefix_columns orderby keys + // the orderby keys are the leading storage key columns; resolve + // each to its ordinal in the read schema for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) { - for (uint32_t idx = 0; idx < _return_columns.size(); idx++) { - if (_return_columns[idx] == i) { - _orderby_key_columns.push_back(idx); - break; - } + int32_t ordinal = _read_schema->ordinal_by_column(_tablet_schema->column(i)); + if (ordinal < 0) { + break; // size check below reports the error } + _orderby_key_columns.push_back(ordinal); } } if (read_params.read_orderby_key_num_prefix_columns != _orderby_key_columns.size()) { @@ -463,25 +357,18 @@ Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) { return Status::OK(); } -Status TabletReader::_init_conditions_param(const ReaderParams& read_params) { +Status TabletReader::_init_column_predicates(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_conditions_param_timer_ns); - std::vector> predicates; - std::copy(read_params.predicates.cbegin(), read_params.predicates.cend(), - std::inserter(predicates, predicates.begin())); - // Function filter push down to storage engine - auto is_like_predicate = [](std::shared_ptr _pred) { - return dynamic_cast(_pred.get()) != nullptr; - }; - - for (const auto& filter : read_params.function_filters) { - predicates.emplace_back(_parse_to_predicate(filter)); - auto pred = predicates.back(); - - const auto& col = _tablet_schema->column(pred->column_id()); + auto predicates = read_params.predicates; + for (const auto& predicate : predicates) { + if (predicate->type() != PredicateType::LIKE) { + continue; + } + const auto& col = *_read_schema->column(predicate->column_id()); const auto* tablet_index = _tablet_schema->get_ngram_bf_index(col.unique_id()); - if (is_like_predicate(pred) && tablet_index && config::enable_query_like_bloom_filter) { + if (tablet_index && config::enable_query_like_bloom_filter) { std::unique_ptr ng_bf; - std::string pattern = pred->get_search_str(); + std::string pattern = predicate->get_search_str(); auto gram_bf_size = tablet_index->get_gram_bf_size(); auto gram_size = tablet_index->get_gram_size(); @@ -491,20 +378,20 @@ Status TabletReader::_init_conditions_param(const ReaderParams& read_params) { if (_token_extractor.string_like_to_bloom_filter(pattern.data(), pattern.length(), *ng_bf)) { - pred->set_page_ng_bf(std::move(ng_bf)); + predicate->set_page_ng_bf(std::move(ng_bf)); } } } - int32_t delete_sign_idx = _tablet_schema->delete_sign_idx(); + int32_t delete_sign_ordinal = _read_schema->delete_sign_ordinal(); for (auto predicate : predicates) { - auto column = _tablet_schema->column(predicate->column_id()); + const auto& column = *_read_schema->column(predicate->column_id()); if (column.aggregation() != FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE) { // When MOR value predicate pushdown is enabled, drop __DORIS_DELETE_SIGN__ // from storage-layer predicates entirely. Delete sign must only be evaluated // post-merge via VExpr to prevent deleted rows from reappearing. - if (read_params.enable_mor_value_predicate_pushdown && delete_sign_idx >= 0 && - predicate->column_id() == static_cast(delete_sign_idx)) { + if (read_params.enable_mor_value_predicate_pushdown && delete_sign_ordinal >= 0 && + predicate->column_id() == static_cast(delete_sign_ordinal)) { continue; } _value_col_predicates.push_back(predicate); @@ -516,19 +403,6 @@ Status TabletReader::_init_conditions_param(const ReaderParams& read_params) { return Status::OK(); } -std::shared_ptr TabletReader::_parse_to_predicate( - const FunctionFilter& function_filter) { - int32_t index = _tablet_schema->field_index(function_filter._col_name); - if (index < 0) { - throw Exception(Status::InternalError("Column {} not found in tablet schema", - function_filter._col_name)); - return nullptr; - } - const TabletColumn& column = materialize_column(_tablet_schema->column(index)); - return create_column_predicate(index, std::make_shared(function_filter), - column.type(), &column); -} - Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_delete_condition_param_timer_ns); // If it's cumu and not allow do delete when cumu @@ -552,48 +426,12 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { // However, queries will not use this condition but generate special where predicates to filter data. // (Though a lille bit confused, it is how the current logic working...) _filter_delete = _delete_sign_available || cumu_delete; - return _delete_handler.init(_tablet_schema, read_params.delete_predicates, - read_params.version.second); -} - -Status TabletReader::init_reader_params_and_create_block( - TabletSharedPtr tablet, ReaderType reader_type, - const std::vector& input_rowsets, - TabletReader::ReaderParams* reader_params, Block* block) { - reader_params->tablet = tablet; - reader_params->reader_type = reader_type; - reader_params->version = - Version(input_rowsets.front()->start_version(), input_rowsets.back()->end_version()); - - TabletReadSource read_source; - for (const auto& rowset : input_rowsets) { - RowsetReaderSharedPtr rs_reader; - RETURN_IF_ERROR(rowset->create_reader(&rs_reader)); - read_source.rs_splits.emplace_back(std::move(rs_reader)); + std::vector dropped_columns; + RETURN_IF_ERROR(_delete_handler.init(read_params.delete_predicates, read_params.version.second, + _read_schema, dropped_columns)); + for (auto& column : dropped_columns) { + _read_schema->append_column(std::make_shared(std::move(column))); } - read_source.fill_delete_predicates(); - reader_params->set_read_source(std::move(read_source)); - - std::vector rowset_metas(input_rowsets.size()); - std::transform(input_rowsets.begin(), input_rowsets.end(), rowset_metas.begin(), - [](const RowsetSharedPtr& rowset) { return rowset->rowset_meta(); }); - TabletSchemaSPtr read_tablet_schema = - tablet->tablet_schema_with_merged_max_schema_version(rowset_metas); - TabletSchemaSPtr merge_tablet_schema = std::make_shared(); - merge_tablet_schema->copy_from(*read_tablet_schema); - - // Merge the columns in delete predicate that not in latest schema in to current tablet schema - for (auto& del_pred : reader_params->delete_predicates) { - merge_tablet_schema->merge_dropped_columns(*del_pred->tablet_schema()); - } - reader_params->tablet_schema = merge_tablet_schema; - - reader_params->return_columns.resize(read_tablet_schema->num_columns()); - std::iota(reader_params->return_columns.begin(), reader_params->return_columns.end(), 0); - reader_params->origin_return_columns = &reader_params->return_columns; - - *block = read_tablet_schema->create_block(); - return Status::OK(); } diff --git a/be/src/storage/tablet/tablet_reader.h b/be/src/storage/tablet/tablet_reader.h index 256a92d0c7c8e1..b8a9f6366c4a92 100644 --- a/be/src/storage/tablet/tablet_reader.h +++ b/be/src/storage/tablet/tablet_reader.h @@ -24,7 +24,6 @@ #include #include -#include #include #include #include @@ -33,7 +32,6 @@ #include "agent/be_exec_version_manager.h" #include "common/status.h" -#include "exprs/function_filter.h" #include "io/io_common.h" #include "storage/delete/delete_handler.h" #include "storage/iterators.h" @@ -61,7 +59,6 @@ class RuntimeProfile; class VCollectIterator; class Block; class VExpr; -class Arena; class VExprContext; // Used to compare row with input scan key. Scan key only contains key columns, @@ -144,7 +141,6 @@ class TabletReader { bool end_key_include = false; std::vector> predicates; - std::vector function_filters; std::vector delete_predicates; // slots that cast may be eliminated in storage layer std::map target_cast_type_for_variants; @@ -156,13 +152,15 @@ class TabletReader { // For unique key table with merge-on-write DeleteBitmapPtr delete_bitmap = nullptr; - // return_columns is init from query schema - std::vector return_columns; - // TSO predicate column that is absent from return_columns but must be read by storage. - std::optional tso_predicate_column_id; + // The read schema of every Block at the TabletReader boundary. + // Query scanners construct the FE-slot prefix in scan-tuple order; + // TabletReader may append storage-only delete-predicate columns. + ReadSchemaSPtr read_schema; // output_columns only contain columns in OrderByExprs and outputExprs + // (column unique ids, not ordinals) std::set output_columns; - // Extra storage key columns that are present only for scan-schema alignment. + // Ordinals (positions in read_schema) of extra storage key columns + // that are present only for scan-schema alignment. // Example: for AGG keys (k1, k2), a query that returns k2 can scan // (k1, k2) and project away k1. Direct readers may avoid reading such // columns only if the lower iterator proves their real values are not @@ -171,9 +169,6 @@ class TabletReader { RuntimeProfile* profile = nullptr; RuntimeState* runtime_state = nullptr; - // use only in vec exec engine - std::vector* origin_return_columns = nullptr; - std::unordered_set* tablet_columns_convert_to_null_set = nullptr; TPushAggOp::type push_down_agg_type_opt = TPushAggOp::NONE; VExprContextSPtrs common_expr_ctxs_push_down; @@ -181,7 +176,6 @@ class TabletReader { bool record_rowids = false; RowIdConversion* rowid_conversion = nullptr; std::vector topn_filter_source_node_ids; - int topn_filter_target_node_id = -1; // used for special optimization for query : ORDER BY key LIMIT n bool read_orderby_key = false; // used for special optimization for query : ORDER BY key DESC LIMIT n @@ -218,6 +212,7 @@ class TabletReader { int64_t batch_size = -1; + // virtual column ordinal (position in read_schema) -> expression std::map virtual_column_exprs; std::shared_ptr score_runtime; @@ -229,11 +224,6 @@ class TabletReader { // General LIMIT budget forwarded to SegmentIterator. -1 means no limit. int64_t general_read_limit = -1; TBinlogScanType::type binlog_scan_type = TBinlogScanType::NONE; - // Binlog/snapshot incremental read TSO range (start_tso, end_tso]. Forced down to - // BetaRowsetReader, which builds the tso predicates directly on read options, - // bypassing the value/key predicate split in _init_conditions_param. - std::optional start_tso; - std::optional end_tso; }; TabletReader() = default; @@ -265,8 +255,6 @@ class TabletReader { void set_batch_size(int batch_size) { _reader_context.batch_size = batch_size; } - int batch_size() const { return _reader_context.batch_size; } - size_t batch_max_rows() const { return _reader_context.batch_size; } void set_preferred_block_size_bytes(size_t bytes) { @@ -282,21 +270,16 @@ class TabletReader { OlapReaderStatistics* mutable_stats() { return &_stats; } virtual void update_profile(RuntimeProfile* profile) {} - static Status init_reader_params_and_create_block( - TabletSharedPtr tablet, ReaderType reader_type, - const std::vector& input_rowsets, - TabletReader::ReaderParams* reader_params, Block* block); // Remove the delete-condition columns from `all_access_paths` so they fall back to a full // read (a meta-only read would make the storage delete predicate match nothing and leak // deleted rows). static void remove_delete_columns_from_access_paths( - const DeleteHandler& delete_handler, const TabletSchema& tablet_schema, + const DeleteHandler& delete_handler, const ReadSchema& read_schema, std::map& all_access_paths); protected: friend class VCollectIterator; - friend class DeleteHandler; Status _init_params(const ReaderParams& read_params); @@ -306,36 +289,18 @@ class TabletReader { Status _init_orderby_keys_param(const ReaderParams& read_params); - Status _init_conditions_param(const ReaderParams& read_params); - - virtual std::shared_ptr _parse_to_predicate( - const FunctionFilter& function_filter); + Status _init_column_predicates(const ReaderParams& read_params); Status _init_delete_condition(const ReaderParams& read_params); - Status _init_return_columns(const ReaderParams& read_params); - const BaseTabletSPtr& tablet() { return _tablet; } - // If original column is a variant type column, and it's predicate is normalized - // so in order to get the real type of column predicate, we need to reset type - // according to the related type in `target_cast_type_for_variants`.Since variant is not - // an predicate applicable type.Otherwise return the original tablet column. - // Eg. `where cast(v:a as bigint) > 1` will elimate cast, and materialize this variant column - // to type bigint - TabletColumn materialize_column(const TabletColumn& orig); - const TabletSchema& tablet_schema() { return *_tablet_schema; } - - Arena _predicate_arena; - std::vector _return_columns; + // The read schema shared by the caller, TabletReader, VCollect and each rowset reader. + ReadSchemaSPtr _read_schema; // used for special optimization for query : ORDER BY key [ASC|DESC] LIMIT n // columns for orderby keys std::vector _orderby_key_columns; - // only use in outer join which change the column nullable which must keep same in - // vec query engine - std::unordered_set* _tablet_columns_convert_to_null_set = nullptr; - BaseTabletSPtr _tablet; RowsetReaderContext _reader_context; TabletSchemaSPtr _tablet_schema; @@ -350,15 +315,10 @@ class TabletReader { bool _aggregation = false; // for agg query, we don't need to finalize when scan agg object data ReaderType _reader_type = ReaderType::READER_QUERY; - bool _next_delete_flag = false; bool _delete_sign_available = false; bool _filter_delete = false; - int32_t _sequence_col_idx = -1; bool _direct_mode = false; - std::vector _key_cids; - std::vector _value_cids; - uint64_t _merged_rows = 0; OlapReaderStatistics _stats; }; diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index a1390582d42729..60d68a0cfd1d01 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -36,7 +36,6 @@ #include "common/consts.h" #include "common/status.h" #include "core/block/block.h" -#include "core/column/column_nothing.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" @@ -883,13 +882,11 @@ void TabletSchema::append_column(TabletColumn column, ColumnType col_type) { } _field_uniqueid_to_index[column.unique_id()] = _num_columns; _cols.push_back(std::make_shared(std::move(column))); - // The dropped column may have same name with exsiting column, so that - // not add to name to index map, only for uid to index map if (col_type == ColumnType::VARIANT || _cols.back()->is_variant_type() || _cols.back()->is_extracted_column()) { _field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns); _field_path_to_index[_cols.back()->path_info_ptr().get()] = _num_columns; - } else if (col_type == ColumnType::NORMAL) { + } else { _field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns); } _num_columns++; @@ -917,9 +914,7 @@ void TabletSchema::append_column(TabletColumn column, ColumnType col_type) { const auto seq_idx = _field_uniqueid_to_index[seq_uid]; for (const auto col_idx : value_cols_index) { _seq_col_idx_to_value_cols_idx[seq_idx].push_back(col_idx); - _value_col_idx_to_seq_col_idx[col_idx] = seq_idx; } - _value_col_idx_to_seq_col_idx[seq_idx] = seq_idx; } } } @@ -985,7 +980,6 @@ void TabletSchema::clear_columns() { _num_null_columns = 0; _num_key_columns = 0; _seq_col_idx_to_value_cols_idx.clear(); - _value_col_idx_to_seq_col_idx.clear(); _cols.clear(); } @@ -1101,7 +1095,6 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _seq_col_uid_to_value_cols_uid.clear(); _value_col_uid_to_seq_col_uid.clear(); _seq_col_idx_to_value_cols_idx.clear(); - _value_col_idx_to_seq_col_idx.clear(); /* * ColumnGroupsPB is a list of cg_pb, and * ColumnGroupsPB do not have begin() or end() method. @@ -1145,13 +1138,6 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac } _value_col_uid_to_seq_col_uid[seq_uid] = seq_uid; } - - for (auto& [seq_idx, value_cols_idx] : _seq_col_idx_to_value_cols_idx) { - for (auto col_idx : value_cols_idx) { - _value_col_idx_to_seq_col_idx[col_idx] = seq_idx; - } - _value_col_idx_to_seq_col_idx[seq_idx] = seq_idx; - } } } // Default to V1 inverted index storage format for backward compatibility if not specified in schema. @@ -1354,25 +1340,6 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version } } -void TabletSchema::merge_dropped_columns(const TabletSchema& src_schema) { - // If they are the same tablet schema object, then just return - if (this == &src_schema) { - return; - } - for (const auto& src_col : src_schema.columns()) { - if (_field_uniqueid_to_index.find(src_col->unique_id()) == _field_uniqueid_to_index.end()) { - CHECK(!src_col->is_key()) - << src_col->name() << " is key column, should not be dropped."; - ColumnPB src_col_pb; - // There are some pointer in tablet column, not sure the reference relation, so - // that deep copy it. - src_col->to_schema_pb(&src_col_pb); - TabletColumn new_col(src_col_pb); - append_column(new_col, TabletSchema::ColumnType::DROPPED); - } - } -} - TabletSchemaSPtr TabletSchema::copy_without_variant_extracted_columns() { TabletSchemaSPtr copy = std::make_shared(); copy->shawdow_copy_without_columns(*this); @@ -1385,16 +1352,6 @@ TabletSchemaSPtr TabletSchema::copy_without_variant_extracted_columns() { return copy; } -// Dropped column is in _field_uniqueid_to_index but not in _field_name_to_index -// Could refer to append_column method -bool TabletSchema::is_dropped_column(const TabletColumn& col) const { - CHECK(_field_uniqueid_to_index.find(col.unique_id()) != _field_uniqueid_to_index.end()) - << "could not find col with unique id = " << col.unique_id() - << " and name = " << col.name() << " table_id=" << _table_id; - auto it = _field_name_to_index.find(StringRef {col.name()}); - return it == _field_name_to_index.end() || _cols[it->second]->unique_id() != col.unique_id(); -} - void TabletSchema::copy_extracted_columns(const TabletSchema& src_schema) { std::unordered_set variant_columns; for (const auto& col : columns()) { @@ -1744,71 +1701,26 @@ const TabletIndex* TabletSchema::get_index(int32_t col_unique_id, IndexType inde return nullptr; } -Block TabletSchema::create_block( - const std::vector& return_columns, - const std::unordered_set* tablet_columns_need_convert_null) const { +Block TabletSchema::create_storage_block(const std::vector& column_ids) const { Block block; - for (int i = 0; i < return_columns.size(); ++i) { - const ColumnId cid = return_columns[i]; + for (int i = 0; i < column_ids.size(); ++i) { + const ColumnId cid = column_ids[i]; const auto& col = *_cols[cid]; - bool is_nullable = (tablet_columns_need_convert_null != nullptr && - tablet_columns_need_convert_null->find(cid) != - tablet_columns_need_convert_null->end()); - auto data_type = DataTypeFactory::instance().create_data_type(col, is_nullable); - if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT || - col.type() == FieldType::OLAP_FIELD_TYPE_MAP || - col.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - if (_pruned_columns_data_type.contains(col.unique_id())) { - data_type = _pruned_columns_data_type.at(col.unique_id()); - } - } - - if (_vir_col_idx_to_unique_id.contains(cid)) { - block.insert({ColumnNothing::create(0), data_type, col.name()}); - VLOG_DEBUG << fmt::format( - "Create block from tablet schema, column cid {} is virtual column, col_name: " - "{}, col_unique_id: {}, type {}", - cid, col.name(), col.unique_id(), data_type->get_name()); - } else { - block.insert({data_type->create_column(), data_type, col.name()}); - } + auto data_type = DataTypeFactory::instance().create_data_type(col); + block.insert({data_type->create_column(), data_type, col.name()}); } return block; } -Block TabletSchema::create_block() const { +Block TabletSchema::create_storage_block() const { Block block; for (const auto& col : _cols) { - if (is_dropped_column(*col)) { - continue; - } - auto data_type = DataTypeFactory::instance().create_data_type(*col); - if (col->type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { - if (_pruned_columns_data_type.contains(col->unique_id())) { - data_type = _pruned_columns_data_type.at(col->unique_id()); - } - } block.insert({data_type->create_column(), data_type, col->name()}); } return block; } -Block TabletSchema::create_block_by_cids(const std::vector& cids) const { - Block block; - for (const auto& cid : cids) { - const auto& col = *_cols[cid]; - auto data_type = DataTypeFactory::instance().create_data_type(col); - if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { - if (_pruned_columns_data_type.contains(col.unique_id())) { - data_type = _pruned_columns_data_type.at(col.unique_id()); - } - } - block.insert({data_type->create_column(), data_type, col.name()}); - } - return block; -} - bool operator==(const TabletColumn& a, const TabletColumn& b) { if (a._unique_id != b._unique_id) return false; if (a._col_name != b._col_name) return false; diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index 7b784d0b615201..ff078382dec937 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -406,7 +406,7 @@ using PathSet = phmap::flat_hash_set; class TabletSchema : public MetadataAdder { public: - enum class ColumnType { NORMAL = 0, DROPPED = 1, VARIANT = 2 }; + enum class ColumnType { NORMAL = 0, VARIANT = 1 }; // TODO(yingchun): better to make constructor as private to avoid // manually init members incorrectly, and define a new function like // void create_from_pb(const TabletSchemaPB& schema, TabletSchema* tablet_schema). @@ -605,10 +605,9 @@ class TabletSchema : public MetadataAdder { // If schema version is not set, it should be -1 int32_t schema_version() const { return _schema_version; } void clear_columns(); - Block create_block( - const std::vector& return_columns, - const std::unordered_set* tablet_columns_need_convert_null = nullptr) const; - Block create_block() const; + // Create Blocks with physical TabletSchema column types and ordinals. + Block create_storage_block(const std::vector& column_ids) const; + Block create_storage_block() const; void set_schema_version(int32_t version) { _schema_version = version; } void set_auto_increment_column(const std::string& auto_increment_column) { _auto_increment_column = auto_increment_column; @@ -623,21 +622,6 @@ class TabletSchema : public MetadataAdder { const OlapTableIndexSchema* index, const TabletSchema& out_tablet_schema); - // Merge columns that not exit in current schema, these column is dropped in current schema - // but they are useful in some cases. For example, - // 1. origin schema is ColA, ColB - // 2. insert values 1, 2 - // 3. delete where ColB = 2 - // 4. drop ColB - // 5. insert values 3 - // 6. add column ColB, although it is name ColB, but it is different with previous ColB, the new ColB we name could call ColB' - // 7. insert value 4, 5 - // Then the read schema should be ColA, ColB, ColB' because the delete predicate need ColB to remove related data. - // Because they have same name, so that the dropped column should not be added to the map, only with unique id. - void merge_dropped_columns(const TabletSchema& src_schema); - - bool is_dropped_column(const TabletColumn& col) const; - // copy extracted columns from src_schema void copy_extracted_columns(const TabletSchema& src_schema); @@ -692,8 +676,6 @@ class TabletSchema : public MetadataAdder { return str; } - Block create_block_by_cids(const std::vector& cids) const; - std::shared_ptr copy_without_variant_extracted_columns(); InvertedIndexStorageFormatPB get_inverted_index_storage_format() const { return _inverted_index_storage_format; @@ -755,18 +737,6 @@ class TabletSchema : public MetadataAdder { bool has_seq_map() const { return !_seq_col_idx_to_value_cols_idx.empty(); } - const std::unordered_map& value_col_idx_to_seq_col_idx() const { - return _value_col_idx_to_seq_col_idx; - } - - void add_pruned_columns_data_type(int32_t col_unique_id, DataTypePtr data_type) { - _pruned_columns_data_type[col_unique_id] = std::move(data_type); - } - - void clear_pruned_columns_data_type() { _pruned_columns_data_type.clear(); } - - bool has_pruned_columns() const { return !_pruned_columns_data_type.empty(); } - TabletStorageFormatPB storage_format() const { return _storage_format; } void set_storage_format(TabletStorageFormatPB v) { _storage_format = v; } @@ -842,7 +812,6 @@ class TabletSchema : public MetadataAdder { bool _deprecated_enable_variant_flatten_nested = false; std::map _vir_col_idx_to_unique_id; - std::map _pruned_columns_data_type; // value: extracted path set and sparse path set std::unordered_map _path_set_info_map; @@ -863,8 +832,6 @@ class TabletSchema : public MetadataAdder { std::unordered_map _value_col_uid_to_seq_col_uid; // Sequence column index mapping to value column index std::unordered_map> _seq_col_idx_to_value_cols_idx; - // Value column index mapping to sequence column index(also map sequence column it self) - std::unordered_map _value_col_idx_to_seq_col_idx; }; bool operator==(const TabletSchema& a, const TabletSchema& b); diff --git a/be/src/storage/task/engine_checksum_task.cpp b/be/src/storage/task/engine_checksum_task.cpp index 01f879aa731076..461b4a52285316 100644 --- a/be/src/storage/task/engine_checksum_task.cpp +++ b/be/src/storage/task/engine_checksum_task.cpp @@ -19,8 +19,11 @@ #include +#include #include +#include #include +#include #include #include "core/block/block.h" @@ -32,6 +35,7 @@ #include "storage/olap_common.h" #include "storage/olap_define.h" #include "storage/rowset/rowset.h" +#include "storage/schema.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_manager.h" @@ -39,6 +43,37 @@ #include "storage/utils.h" namespace doris { +namespace { + +Status prepare_checksum_reader(TabletSharedPtr tablet, + const std::vector& input_rowsets, + TabletReader::ReaderParams* reader_params, Block* block) { + reader_params->tablet = tablet; + reader_params->reader_type = ReaderType::READER_CHECKSUM; + reader_params->version = + Version(input_rowsets.front()->start_version(), input_rowsets.back()->end_version()); + + TabletReadSource read_source; + for (const auto& rowset : input_rowsets) { + RowsetReaderSharedPtr rs_reader; + RETURN_IF_ERROR(rowset->create_reader(&rs_reader)); + read_source.rs_splits.emplace_back(std::move(rs_reader)); + } + read_source.fill_delete_predicates(); + reader_params->set_read_source(std::move(read_source)); + + std::vector rowset_metas(input_rowsets.size()); + std::transform(input_rowsets.begin(), input_rowsets.end(), rowset_metas.begin(), + [](const RowsetSharedPtr& rowset) { return rowset->rowset_meta(); }); + auto read_tablet_schema = tablet->tablet_schema_with_merged_max_schema_version(rowset_metas); + reader_params->tablet_schema = read_tablet_schema; + reader_params->read_schema = std::make_shared(read_tablet_schema->columns()); + *block = reader_params->read_schema->create_read_block(); + + return Status::OK(); +} + +} // namespace EngineChecksumTask::EngineChecksumTask(StorageEngine& engine, TTabletId tablet_id, TSchemaHash schema_hash, TVersion version, @@ -90,8 +125,7 @@ Status EngineChecksumTask::_compute_checksum() { return std::move(ret.error()); } - RETURN_IF_ERROR(TabletReader::init_reader_params_and_create_block( - tablet, ReaderType::READER_CHECKSUM, input_rowsets, &reader_params, &block)); + RETURN_IF_ERROR(prepare_checksum_reader(tablet, input_rowsets, &reader_params, &block)); } size_t input_size = 0; for (const auto& rowset : input_rowsets) { diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index f85fcee9ddf44e..02bd557f0540f7 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -29,6 +29,7 @@ #include "storage/olap_define.h" #include "storage/rowset/beta_rowset.h" #include "storage/rowset/rowset_writer_context.h" +#include "storage/schema.h" #include "storage/segment/segment_loader.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_schema.h" @@ -565,8 +566,8 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta OlapReaderStatistics stats; read_options.stats = &stats; read_options.tablet_schema = output_rowset_schema; - std::shared_ptr schema = - std::make_shared(output_rowset_schema->columns(), return_columns); + auto schema = + std::make_shared(output_rowset_schema->columns(), return_columns); std::unique_ptr iter; auto res = seg_ptr->new_iterator(schema, read_options, &iter); DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_create_iterator_error", { @@ -579,7 +580,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta return Status::Error(res.to_string()); } - auto block = Block::create_unique(output_rowset_schema->create_block(return_columns)); + auto block = Block::create_unique(schema->create_read_block()); while (true) { auto status = iter->next_batch(block.get()); DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_iterator_next_batch_error", { diff --git a/be/test/core/column/common_column_test.h b/be/test/core/column/common_column_test.h index 149776b238a485..2f16153a84bb47 100644 --- a/be/test/core/column/common_column_test.h +++ b/be/test/core/column/common_column_test.h @@ -1575,7 +1575,8 @@ class CommonColumnTest : public ::testing::Test { //virtual void //for_each_subcolumn (ColumnCallback) //virtual void - //replace_column_data (const IColumn &, size_t row, size_t self_row=0) used in BlockReader(VerticalBlockReader)::_copy_agg_data() for agg value data + // replace_column_data(const IColumn&, size_t row, size_t self_row = 0) is used by + // VerticalBlockReader::_copy_agg_data() for aggregate value data. // the passed column must be non-variable length column: like columnVector... static void assert_replace_column_data_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) { diff --git a/be/test/exec/common/schema_util_rowset_test.cpp b/be/test/exec/common/schema_util_rowset_test.cpp index 2f3f32226ed7ae..fd78f8254202f1 100644 --- a/be/test/exec/common/schema_util_rowset_test.cpp +++ b/be/test/exec/common/schema_util_rowset_test.cpp @@ -197,7 +197,7 @@ static RowsetWriterContext rowset_writer_context(const std::unique_ptr& } static RowsetSharedPtr create_rowset(auto& rowset_writer, const TabletSchemaSPtr& tablet_schema) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); fill_block_with_test_data(&block, 1000); auto st = rowset_writer->add_block(&block); EXPECT_TRUE(st.ok()) << st.msg(); @@ -528,7 +528,18 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche StorageReadOptions type_opts; type_opts.tablet_schema = out_rowset->tablet_schema(); type_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - auto data_type = segment->get_data_type_of(subcolumn_in_sparse, type_opts); + TabletColumn non_variant_column; + non_variant_column.set_name("non_variant_column"); + non_variant_column.set_type(FieldType::OLAP_FIELD_TYPE_INT); + DataTypePtr expected_read_type = std::make_shared(); + auto data_type = segment->get_data_type_of(non_variant_column, expected_read_type, type_opts); + EXPECT_TRUE(data_type->equals(*expected_read_type)); + + auto get_data_type = [&](const TabletColumn& column) { + return segment->get_data_type_of(column, column.get_vec_type(), type_opts); + }; + + data_type = get_data_type(subcolumn_in_sparse); EXPECT_TRUE(data_type != nullptr); EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_VARIANT); @@ -537,7 +548,7 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche subcolumn_in_sparse.set_path_info(PathInData("v1.keya")); // keya is not in the segment, return string type; - data_type = segment->get_data_type_of(subcolumn_in_sparse, type_opts); + data_type = get_data_type(subcolumn_in_sparse); EXPECT_TRUE(data_type != nullptr); EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_STRING); @@ -565,7 +576,7 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche value_col.set_is_nullable(true); sparse_typed_col.add_sub_column(value_col); } - data_type = segment->get_data_type_of(sparse_typed_col, type_opts); + data_type = get_data_type(sparse_typed_col); EXPECT_TRUE(data_type != nullptr); EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_MAP); @@ -575,7 +586,7 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche subcolumn_in_sparse._column_path->has_nested = true; // keyb has nested part, return int type; - data_type = segment->get_data_type_of(subcolumn_in_sparse, type_opts); + data_type = get_data_type(subcolumn_in_sparse); EXPECT_TRUE(data_type != nullptr); EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_INT); @@ -583,7 +594,7 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche subcolumn_in_sparse.set_name("v1.key1"); subcolumn_in_sparse.set_type(FieldType::OLAP_FIELD_TYPE_STRING); subcolumn_in_sparse.set_path_info(PathInData("v1.key1")); - data_type = segment->get_data_type_of(subcolumn_in_sparse, type_opts); + data_type = get_data_type(subcolumn_in_sparse); EXPECT_TRUE(data_type != nullptr); EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_STRING); diff --git a/be/test/exec/operator/scan_normalize_predicate_test.cpp b/be/test/exec/operator/scan_normalize_predicate_test.cpp index d6fcd73d2be4ea..dcc65552bd58ba 100644 --- a/be/test/exec/operator/scan_normalize_predicate_test.cpp +++ b/be/test/exec/operator/scan_normalize_predicate_test.cpp @@ -19,6 +19,7 @@ #include #include +#include "common/object_pool.h" #include "core/block/block.h" #include "core/column/column_const.h" #include "core/data_type/data_type_factory.hpp" @@ -43,7 +44,11 @@ struct ScanNormalizePredicate : public ::testing::Test { void SetUp() override { state = std::make_shared(); op = std::make_shared(); + op->_output_row_descriptor = std::make_unique( + std::vector {std::make_shared()}, &pool); + op->_output_tuple_desc = op->_output_row_descriptor->tuple_descriptors()[0]; } + ObjectPool pool; std::shared_ptr state; std::shared_ptr op; }; @@ -206,6 +211,10 @@ TEST_F(ScanNormalizePredicate, test_is_predicate_acting_on_slot1) { conjunct_expr_root->root(), new_root); EXPECT_TRUE(st.ok()); std::cout << st.msg() << std::endl; + + const auto& predicates = local_state->_slot_id_to_predicates[SlotId]; + ASSERT_EQ(predicates.size(), 1); + EXPECT_EQ(predicates.front()->column_id(), 0); } EXPECT_TRUE(local_state->_slot_id_to_value_range.contains(SlotId)); diff --git a/be/test/exec/scan/vgeneric_iterators_test.cpp b/be/test/exec/scan/vgeneric_iterators_test.cpp index 71b71847b86a8a..5e70d54590cf10 100644 --- a/be/test/exec/scan/vgeneric_iterators_test.cpp +++ b/be/test/exec/scan/vgeneric_iterators_test.cpp @@ -50,7 +50,7 @@ class VGenericIteratorsTest : public testing::Test { virtual ~VGenericIteratorsTest() {} }; -static std::vector create_col_schemas() { +static ReadSchema create_schema() { std::vector col_schemas; auto c1 = std::make_shared(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, FieldType::OLAP_FIELD_TYPE_SMALLINT, true); @@ -65,25 +65,13 @@ static std::vector create_col_schemas() { col_schemas.emplace_back( std::make_shared(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_SUM, FieldType::OLAP_FIELD_TYPE_BIGINT, true)); - return col_schemas; + return ReadSchema(std::move(col_schemas)); } -static Schema create_schema() { - std::vector col_schemas = create_col_schemas(); - - std::vector column_ids(col_schemas.size()); - for (uint32_t cid = 0; cid < column_ids.size(); ++cid) { - column_ids[cid] = cid; - } - - Schema schema(col_schemas, column_ids); - return schema; -} - -static void create_block(Schema& schema, Block& block) { +static void create_block(ReadSchema& schema, Block& block) { for (auto& column_desc : schema.columns()) { EXPECT_TRUE(column_desc); - auto data_type = Schema::get_data_type_ptr(*column_desc); + auto data_type = column_desc->get_vec_type(); EXPECT_NE(data_type, nullptr); auto column = data_type->create_column(); ColumnWithTypeAndName ctn(std::move(column), data_type, column_desc->name()); @@ -168,7 +156,7 @@ TEST(VGenericIteratorsTest, StatisticsIteratorPreservesNullForNullableChar) { .ok()); std::vector column_ids {0, 1}; - Schema schema(tablet_schema->columns(), column_ids); + ReadSchema schema(tablet_schema->columns(), column_ids); VStatisticsIterator iterator(segment, schema); StorageReadOptions read_options; OlapReaderStatistics stats; @@ -193,7 +181,7 @@ TEST(VGenericIteratorsTest, StatisticsIteratorPreservesNullForNullableChar) { TEST(VGenericIteratorsTest, Union) { auto schema = create_schema(); - auto output_schema = std::make_shared(schema); + auto output_schema = std::make_shared(schema); std::vector inputs; inputs.push_back(new_auto_increment_iterator(schema, 100)); @@ -238,7 +226,7 @@ TEST(VGenericIteratorsTest, Union) { TEST(VGenericIteratorsTest, MergeAgg) { EXPECT_TRUE(1); auto schema = create_schema(); - auto output_schema = std::make_shared(schema); + auto output_schema = std::make_shared(schema); std::vector inputs; inputs.push_back(new_auto_increment_iterator(schema, 100)); @@ -288,7 +276,7 @@ TEST(VGenericIteratorsTest, MergeAgg) { TEST(VGenericIteratorsTest, MergeUnique) { EXPECT_TRUE(1); auto schema = create_schema(); - auto output_schema = std::make_shared(schema); + auto output_schema = std::make_shared(schema); std::vector inputs; inputs.push_back(new_auto_increment_iterator(schema, 100)); @@ -331,7 +319,7 @@ TEST(VGenericIteratorsTest, MergeUnique) { class SeqColumnUtIterator : public RowwiseIterator { public: // Will generate num_rows rows in total - SeqColumnUtIterator(const Schema& schema, size_t num_rows, size_t rows_returned, + SeqColumnUtIterator(const ReadSchema& schema, size_t num_rows, size_t rows_returned, size_t seq_col_idx, size_t seq_col_rows_returned) : _schema(schema), _num_rows(num_rows), @@ -346,7 +334,7 @@ class SeqColumnUtIterator : public RowwiseIterator { Status next_batch(Block* block) override { int row_idx = 0; while (_rows_returned < _num_rows) { - for (int j = 0; j < _schema.num_columns(); ++j) { + for (int j = 0; j < _schema.num_block_columns(); ++j) { ColumnWithTypeAndName& vc = block->get_by_position(j); IColumn& vi = (IColumn&)(*vc.column); @@ -390,9 +378,9 @@ class SeqColumnUtIterator : public RowwiseIterator { return Status::EndOfFile("End of VAutoIncrementIterator"); } - const Schema& schema() const override { return _schema; } + const ReadSchema& schema() const override { return _schema; } - const Schema& _schema; + const ReadSchema& _schema; size_t _num_rows; size_t _rows_returned; int _seq_col_idx = -1; @@ -402,7 +390,7 @@ class SeqColumnUtIterator : public RowwiseIterator { TEST(VGenericIteratorsTest, MergeWithSeqColumn) { EXPECT_TRUE(1); auto schema = create_schema(); - auto output_schema = std::make_shared(schema); + auto output_schema = std::make_shared(schema); std::vector inputs; int seq_column_id = 2; @@ -448,7 +436,7 @@ TEST(VGenericIteratorsTest, MergeWithSeqColumn) { // iterator should keep exactly one row whose seq value is the smallest (0). TEST(VGenericIteratorsTest, MergeWithSeqColumnSmallSeqFirst) { auto schema = create_schema(); - auto output_schema = std::make_shared(schema); + auto output_schema = std::make_shared(schema); std::vector inputs; int seq_column_id = 2; @@ -486,100 +474,4 @@ TEST(VGenericIteratorsTest, MergeWithSeqColumnSmallSeqFirst) { EXPECT_EQ(0, actual_value); } -// Emits num_rows rows for a schema whose projection (col_ids) may be narrower than the -// full tablet schema. The filled block layout follows the projection, matching how -// VMergeIteratorContext::block_reset builds its block from the output schema. -class ProjectedColumnsUtIterator : public RowwiseIterator { -public: - ProjectedColumnsUtIterator(Schema schema, size_t num_rows) - : _schema(std::move(schema)), _num_rows(num_rows) {} - ~ProjectedColumnsUtIterator() override = default; - - Status init(const StorageReadOptions& opts) override { return Status::OK(); } - - Status next_batch(Block* block) override { - if (_rows_returned >= _num_rows) { - return Status::EndOfFile("End of ProjectedColumnsUtIterator"); - } - while (_rows_returned < _num_rows) { - for (size_t j = 0; j < _schema.num_column_ids(); ++j) { - ColumnWithTypeAndName& vc = block->get_by_position(j); - IColumn& vi = (IColumn&)(*vc.column); - - char data[16] = {}; - size_t data_len = 0; - const auto* col_schema = _schema.column(_schema.column_id(j)); - switch (col_schema->type()) { - case FieldType::OLAP_FIELD_TYPE_SMALLINT: - *(int16_t*)data = static_cast(_rows_returned); - data_len = sizeof(int16_t); - break; - case FieldType::OLAP_FIELD_TYPE_INT: - *(int32_t*)data = static_cast(_rows_returned); - data_len = sizeof(int32_t); - break; - case FieldType::OLAP_FIELD_TYPE_BIGINT: - *(int64_t*)data = static_cast(_rows_returned); - data_len = sizeof(int64_t); - break; - default: - break; - } - - vi.insert_data(data, data_len); - } - ++_rows_returned; - } - return Status::OK(); - } - - const Schema& schema() const override { return _schema; } - -private: - Schema _schema; - size_t _num_rows; - size_t _rows_returned = 0; -}; - -// The merge-heap comparator compares the first num_key_columns block positions when no -// explicit compare columns are given, and num_key_columns counts the key columns of the -// WHOLE tablet schema. A projection narrower than the key prefix must be rejected at -// init time with an error instead of crashing inside std::push_heap (issue #66390: a -// ROW-binlog APPEND_ONLY scan projected value columns only). -TEST(VGenericIteratorsTest, MergeRejectsProjectionMissingKeyPrefix) { - // Full tablet schema: k0(smallint), k1(int), v2(bigint); project only v2. - Schema projected(create_col_schemas(), std::vector {2}); - auto output_schema = std::make_shared(projected); - - std::vector inputs; - inputs.push_back(std::make_unique(projected, 10)); - inputs.push_back(std::make_unique(projected, 10)); - - auto iter = new_merge_iterator(std::move(inputs), -1, false, false, nullptr, output_schema); - StorageReadOptions opts; - auto st = iter->init(opts); - EXPECT_FALSE(st.ok()); - EXPECT_TRUE(st.to_string().find("compare contract violated") != std::string::npos) - << st.to_string(); -} - -// Same as above, but the projection has enough columns while not starting with the full -// ordered key prefix (k0 is missing): position 0 would be compared as if it were k0. -TEST(VGenericIteratorsTest, MergeRejectsProjectionWithoutLeadingKey) { - // Full tablet schema: k0(smallint), k1(int), v2(bigint); project {k1, v2}. - Schema projected(create_col_schemas(), std::vector {1, 2}); - auto output_schema = std::make_shared(projected); - - std::vector inputs; - inputs.push_back(std::make_unique(projected, 10)); - inputs.push_back(std::make_unique(projected, 10)); - - auto iter = new_merge_iterator(std::move(inputs), -1, false, false, nullptr, output_schema); - StorageReadOptions opts; - auto st = iter->init(opts); - EXPECT_FALSE(st.ok()); - EXPECT_TRUE(st.to_string().find("compare contract violated") != std::string::npos) - << st.to_string(); -} - } // namespace doris diff --git a/be/test/exprs/function/function_is_null_test.cpp b/be/test/exprs/function/function_is_null_test.cpp index b10e7d76f31b05..afcf80f2119799 100644 --- a/be/test/exprs/function/function_is_null_test.cpp +++ b/be/test/exprs/function/function_is_null_test.cpp @@ -158,7 +158,7 @@ TEST_F(FunctionIsNullTest, gc_binlogs_test) { EXPECT_TRUE(res.has_value()) << res.error(); const auto& rowset_writer = res.value(); - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); Field key = Field::create_field(10); @@ -324,7 +324,7 @@ TEST_F(FunctionIsNullTest, evaluate_inverted_index_corner_cases) { EXPECT_TRUE(res.has_value()) << res.error(); const auto& rowset_writer = res.value(); - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Create block with NO null values to test the scenario where diff --git a/be/test/exprs/vexpr_evalute_inverted_index_test.cpp b/be/test/exprs/vexpr_evalute_inverted_index_test.cpp index 9cae773f73e60b..cace598b30f3d2 100644 --- a/be/test/exprs/vexpr_evalute_inverted_index_test.cpp +++ b/be/test/exprs/vexpr_evalute_inverted_index_test.cpp @@ -157,13 +157,12 @@ TEST(TExprInvertedIndexTest, test_expr_evaluate_inverted_index) { std::unordered_map> common_expr_inverted_index_status; - std::vector col_ids; std::vector> index_iterators; std::vector storage_types; doris::segment_v2::ColumnIteratorOptions column_iter_opts; auto inverted_index_context = std::make_shared( - col_ids, index_iterators, storage_types, common_expr_inverted_index_status, nullptr, - nullptr, column_iter_opts); + index_iterators, storage_types, common_expr_inverted_index_status, nullptr, nullptr, + column_iter_opts); expr_ctx.set_index_context(inverted_index_context); doris::RuntimeState state; doris::RowDescriptor row_desc; diff --git a/be/test/exprs/vsearch_expr_test.cpp b/be/test/exprs/vsearch_expr_test.cpp index 9703a4650934f7..7c773818611191 100644 --- a/be/test/exprs/vsearch_expr_test.cpp +++ b/be/test/exprs/vsearch_expr_test.cpp @@ -98,13 +98,12 @@ VExprSPtr create_slot_ref(int column_id, const std::string& column_name) { } std::shared_ptr make_inverted_context( - std::vector& col_ids, std::vector>& index_iterators, std::vector& storage_types, std::unordered_map>& status_map) { segment_v2::ColumnIteratorOptions column_iter_opts; - return std::make_shared(col_ids, index_iterators, storage_types, status_map, - nullptr, nullptr, column_iter_opts); + return std::make_shared(index_iterators, storage_types, status_map, nullptr, + nullptr, column_iter_opts); } std::shared_ptr make_segment_with_variant_parent() { @@ -1301,12 +1300,11 @@ TEST_F(VSearchExprTest, FastExecuteReturnsPrecomputedColumn) { auto expr = VSearchExpr::create_shared(test_node); auto context = std::make_shared(expr); - std::vector col_ids; std::vector> index_iterators; std::vector storage_types; std::unordered_map> status_map; - auto inverted_ctx = make_inverted_context(col_ids, index_iterators, storage_types, status_map); + auto inverted_ctx = make_inverted_context(index_iterators, storage_types, status_map); MutableColumnPtr result_column = ColumnUInt8::create(); inverted_ctx->set_index_result_column_for_expr(expr.get(), std::move(result_column)); context->set_index_context(inverted_ctx); @@ -1319,36 +1317,15 @@ TEST_F(VSearchExprTest, FastExecuteReturnsPrecomputedColumn) { EXPECT_EQ(0, result_column_id); } -TEST_F(VSearchExprTest, EvaluateInvertedIndexFailsWithoutStorageType) { - auto expr = VSearchExpr::create_shared(test_node); - expr->add_child(create_slot_ref(0, "title")); - - std::vector col_ids = {0}; - std::vector> index_iterators; - index_iterators.emplace_back(std::make_unique()); - std::vector storage_types; // intentionally empty - std::unordered_map> status_map; - status_map[0][expr.get()] = false; - - auto inverted_ctx = make_inverted_context(col_ids, index_iterators, storage_types, status_map); - auto context = std::make_shared(expr); - context->set_index_context(inverted_ctx); - - auto status = expr->evaluate_inverted_index(context.get(), 128); - EXPECT_FALSE(status.ok()); - EXPECT_EQ(ErrorCode::INTERNAL_ERROR, status.code()); -} - TEST_F(VSearchExprTest, EvaluateInvertedIndexWithUnsupportedChildReturnsError) { auto expr = VSearchExpr::create_shared(test_node); expr->add_child(std::make_shared()); - std::vector col_ids; std::vector> index_iterators; std::vector storage_types; std::unordered_map> status_map; - auto inverted_ctx = make_inverted_context(col_ids, index_iterators, storage_types, status_map); + auto inverted_ctx = make_inverted_context(index_iterators, storage_types, status_map); auto context = std::make_shared(expr); context->set_index_context(inverted_ctx); @@ -1361,14 +1338,14 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexHandlesMissingIterators) { auto expr = VSearchExpr::create_shared(test_node); expr->add_child(create_slot_ref(0, "title")); - std::vector col_ids = {0}; std::vector> index_iterators; - index_iterators.emplace_back(nullptr); // iterator unavailable - std::vector storage_types; // unused because iterator is null + index_iterators.emplace_back(nullptr); // iterator unavailable + std::vector storage_types; + storage_types.emplace_back("stored_title", std::make_shared()); std::unordered_map> status_map; status_map[0][expr.get()] = false; - auto inverted_ctx = make_inverted_context(col_ids, index_iterators, storage_types, status_map); + auto inverted_ctx = make_inverted_context(index_iterators, storage_types, status_map); auto context = std::make_shared(expr); context->set_index_context(inverted_ctx); @@ -1393,7 +1370,6 @@ TEST_F(VSearchExprTest, MissingVariantChildIteratorDoesNotUseParentIterator) { // Scan column 0 is the Variant parent and has an iterator. Scan column 1 is the requested // child and intentionally has none. SEARCH must not reinterpret the parent iterator as the // child's index. - std::vector col_ids = {0, 1}; std::vector> index_iterators; index_iterators.emplace_back(std::make_unique()); index_iterators.emplace_back(nullptr); @@ -1407,9 +1383,8 @@ TEST_F(VSearchExprTest, MissingVariantChildIteratorDoesNotUseParentIterator) { // Keep the parent in the segment schema so a parent-rebinding implementation would find it. segment_v2::ColumnIteratorOptions column_iter_opts; auto segment = make_segment_with_variant_parent(); - auto inverted_ctx = - std::make_shared(col_ids, index_iterators, storage_types, status_map, - nullptr, segment.get(), column_iter_opts); + auto inverted_ctx = std::make_shared( + index_iterators, storage_types, status_map, nullptr, segment.get(), column_iter_opts); auto context = std::make_shared(expr); context->set_index_context(inverted_ctx); @@ -1440,11 +1415,10 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexNestedFallbackReturnsNotSupportedIn nested_node.__isset.search_param = true; auto expr = VSearchExpr::create_shared(nested_node); - std::vector col_ids; std::vector> index_iterators; std::vector storage_types; std::unordered_map> status_map; - auto inverted_ctx = make_inverted_context(col_ids, index_iterators, storage_types, status_map); + auto inverted_ctx = make_inverted_context(index_iterators, storage_types, status_map); auto context = std::make_shared(expr); context->set_index_context(inverted_ctx); @@ -1466,7 +1440,6 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexPropagatesFunctionFailure) { auto expr = VSearchExpr::create_shared(test_node); expr->add_child(create_slot_ref(0, "title")); - std::vector col_ids = {0}; std::vector> index_iterators; index_iterators.emplace_back(std::make_unique()); std::vector storage_types; @@ -1474,7 +1447,7 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexPropagatesFunctionFailure) { std::unordered_map> status_map; status_map[0][expr.get()] = false; - auto inverted_ctx = make_inverted_context(col_ids, index_iterators, storage_types, status_map); + auto inverted_ctx = make_inverted_context(index_iterators, storage_types, status_map); auto context = std::make_shared(expr); context->set_index_context(inverted_ctx); diff --git a/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp b/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp index 9afa6c3cb939f0..14696f15889aee 100644 --- a/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp +++ b/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp @@ -71,17 +71,15 @@ class OlapMeta; static const uint32_t MAX_PATH_LEN = 1024; static StorageEngine* engine_ref = nullptr; -static std::shared_ptr create_full_schema(const TabletSchemaSPtr& tablet_schema) { +static std::shared_ptr create_full_schema(const TabletSchemaSPtr& tablet_schema) { size_t num_columns = tablet_schema->num_columns(); if (num_columns > 0 && tablet_schema->columns().back()->name() == BeConsts::ROW_STORE_COL) { --num_columns; } - std::vector column_ids(num_columns); - for (uint32_t cid = 0; cid < num_columns; ++cid) { - column_ids[cid] = cid; - } - return std::make_shared(tablet_schema->columns(), column_ids); + std::vector columns(tablet_schema->columns().begin(), + tablet_schema->columns().begin() + num_columns); + return std::make_shared(std::move(columns)); } static void set_up() { @@ -354,10 +352,10 @@ TEST_F(TestDeltaWriterClusterKey, vec_sequence_col) { opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; - std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); + std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); auto s = segments[0]->new_iterator(schema, opts, &iter); ASSERT_TRUE(s.ok()); - auto read_block = rowset->tablet_schema()->create_block(); + auto read_block = schema->create_read_block(); res = iter->next_batch(&read_block); ASSERT_TRUE(res.ok()) << res; ASSERT_EQ(rows, read_block.rows()); diff --git a/be/test/load/delta_writer/delta_writer_test.cpp b/be/test/load/delta_writer/delta_writer_test.cpp index e2b9419b8973d5..e0f68527f7391d 100644 --- a/be/test/load/delta_writer/delta_writer_test.cpp +++ b/be/test/load/delta_writer/delta_writer_test.cpp @@ -72,17 +72,15 @@ class OlapMeta; static const uint32_t MAX_PATH_LEN = 1024; static StorageEngine* engine_ref = nullptr; -static std::shared_ptr create_full_schema(const TabletSchemaSPtr& tablet_schema) { +static std::shared_ptr create_full_schema(const TabletSchemaSPtr& tablet_schema) { size_t num_columns = tablet_schema->num_columns(); if (num_columns > 0 && tablet_schema->columns().back()->name() == BeConsts::ROW_STORE_COL) { --num_columns; } - std::vector column_ids(num_columns); - for (uint32_t cid = 0; cid < num_columns; ++cid) { - column_ids[cid] = cid; - } - return std::make_shared(tablet_schema->columns(), column_ids); + std::vector columns(tablet_schema->columns().begin(), + tablet_schema->columns().begin() + num_columns); + return std::make_shared(std::move(columns)); } static void set_up() { @@ -838,10 +836,10 @@ TEST_F(TestDeltaWriter, vec_sequence_col) { opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; - std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); + std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); auto s = segments[0]->new_iterator(schema, opts, &iter); ASSERT_TRUE(s.ok()); - auto read_block = rowset->tablet_schema()->create_block(); + auto read_block = schema->create_read_block(); res = iter->next_batch(&read_block); ASSERT_TRUE(res.ok()) << res; ASSERT_EQ(1, read_block.rows()); @@ -1046,12 +1044,12 @@ TEST_F(TestDeltaWriter, vec_sequence_col_concurrent_write) { opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap().get_agg( {rowset1->rowset_id(), 0, cur_version})); std::unique_ptr iter; - std::shared_ptr schema = create_full_schema(rowset1->tablet_schema()); + std::shared_ptr schema = create_full_schema(rowset1->tablet_schema()); std::vector segments; static_cast(((BetaRowset*)rowset1.get())->load_segments(&segments)); auto s = segments[0]->new_iterator(schema, opts, &iter); ASSERT_TRUE(s.ok()); - auto read_block = rowset1->tablet_schema()->create_block(); + auto read_block = schema->create_read_block(); res = iter->next_batch(&read_block); ASSERT_TRUE(res.ok()) << res; // key of (10, 123) is deleted @@ -1074,12 +1072,12 @@ TEST_F(TestDeltaWriter, vec_sequence_col_concurrent_write) { opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap().get_agg( {rowset2->rowset_id(), 0, cur_version})); std::unique_ptr iter; - std::shared_ptr schema = create_full_schema(rowset2->tablet_schema()); + std::shared_ptr schema = create_full_schema(rowset2->tablet_schema()); std::vector segments; static_cast(((BetaRowset*)rowset2.get())->load_segments(&segments)); auto s = segments[0]->new_iterator(schema, opts, &iter); ASSERT_TRUE(s.ok()); - auto read_block = rowset2->tablet_schema()->create_block(); + auto read_block = schema->create_read_block(); res = iter->next_batch(&read_block); ASSERT_TRUE(res.ok()); // key of (20, 123) is deleted, because it's seq value is low diff --git a/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp b/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp index 5c498ea9f1fd1d..7f337a5ded63c6 100644 --- a/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp +++ b/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp @@ -252,7 +252,7 @@ class CloudGroupRowsetBuilderWriterTest : public testing::Test { } Block create_block(int start_key, int num_rows) const { - Block block = _tablet->tablet_schema()->create_block(); + Block block = _tablet->tablet_schema()->create_storage_block(); { auto columns_guard = block.mutate_columns_scoped(); auto& columns = columns_guard.mutable_columns(); diff --git a/be/test/olap/rowset/group_rowset_writer_test.cpp b/be/test/olap/rowset/group_rowset_writer_test.cpp index d1bc178bf8a575..21a5cf72458bb2 100644 --- a/be/test/olap/rowset/group_rowset_writer_test.cpp +++ b/be/test/olap/rowset/group_rowset_writer_test.cpp @@ -40,6 +40,7 @@ #include "storage/partial_update_info.h" #include "storage/rowset/rowset_reader.h" #include "storage/rowset/rowset_reader_context.h" +#include "storage/schema.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_manager.h" @@ -123,7 +124,7 @@ class GroupRowsetWriterTest : public testing::Test { } Block create_block(int start_key, int num_rows) const { - Block block = _tablet->tablet_schema()->create_block(); + Block block = _tablet->tablet_schema()->create_storage_block(); { auto columns_guard = block.mutate_columns_scoped(); auto& columns = columns_guard.mutable_columns(); @@ -144,7 +145,7 @@ class GroupRowsetWriterTest : public testing::Test { } Block create_partial_update_block() const { - Block block = _tablet->tablet_schema()->create_block_by_cids({0, 1, 2, 3}); + Block block = _tablet->tablet_schema()->create_storage_block({0, 1, 2, 3}); auto columns_guard = block.mutate_columns_scoped(); auto& columns = columns_guard.mutable_columns(); columns[0]->insert(Field::create_field(1)); @@ -343,21 +344,22 @@ TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { const auto& row_binlog_schema = _row_binlog_tablet->tablet_schema(); ASSERT_EQ(7, row_binlog_schema->num_columns()); - std::vector return_columns {0, 1, 2, 3, 4, 5, 6}; RowsetReaderContext reader_context; reader_context.tablet_schema = row_binlog_schema; reader_context.need_ordered_result = false; - reader_context.return_columns = &return_columns; + // Read schema covers all row-binlog columns in order. + auto read_schema = std::make_shared(row_binlog_schema->columns()); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr rowset_reader; ASSERT_TRUE(row_binlog_rowset->create_reader(&rowset_reader).ok()); ASSERT_TRUE(rowset_reader->init(&reader_context).ok()); - Block output_block = row_binlog_schema->create_block(); + Block output_block = read_schema->create_read_block(); auto status = rowset_reader->next_batch(&output_block); ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(1, output_block.rows()); - ASSERT_EQ(return_columns.size(), output_block.columns()); + ASSERT_EQ(read_schema->num_block_columns(), output_block.columns()); EXPECT_EQ(1, (*output_block.get_by_position(0).column)[0].get()); EXPECT_EQ(1001, (*output_block.get_by_position(1).column)[0].get()); @@ -367,7 +369,7 @@ TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { EXPECT_EQ(1000, (*output_block.get_by_position(5).column)[0].get()); EXPECT_EQ(ROW_BINLOG_APPEND, (*output_block.get_by_position(6).column)[0].get()); - Block eof_block = row_binlog_schema->create_block(); + Block eof_block = read_schema->create_read_block(); status = rowset_reader->next_batch(&eof_block); EXPECT_TRUE(status.is()) << status; } diff --git a/be/test/runtime/descriptor_test.cpp b/be/test/runtime/descriptor_test.cpp index 20c2b1000c8303..7dae34a1517eb7 100644 --- a/be/test/runtime/descriptor_test.cpp +++ b/be/test/runtime/descriptor_test.cpp @@ -19,8 +19,12 @@ #include #include +#include + #include "common/exception.h" +#include "core/data_type/data_type_number.h" #include "runtime/descriptors.h" +#include "testutil/desc_tbl_builder.h" namespace doris { @@ -196,4 +200,18 @@ TEST_F(SlotDescriptorTest, DebugString) { EXPECT_TRUE(debug_str2.find("is_virtual=true") != std::string::npos); } -} // namespace doris \ No newline at end of file +TEST(TupleDescriptorTest, GetColumnId) { + ObjectPool pool; + DescriptorTblBuilder builder(&pool); + builder.declare_tuple() << std::make_shared() + << std::make_shared(); + auto* desc_tbl = builder.build(); + auto* tuple_desc = desc_tbl->get_tuple_descriptor(0); + + ASSERT_EQ(tuple_desc->slots().size(), 2); + EXPECT_EQ(tuple_desc->get_column_id(tuple_desc->slots()[0]->id()), 0); + EXPECT_EQ(tuple_desc->get_column_id(tuple_desc->slots()[1]->id()), 1); + EXPECT_EQ(tuple_desc->get_column_id(100), -1); +} + +} // namespace doris diff --git a/be/test/storage/cloud_file_cache_write_index_only_test.cpp b/be/test/storage/cloud_file_cache_write_index_only_test.cpp index c330b93fee489b..4dfbbf9745f9eb 100644 --- a/be/test/storage/cloud_file_cache_write_index_only_test.cpp +++ b/be/test/storage/cloud_file_cache_write_index_only_test.cpp @@ -296,7 +296,7 @@ class CloudFileCacheWriteIndexOnlyTest : public testing::Test { } Block create_full_block(const TabletSchemaSPtr& tablet_schema, int32_t start_key = 1) { - auto block = tablet_schema->create_block(); + auto block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int32_t i = 0; i < 8; ++i) { int32_t key = start_key + i; @@ -311,7 +311,7 @@ class CloudFileCacheWriteIndexOnlyTest : public testing::Test { Block create_column_block(const TabletSchemaSPtr& tablet_schema, const std::vector& column_ids, int32_t row_count = 8, int32_t start_key = 1) { - auto block = tablet_schema->create_block(column_ids); + auto block = tablet_schema->create_storage_block(column_ids); auto columns = std::move(block).mutate_columns(); for (int32_t i = 0; i < row_count; ++i) { int32_t key = start_key + i; diff --git a/be/test/storage/compaction/collection_statistics_test.cpp b/be/test/storage/compaction/collection_statistics_test.cpp index b78355b316ebdd..e10fa8b60bc49a 100644 --- a/be/test/storage/compaction/collection_statistics_test.cpp +++ b/be/test/storage/compaction/collection_statistics_test.cpp @@ -194,6 +194,7 @@ class MockRowsetReader : public RowsetReader { MockRowsetReader(std::shared_ptr rowset) : _rowset(rowset) {} Status init(RowsetReaderContext* read_context, const RowSetSplits& rs_splits) override { + _read_schema = read_context->read_schema; return Status::OK(); } @@ -223,6 +224,8 @@ class MockRowsetReader : public RowsetReader { RowsetSharedPtr rowset() override { return _rowset; } + const ReadSchema& read_schema() const override { return *_read_schema; } + int64_t filtered_rows() override { return 0; } uint64_t merged_rows() override { return 0; } @@ -239,6 +242,7 @@ class MockRowsetReader : public RowsetReader { private: std::shared_ptr _rowset; + ReadSchemaSPtr _read_schema; }; } // namespace collection_statistics diff --git a/be/test/storage/compaction/ordered_data_compaction_test.cpp b/be/test/storage/compaction/ordered_data_compaction_test.cpp index 9cc991de654758..9fc895017ad6a6 100644 --- a/be/test/storage/compaction/ordered_data_compaction_test.cpp +++ b/be/test/storage/compaction/ordered_data_compaction_test.cpp @@ -304,7 +304,7 @@ class OrderedDataCompactionTest : public ::testing::Test { uint32_t num_rows = 0; for (int i = 0; i < rowset_data.size(); ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rowset_data[i].size(); ++rid) { int32_t c1 = std::get<0>(rowset_data[i][rid]); @@ -451,27 +451,6 @@ class OrderedDataCompactionTest : public ::testing::Test { } } - void block_create(TabletSchemaSPtr tablet_schema, Block* block) { - block->clear(); - size_t num_columns = tablet_schema->num_columns(); - if (num_columns > 0 && tablet_schema->columns().back()->name() == BeConsts::ROW_STORE_COL) { - --num_columns; - } - std::vector schema_column_ids(num_columns); - for (uint32_t cid = 0; cid < num_columns; ++cid) { - schema_column_ids[cid] = cid; - } - Schema schema(tablet_schema->columns(), schema_column_ids); - const auto& column_ids = schema.column_ids(); - for (size_t i = 0; i < schema.num_column_ids(); ++i) { - auto column_desc = schema.column(column_ids[i]); - auto data_type = Schema::get_data_type_ptr(*column_desc); - EXPECT_TRUE(data_type != nullptr); - auto column = data_type->create_column(); - block->insert(ColumnWithTypeAndName(std::move(column), data_type, column_desc->name())); - } - } - private: const std::string kTestDir = "/ut_dir/ordered_compaction_test"; const std::string tmp_dir = "./ut_dir/ordered_compaction_test/tmp"; @@ -515,8 +494,9 @@ TEST_F(OrderedDataCompactionTest, test_01) { RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1}; - reader_context.return_columns = &return_columns; + auto read_schema = + std::make_shared(tablet_schema->columns(), std::vector {0, 1}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -526,7 +506,7 @@ TEST_F(OrderedDataCompactionTest, test_01) { std::vector> output_data; Status s = Status::OK(); do { - block_create(tablet_schema, &output_block); + output_block = read_schema->create_read_block(); s = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); EXPECT_EQ(columns.size(), 2); @@ -582,7 +562,7 @@ TEST_F(OrderedDataCompactionTest, test_index_disk_size) { uint32_t num_rows = 0; for (int j = 0; j < input_data[i].size(); ++j) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < input_data[i][j].size(); ++rid) { int32_t c1 = std::get<0>(input_data[i][j][rid]); diff --git a/be/test/storage/compaction/segcompaction_mow_test.cpp b/be/test/storage/compaction/segcompaction_mow_test.cpp index 28a9fed8021eb2..52b3348768e6f6 100644 --- a/be/test/storage/compaction/segcompaction_mow_test.cpp +++ b/be/test/storage/compaction/segcompaction_mow_test.cpp @@ -36,6 +36,7 @@ #include "storage/rowset/rowset_reader_context.h" #include "storage/rowset/rowset_writer.h" #include "storage/rowset/rowset_writer_context.h" +#include "storage/schema.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" @@ -245,7 +246,8 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { reader_context.reader_type = ReaderType::READER_QUERY; reader_context.need_ordered_result = true; std::vector return_columns = {0, 1, 2}; - reader_context.return_columns = &return_columns; + auto read_schema = std::make_shared(tablet_schema->columns(), return_columns); + reader_context.read_schema = read_schema; reader_context.stats = &_stats; reader_context.delete_bitmap = delete_bitmap; @@ -259,8 +261,7 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { uint32_t num_rows_read = 0; bool eof = false; while (!eof) { - std::shared_ptr output_block = - std::make_shared(tablet_schema->create_block(return_columns)); + auto output_block = std::make_shared(read_schema->create_read_block()); std::vector row_is_same; BlockWithSameBit block_with_same_bit {.block = output_block.get(), .same_bit = row_is_same}; @@ -342,7 +343,7 @@ TEST_P(SegCompactionMoWTest, SegCompactionThenRead) { // k2 := k1 * 10 // k3 := rid for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -448,7 +449,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { uint32_t rows_per_segment = 4096; int segid = 0; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -476,7 +477,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 2; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -504,7 +505,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 1; rows_per_segment = 4096; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -532,7 +533,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 1; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -561,7 +562,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { rows_per_segment = 4096; std::map unique_keys; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { // generate some duplicate rows, segment compaction will merge them @@ -600,7 +601,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 1; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -678,7 +679,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_OoOoO) { uint32_t rows_per_segment = 6400; int segid = 0; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -706,7 +707,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_OoOoO) { num_segments = 1; rows_per_segment = 4096; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -734,7 +735,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_OoOoO) { num_segments = 1; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -762,7 +763,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_OoOoO) { num_segments = 1; rows_per_segment = 4096; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -790,7 +791,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_OoOoO) { num_segments = 1; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + segid; @@ -863,7 +864,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionNotTrigger) { // k2 := k1 * 10 // k3 := rid for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; diff --git a/be/test/storage/compaction/segcompaction_test.cpp b/be/test/storage/compaction/segcompaction_test.cpp index d3b843c050da2e..c3a3ee53239d60 100644 --- a/be/test/storage/compaction/segcompaction_test.cpp +++ b/be/test/storage/compaction/segcompaction_test.cpp @@ -36,6 +36,7 @@ #include "storage/rowset/rowset_reader_context.h" #include "storage/rowset/rowset_writer.h" #include "storage/rowset/rowset_writer_context.h" +#include "storage/schema.h" #include "storage/segment/segment_writer.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_meta.h" @@ -314,7 +315,7 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { // k2 := k1 * 10 // k3 := rid for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -357,7 +358,8 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; std::vector return_columns = {0, 1, 2}; - reader_context.return_columns = &return_columns; + auto read_schema = std::make_shared(tablet_schema->columns(), return_columns); + reader_context.read_schema = read_schema; reader_context.stats = &_stats; // without predicates @@ -368,8 +370,7 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { uint32_t num_rows_read = 0; bool eof = false; while (!eof) { - std::shared_ptr output_block = - std::make_shared(tablet_schema->create_block(return_columns)); + auto output_block = std::make_shared(read_schema->create_read_block()); std::vector row_is_same; BlockWithSameBit block_with_same_bit {.block = output_block.get(), .same_bit = row_is_same}; @@ -435,7 +436,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { int num_segments = 4; uint32_t rows_per_segment = 4096; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -453,7 +454,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 2; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -471,7 +472,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 1; rows_per_segment = 4096; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -489,7 +490,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 1; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -507,7 +508,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 8; rows_per_segment = 4096; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -526,7 +527,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { num_segments = 1; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -589,7 +590,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_OoOoO) { int num_segments = 1; uint32_t rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -607,7 +608,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_OoOoO) { num_segments = 1; rows_per_segment = 4096; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -625,7 +626,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_OoOoO) { num_segments = 1; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -643,7 +644,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_OoOoO) { num_segments = 1; rows_per_segment = 4096; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -661,7 +662,7 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_OoOoO) { num_segments = 1; rows_per_segment = 6400; for (int i = 0; i < num_segments; ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { uint32_t k1 = rid * 100 + i; @@ -717,7 +718,7 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { uint32_t k2 = 0; uint32_t k3 = 0; - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // segment#0 k1 = k2 = 1; @@ -864,7 +865,8 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; std::vector return_columns = {0, 1, 2}; - reader_context.return_columns = &return_columns; + auto read_schema = std::make_shared(tablet_schema->columns(), return_columns); + reader_context.read_schema = read_schema; reader_context.stats = &_stats; reader_context.is_unique = true; @@ -876,8 +878,7 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { uint32_t num_rows_read = 0; bool eof = false; while (!eof) { - std::shared_ptr output_block = - std::make_shared(tablet_schema->create_block(return_columns)); + auto output_block = std::make_shared(read_schema->create_read_block()); std::vector row_is_same; BlockWithSameBit block_with_same_bit {.block = output_block.get(), .same_bit = row_is_same}; @@ -983,7 +984,7 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { uint32_t k2 = 0; uint32_t k3 = 0; - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // segment#0 @@ -1131,7 +1132,8 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; std::vector return_columns = {0, 1, 2}; - reader_context.return_columns = &return_columns; + auto read_schema = std::make_shared(tablet_schema->columns(), return_columns); + reader_context.read_schema = read_schema; reader_context.stats = &_stats; // reader_context.is_unique = true; @@ -1143,8 +1145,7 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { uint32_t num_rows_read = 0; bool eof = false; while (!eof) { - std::shared_ptr output_block = - std::make_shared(tablet_schema->create_block(return_columns)); + auto output_block = std::make_shared(read_schema->create_read_block()); std::vector row_is_same; BlockWithSameBit block_with_same_bit {.block = output_block.get(), .same_bit = row_is_same}; diff --git a/be/test/storage/compaction/vertical_compaction_test.cpp b/be/test/storage/compaction/vertical_compaction_test.cpp index 42c6b5a0dc7316..4974c7bddc8187 100644 --- a/be/test/storage/compaction/vertical_compaction_test.cpp +++ b/be/test/storage/compaction/vertical_compaction_test.cpp @@ -253,7 +253,7 @@ class VerticalCompactionTest : public ::testing::Test { uint32_t num_rows = 0; for (int i = 0; i < rowset_data.size(); ++i) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rowset_data[i].size(); ++rid) { int32_t c1 = std::get<0>(rowset_data[i][rid]); @@ -371,27 +371,6 @@ class VerticalCompactionTest : public ::testing::Test { } } - void block_create(TabletSchemaSPtr tablet_schema, Block* block) { - block->clear(); - size_t num_columns = tablet_schema->num_columns(); - if (num_columns > 0 && tablet_schema->columns().back()->name() == BeConsts::ROW_STORE_COL) { - --num_columns; - } - std::vector schema_column_ids(num_columns); - for (uint32_t cid = 0; cid < num_columns; ++cid) { - schema_column_ids[cid] = cid; - } - Schema schema(tablet_schema->columns(), schema_column_ids); - const auto& column_ids = schema.column_ids(); - for (size_t i = 0; i < schema.num_column_ids(); ++i) { - auto column_desc = schema.column(column_ids[i]); - auto data_type = Schema::get_data_type_ptr(*column_desc); - EXPECT_TRUE(data_type != nullptr); - auto column = data_type->create_column(); - block->insert(ColumnWithTypeAndName(std::move(column), data_type, column_desc->name())); - } - } - private: const std::string kTestDir = "/ut_dir/vertical_compaction_test"; std::string absolute_dir; @@ -606,8 +585,9 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMerge) { RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1}; - reader_context.return_columns = &return_columns; + auto read_schema = + std::make_shared(tablet_schema->columns(), std::vector {0, 1}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -616,7 +596,7 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMerge) { Block output_block; std::vector> output_data; do { - block_create(tablet_schema, &output_block); + output_block = read_schema->create_read_block(); s = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); EXPECT_EQ(columns.size(), 2); @@ -711,8 +691,9 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMerge) { RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1}; - reader_context.return_columns = &return_columns; + auto read_schema = + std::make_shared(tablet_schema->columns(), std::vector {0, 1}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -721,7 +702,7 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMerge) { Block output_block; std::vector> output_data; do { - block_create(tablet_schema, &output_block); + output_block = read_schema->create_read_block(); s = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); EXPECT_EQ(columns.size(), 2); @@ -817,8 +798,9 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMerge) { RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1}; - reader_context.return_columns = &return_columns; + auto read_schema = + std::make_shared(tablet_schema->columns(), std::vector {0, 1}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -827,7 +809,7 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMerge) { Block output_block; std::vector> output_data; do { - block_create(tablet_schema, &output_block); + output_block = read_schema->create_read_block(); s = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); EXPECT_EQ(columns.size(), 2); @@ -1015,7 +997,7 @@ TEST_F(VerticalCompactionTest, TestUniqueKeySegmentContextMemoryAmplification) { auto rowset_writer = std::move(res).value(); for (uint32_t segment_id = 0; segment_id < num_segments_per_rowset; ++segment_id) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (uint32_t row_id = 0; row_id < rows_per_segment; ++row_id) { int32_t logical_row = segment_id * rows_per_segment + row_id; @@ -1077,15 +1059,16 @@ TEST_F(VerticalCompactionTest, TestUniqueKeySegmentContextMemoryAmplification) { RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1, 2}; - reader_context.return_columns = &return_columns; + auto read_schema = std::make_shared(tablet_schema->columns(), + std::vector {0, 1, 2}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_rs_reader); int64_t expected_key = 0; Status read_status; do { - Block output_block = tablet_schema->create_block(); + Block output_block = tablet_schema->create_storage_block(); read_status = output_rs_reader->next_batch(&output_block); const auto& output_columns = output_block.get_columns_with_type_and_name(); ASSERT_EQ(3, output_columns.size()); @@ -1195,8 +1178,9 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMergeWithDelete) { RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1}; - reader_context.return_columns = &return_columns; + auto read_schema = + std::make_shared(tablet_schema->columns(), std::vector {0, 1}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1205,7 +1189,7 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMergeWithDelete) { Block output_block; std::vector> output_data; do { - block_create(tablet_schema, &output_block); + output_block = read_schema->create_read_block(); st = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); EXPECT_EQ(columns.size(), 2); @@ -1296,8 +1280,9 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMergeWithDelete) { RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1}; - reader_context.return_columns = &return_columns; + auto read_schema = + std::make_shared(tablet_schema->columns(), std::vector {0, 1}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1306,7 +1291,7 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMergeWithDelete) { Block output_block; std::vector> output_data; do { - block_create(tablet_schema, &output_block); + output_block = read_schema->create_read_block(); st = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); EXPECT_EQ(columns.size(), 2); @@ -1388,8 +1373,9 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge) { RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1}; - reader_context.return_columns = &return_columns; + auto read_schema = + std::make_shared(tablet_schema->columns(), std::vector {0, 1}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1398,7 +1384,7 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge) { Block output_block; std::vector> output_data; do { - block_create(tablet_schema, &output_block); + output_block = read_schema->create_read_block(); s = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); EXPECT_EQ(columns.size(), 2); @@ -1517,7 +1503,8 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMergeWithNullableSparseColum ASSERT_TRUE(res.has_value()) << res.error(); auto rowset_writer = std::move(res).value(); - Block block = tablet_schema->create_block(); + // Create block with nullable v1 column. + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rows_per_segment; ++rid) { @@ -1581,15 +1568,16 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMergeWithNullableSparseColum RowsetReaderContext reader_context; reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; - std::vector return_columns = {0, 1, 2}; - reader_context.return_columns = &return_columns; + auto read_schema = + std::make_shared(tablet_schema->columns(), std::vector {0, 1, 2}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); Block output_block; size_t output_rows = 0; do { - block_create(tablet_schema, &output_block); + output_block = read_schema->create_read_block(); s = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); ASSERT_EQ(columns.size(), 3); @@ -1723,7 +1711,7 @@ TEST_F(VerticalCompactionTest, TestFooterRawDataBytesAccuracy) { ASSERT_TRUE(res.has_value()) << res.error(); auto rowset_writer = std::move(res).value(); - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int i = 0; i < kNumRows; i++) { int32_t int_val = i; @@ -1819,7 +1807,7 @@ TEST_F(VerticalCompactionTest, TestFooterRawDataBytesNullableSparse) { ASSERT_TRUE(res.has_value()) << res.error(); auto rowset_writer = std::move(res).value(); - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int i = 0; i < kNumRows; i++) { int32_t key_val = i; diff --git a/be/test/storage/delete/delete_handler_test.cpp b/be/test/storage/delete/delete_handler_test.cpp index f929b94b8e1a58..0e78a4c6d29f37 100644 --- a/be/test/storage/delete/delete_handler_test.cpp +++ b/be/test/storage/delete/delete_handler_test.cpp @@ -32,9 +32,11 @@ #include #include #include +#include #include #include "common/config.h" +#include "core/block/block.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" #include "json2pb/json_to_pb.h" @@ -43,9 +45,11 @@ #include "storage/olap_define.h" #include "storage/olap_tuple.h" #include "storage/options.h" +#include "storage/predicate/block_column_predicate.h" #include "storage/row_cursor.h" #include "storage/rowset/beta_rowset.h" #include "storage/rowset/rowset.h" +#include "storage/schema.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_manager.h" @@ -1053,6 +1057,46 @@ class TestDeleteHandler : public testing::Test { return delete_preds; } + ReadSchemaSPtr create_read_schema() { + return std::make_shared(tablet->tablet_schema()->columns()); + } + + Status init_delete_handler(int64_t version) { + auto read_schema = create_read_schema(); + std::vector dropped_columns; + return _delete_handler.init(get_delete_predicates(), version, read_schema, dropped_columns); + } + + TabletSchemaSPtr create_single_column_schema(int32_t unique_id, const std::string& name, + const std::string& type) { + TabletSchemaPB schema_pb; + auto* column = schema_pb.add_column(); + column->set_unique_id(unique_id); + column->set_name(name); + column->set_type(type); + column->set_is_nullable(false); + if (type == "INT") { + column->set_length(4); + column->set_index_length(4); + } else { + column->set_length(65535); + column->set_index_length(0); + } + + auto schema = std::make_shared(); + schema->init_from_pb(schema_pb); + return schema; + } + + RowsetMetaSharedPtr create_delete_predicate_meta(const DeletePredicatePB& delete_predicate, + const TabletSchemaSPtr& historical_schema) { + auto rowset_meta = std::make_shared(); + init_rs_meta(rowset_meta, 2, 2); + rowset_meta->set_delete_predicate(delete_predicate); + rowset_meta->set_tablet_schema(historical_schema); + return rowset_meta; + } + std::string _tablet_path; TabletSharedPtr tablet; TCreateTabletReq _create_tablet; @@ -1060,6 +1104,91 @@ class TestDeleteHandler : public testing::Test { std::string _json_rowset_meta; }; +TEST_F(TestDeleteHandler, ReturnsHistoricalInPredicateColumnByUid) { + constexpr int32_t old_unique_id = 100; + constexpr int32_t new_unique_id = 200; + constexpr auto column_name = "dropped_column"; + + auto historical_schema = create_single_column_schema(old_unique_id, column_name, "INT"); + auto current_schema = create_single_column_schema(new_unique_id, column_name, "STRING"); + ReadSchemaSPtr read_schema = std::make_shared(current_schema->columns()); + + DeletePredicatePB delete_predicate; + delete_predicate.set_version(-1); + auto* in_predicate = delete_predicate.add_in_predicates(); + in_predicate->set_column_name(column_name); + in_predicate->set_column_unique_id(old_unique_id); + in_predicate->add_values("1"); + in_predicate->add_values("2"); + + auto rowset_meta = create_delete_predicate_meta(delete_predicate, historical_schema); + std::vector dropped_columns; + auto status = _delete_handler.init({rowset_meta}, 2, read_schema, dropped_columns); + ASSERT_TRUE(status.ok()) << status; + + ASSERT_EQ(1, read_schema->num_block_columns()); + ASSERT_EQ(1, read_schema->num_read_columns()); + EXPECT_EQ(0, read_schema->ordinal_by_uid(new_unique_id)); + EXPECT_EQ(-1, read_schema->ordinal_by_uid(old_unique_id)); + ASSERT_EQ(1, dropped_columns.size()); + EXPECT_EQ(old_unique_id, dropped_columns[0].unique_id()); + EXPECT_EQ(FieldType::OLAP_FIELD_TYPE_INT, dropped_columns[0].type()); + EXPECT_EQ(1, read_schema->create_read_block().columns()); + + AndBlockColumnPredicate delete_conditions; + std::unordered_map>> + predicates_for_zone_map; + _delete_handler.get_delete_conditions_after_version(0, &delete_conditions, + &predicates_for_zone_map); + ASSERT_EQ(1, predicates_for_zone_map.size()); + ASSERT_TRUE(predicates_for_zone_map.contains(1)); + ASSERT_EQ(1, predicates_for_zone_map.at(1).size()); + const auto& predicate = predicates_for_zone_map.at(1)[0]; + EXPECT_EQ(1, predicate->column_id()); + EXPECT_EQ(PredicateType::IN_LIST, predicate->type()); +} + +TEST_F(TestDeleteHandler, ReturnsHistoricalLegacyPredicateColumnAfterSecondDrop) { + constexpr int32_t old_unique_id = 101; + constexpr int32_t retained_unique_id = 201; + constexpr auto column_name = "dropped_column"; + + auto historical_schema = create_single_column_schema(old_unique_id, column_name, "INT"); + auto current_schema = + create_single_column_schema(retained_unique_id, "retained_column", "STRING"); + ReadSchemaSPtr read_schema = std::make_shared(current_schema->columns()); + + DeletePredicatePB delete_predicate; + // Doris 2.0 stored delete conditions in sub_predicates without a column UID. + delete_predicate.set_version(-1); + delete_predicate.add_sub_predicates(fmt::format("{}='7'", column_name)); + auto rowset_meta = create_delete_predicate_meta(delete_predicate, historical_schema); + std::vector dropped_columns; + auto status = _delete_handler.init({rowset_meta}, 2, read_schema, dropped_columns); + ASSERT_TRUE(status.ok()) << status; + + ASSERT_EQ(1, read_schema->num_block_columns()); + ASSERT_EQ(1, read_schema->num_read_columns()); + EXPECT_EQ(0, read_schema->ordinal_by_uid(retained_unique_id)); + EXPECT_EQ(-1, read_schema->ordinal_by_uid(old_unique_id)); + ASSERT_EQ(1, dropped_columns.size()); + EXPECT_EQ(old_unique_id, dropped_columns[0].unique_id()); + EXPECT_EQ(FieldType::OLAP_FIELD_TYPE_INT, dropped_columns[0].type()); + EXPECT_EQ(1, read_schema->create_read_block().columns()); + + AndBlockColumnPredicate delete_conditions; + std::unordered_map>> + predicates_for_zone_map; + _delete_handler.get_delete_conditions_after_version(0, &delete_conditions, + &predicates_for_zone_map); + ASSERT_EQ(1, predicates_for_zone_map.size()); + ASSERT_TRUE(predicates_for_zone_map.contains(1)); + ASSERT_EQ(1, predicates_for_zone_map.at(1).size()); + const auto& predicate = predicates_for_zone_map.at(1)[0]; + EXPECT_EQ(1, predicate->column_id()); + EXPECT_EQ(PredicateType::EQ, predicate->type()); +} + TEST_F(TestDeleteHandler, ValueWithQuote) { DeletePredicatePB del_predicate; del_predicate.add_sub_predicates("k1='b'"); @@ -1072,7 +1201,7 @@ TEST_F(TestDeleteHandler, ValueWithQuote) { add_delete_predicate(del_predicate, 2); - EXPECT_FALSE(_delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5).ok()); + EXPECT_FALSE(init_delete_handler(5).ok()); } TEST_F(TestDeleteHandler, timestamptz_ValueWithQuote) { @@ -1082,8 +1211,7 @@ TEST_F(TestDeleteHandler, timestamptz_ValueWithQuote) { del_predicate.set_version(2); add_delete_predicate(del_predicate, 2); - EXPECT_FALSE( - _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5).ok()); + EXPECT_FALSE(init_delete_handler(5).ok()); } { DeletePredicatePB del_predicate; @@ -1091,8 +1219,7 @@ TEST_F(TestDeleteHandler, timestamptz_ValueWithQuote) { del_predicate.set_version(2); add_delete_predicate(del_predicate, 2); - EXPECT_FALSE( - _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5).ok()); + EXPECT_FALSE(init_delete_handler(5).ok()); } { DeletePredicatePB del_predicate; @@ -1100,8 +1227,7 @@ TEST_F(TestDeleteHandler, timestamptz_ValueWithQuote) { del_predicate.set_version(2); add_delete_predicate(del_predicate, 2); - EXPECT_FALSE( - _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5).ok()); + EXPECT_FALSE(init_delete_handler(5).ok()); } } @@ -1112,8 +1238,7 @@ TEST_F(TestDeleteHandler, timestamptz_ValueWithoutQuote) { del_predicate.set_version(2); add_delete_predicate(del_predicate, 2); - EXPECT_FALSE( - _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5).ok()); + EXPECT_FALSE(init_delete_handler(5).ok()); } { DeletePredicatePB del_predicate; @@ -1121,8 +1246,7 @@ TEST_F(TestDeleteHandler, timestamptz_ValueWithoutQuote) { del_predicate.set_version(2); add_delete_predicate(del_predicate, 2); - EXPECT_FALSE( - _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5).ok()); + EXPECT_FALSE(init_delete_handler(5).ok()); } } @@ -1259,7 +1383,7 @@ TEST_F(TestDeleteHandler, timestamptz) { add_delete_predicate(del_pred, 2); - auto res = _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5); + auto res = init_delete_handler(5); EXPECT_EQ(Status::OK(), res); } @@ -1271,7 +1395,7 @@ TEST_F(TestDeleteHandler, ValueWithoutQuote) { add_delete_predicate(del_predicate, 2); - EXPECT_FALSE(_delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5).ok()); + EXPECT_FALSE(init_delete_handler(5).ok()); } TEST_F(TestDeleteHandler, InitSuccess) { @@ -1343,7 +1467,7 @@ TEST_F(TestDeleteHandler, InitSuccess) { add_delete_predicate(del_pred_4, 5); // Get delete conditions which version <= 5 - res = _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 5); + res = init_delete_handler(5); EXPECT_EQ(Status::OK(), res); } @@ -1374,7 +1498,7 @@ TEST_F(TestDeleteHandler, FilterDataSubconditions) { add_delete_predicate(del_pred, 2); // 指定版本号为10以载入Header中的所有过滤条件(在这个case中，只有过滤条件1) - res = _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 4); + res = init_delete_handler(4); EXPECT_EQ(Status::OK(), res); } @@ -1433,7 +1557,7 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { add_delete_predicate(del_pred_3, 4); // 指定版本号为4以载入meta中的所有过滤条件(在这个case中，只有过滤条件1) - res = _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 4); + res = init_delete_handler(4); EXPECT_EQ(Status::OK(), res); } @@ -1503,7 +1627,7 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { add_delete_predicate(del_pred_4, 6); // 指定版本号为6以载入meta中的所有过滤条件(过滤条件1，过滤条件2) - res = _delete_handler.init(tablet->tablet_schema(), get_delete_predicates(), 6); + res = init_delete_handler(6); EXPECT_EQ(Status::OK(), res); } diff --git a/be/test/storage/index/ann/ann_range_search_test.cpp b/be/test/storage/index/ann/ann_range_search_test.cpp index 0ba7b122df69b9..b512780c9c85ed 100644 --- a/be/test/storage/index/ann/ann_range_search_test.cpp +++ b/be/test/storage/index/ann/ann_range_search_test.cpp @@ -57,13 +57,12 @@ std::string thrift_table_desc = R"xxx({"1":{"lst":["rec",7,{"1":{"i32":0},"2":{"i32":0},"3":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":-1},"5":{"i32":-1},"6":{"i32":0},"7":{"i32":-1},"8":{"str":"id"},"9":{"i32":0},"10":{"tf":1},"11":{"i32":0},"12":{"tf":1},"13":{"tf":1},"14":{"tf":0},"17":{"i32":5}},{"1":{"i32":1},"2":{"i32":0},"3":{"rec":{"1":{"lst":["rec",2,{"1":{"i32":1},"4":{"tf":1},"5":{"lst":["tf",1,1]}},{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":-1},"5":{"i32":-1},"6":{"i32":0},"7":{"i32":-1},"8":{"str":"embedding"},"9":{"i32":3},"10":{"tf":1},"11":{"i32":1},"12":{"tf":0},"13":{"tf":1},"14":{"tf":0},"17":{"i32":20}},{"1":{"i32":3},"2":{"i32":0},"3":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":-1},"5":{"i32":-1},"6":{"i32":0},"7":{"i32":0},"8":{"str":"value"},"9":{"i32":1},"10":{"tf":1},"11":{"i32":3},"12":{"tf":0},"13":{"tf":1},"14":{"tf":0},"17":{"i32":5}},{"1":{"i32":4},"2":{"i32":0},"3":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":-1},"5":{"i32":-1},"6":{"i32":0},"7":{"i32":-1},"8":{"str":"__DORIS_VIRTUAL_COL__1"},"9":{"i32":2},"10":{"tf":1},"11":{"i32":2147483646},"12":{"tf":0},"13":{"tf":1},"14":{"tf":0},"17":{"i32":7},"18":{"rec":{"1":{"lst":["rec",11,{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"l2_distance_approximate"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",2,{"1":{"i32":1},"4":{"tf":1},"5":{"lst":["tf",1,1]}},{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",2,{"1":{"i32":1},"4":{"tf":1},"5":{"lst":["tf",1,1]}},{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"l2_distance_approximate(array, array)"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":0}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",2,{"1":{"i32":1},"4":{"tf":1},"5":{"lst":["tf",1,1]}},{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":1},"2":{"i32":0},"3":{"i32":1},"4":{"tf":0}}},"20":{"i32":-1},"29":{"tf":0},"36":{"str":"embedding"}},{"1":{"i32":21},"2":{"rec":{"1":{"lst":["rec",2,{"1":{"i32":1},"4":{"tf":1},"5":{"lst":["tf",1,1]}},{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":8},"20":{"i32":-1},"28":{"i32":7},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":1}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":2}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":3}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":4}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":5}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":6}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":7}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":20}}},"20":{"i32":-1},"29":{"tf":0}}]}}}},{"1":{"i32":5},"2":{"i32":1},"3":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":-1},"5":{"i32":-1},"6":{"i32":0},"7":{"i32":-1},"8":{"str":"id"},"9":{"i32":0},"10":{"tf":1},"11":{"i32":0},"12":{"tf":1},"13":{"tf":1},"14":{"tf":0},"17":{"i32":5}},{"1":{"i32":6},"2":{"i32":1},"3":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":-1},"5":{"i32":-1},"6":{"i32":0},"7":{"i32":0},"8":{"str":"value"},"9":{"i32":1},"10":{"tf":1},"11":{"i32":3},"12":{"tf":0},"13":{"tf":1},"14":{"tf":0},"17":{"i32":5}},{"1":{"i32":7},"2":{"i32":1},"3":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":7},"5":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":-1},"5":{"i32":-1},"6":{"i32":0},"7":{"i32":-1},"8":{"str":""},"9":{"i32":2},"10":{"tf":1},"11":{"i32":-1},"12":{"tf":0},"13":{"tf":1},"14":{"tf":0},"17":{"i32":0}}]},"2":{"lst":["rec",2,{"1":{"i32":0},"2":{"i32":0},"3":{"i32":0},"4":{"i64":-794312748}},{"1":{"i32":1},"2":{"i32":0},"3":{"i32":0},"4":{"i64":-794312748}}]},"3":{"lst":["rec",1,{"1":{"i64":1755847311316},"2":{"i32":1},"3":{"i32":4},"4":{"i32":0},"7":{"str":"ann_with_fulltext"},"8":{"str":""},"11":{"rec":{"1":{"str":"ann_with_fulltext"}}}}]}})xxx"; static std::shared_ptr create_index_context( - const std::vector& col_ids, const std::vector>& index_iterators, std::vector& storage_name_and_type, std::unordered_map>& common_expr_index_status) { segment_v2::ColumnIteratorOptions column_iter_opts; - return std::make_shared(col_ids, index_iterators, storage_name_and_type, + return std::make_shared(index_iterators, storage_name_and_type, common_expr_index_status, nullptr, nullptr, column_iter_opts); } @@ -133,16 +132,9 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearch) { ASSERT_EQ(range_search_ctx->_ann_range_search_runtime.dst_col_idx, 3); ASSERT_EQ(range_search_ctx->_ann_range_search_runtime.radius, 10); - std::vector idx_to_cid; - idx_to_cid.resize(4); - idx_to_cid[0] = 0; - idx_to_cid[1] = 1; - idx_to_cid[2] = 2; - idx_to_cid[3] = 3; - std::vector> cid_to_index_iterators; - cid_to_index_iterators.resize(4); - cid_to_index_iterators[1] = - std::make_unique(); + std::vector> index_iterators; + index_iterators.resize(4); + index_iterators[1] = std::make_unique(); std::vector> column_iterators; column_iterators.resize(4); column_iterators[3] = std::make_unique(); @@ -150,7 +142,7 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearch) { roaring::Roaring row_bitmap; doris::vector_search_utils::MockAnnIndexIterator* mock_ann_index_iter = dynamic_cast( - cid_to_index_iterators[1].get()); + index_iterators[1].get()); std::map properties; properties["index_type"] = "hnsw"; @@ -183,12 +175,12 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearch) { common_expr_to_slotref_map; std::vector storage_name_and_type(4); std::unordered_map> common_expr_index_status; - range_search_ctx->set_index_context(create_index_context( - idx_to_cid, cid_to_index_iterators, storage_name_and_type, common_expr_index_status)); + range_search_ctx->set_index_context( + create_index_context(index_iterators, storage_name_and_type, common_expr_index_status)); bool ann_range_search_executed = false; ASSERT_TRUE(range_search_ctx - ->evaluate_ann_range_search(cid_to_index_iterators, idx_to_cid, - column_iterators, common_expr_to_slotref_map, + ->evaluate_ann_range_search(index_iterators, column_iterators, + common_expr_to_slotref_map, row_bitmap.cardinality(), row_bitmap, stats, false, &ann_range_search_executed) .ok()); @@ -234,23 +226,16 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearch2) { ASSERT_EQ(range_search_ctx->_ann_range_search_runtime.dst_col_idx, 3); ASSERT_EQ(range_search_ctx->_ann_range_search_runtime.radius, 10); - std::vector idx_to_cid; - idx_to_cid.resize(4); - idx_to_cid[0] = 0; - idx_to_cid[1] = 1; - idx_to_cid[2] = 2; - idx_to_cid[3] = 3; - std::vector> cid_to_index_iterators; - cid_to_index_iterators.resize(4); - cid_to_index_iterators[1] = - std::make_unique(); + std::vector> index_iterators; + index_iterators.resize(4); + index_iterators[1] = std::make_unique(); std::vector> column_iterators; column_iterators.resize(4); column_iterators[3] = std::make_unique(); roaring::Roaring row_bitmap; doris::vector_search_utils::MockAnnIndexIterator* mock_ann_index_iter = dynamic_cast( - cid_to_index_iterators[1].get()); + index_iterators[1].get()); std::map properties; properties["index_type"] = "hnsw"; properties["metric_type"] = "l2_distance"; @@ -287,12 +272,12 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearch2) { common_expr_to_slotref_map; std::vector storage_name_and_type(4); std::unordered_map> common_expr_index_status; - range_search_ctx->set_index_context(create_index_context( - idx_to_cid, cid_to_index_iterators, storage_name_and_type, common_expr_index_status)); + range_search_ctx->set_index_context( + create_index_context(index_iterators, storage_name_and_type, common_expr_index_status)); bool ann_range_search_executed = false; ASSERT_TRUE(range_search_ctx - ->evaluate_ann_range_search(cid_to_index_iterators, idx_to_cid, - column_iterators, common_expr_to_slotref_map, + ->evaluate_ann_range_search(index_iterators, column_iterators, + common_expr_to_slotref_map, row_bitmap.cardinality(), row_bitmap, stats, false, &ann_range_search_executed) .ok()); @@ -337,7 +322,6 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearchStateDoesNotLeakAcrossClones) ASSERT_TRUE(range_search_ctx->clone(state.get(), segment_without_ann_ctx).ok()); ASSERT_EQ(segment_with_ann_ctx->root().get(), segment_without_ann_ctx->root().get()); - std::vector idx_to_cid = {0, 1, 2, 3}; std::vector> ann_index_iterators(4); ann_index_iterators[1] = std::make_unique(); auto* mock_ann_index_iter = dynamic_cast( @@ -353,8 +337,8 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearchStateDoesNotLeakAcrossClones) ann_column_iterators[3] = std::make_unique(); std::vector ann_storage_name_and_type(4); std::unordered_map> ann_index_status; - segment_with_ann_ctx->set_index_context(create_index_context( - idx_to_cid, ann_index_iterators, ann_storage_name_and_type, ann_index_status)); + segment_with_ann_ctx->set_index_context( + create_index_context(ann_index_iterators, ann_storage_name_and_type, ann_index_status)); EXPECT_CALL(*mock_ann_index_iter, range_search(testing::_, testing::_, testing::_, testing::_)) .WillOnce(testing::Invoke([](const doris::segment_v2::AnnRangeSearchParams& params, @@ -375,10 +359,10 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearchStateDoesNotLeakAcrossClones) common_expr_to_slotref_map; bool ann_range_search_executed = false; ASSERT_TRUE(segment_with_ann_ctx - ->evaluate_ann_range_search( - ann_index_iterators, idx_to_cid, ann_column_iterators, - common_expr_to_slotref_map, ann_row_bitmap.cardinality(), - ann_row_bitmap, ann_stats, false, &ann_range_search_executed) + ->evaluate_ann_range_search(ann_index_iterators, ann_column_iterators, + common_expr_to_slotref_map, + ann_row_bitmap.cardinality(), ann_row_bitmap, + ann_stats, false, &ann_range_search_executed) .ok()); EXPECT_TRUE(ann_range_search_executed); const auto* ann_result = segment_with_ann_ctx->get_index_context()->get_index_result_for_expr( @@ -393,18 +377,18 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearchStateDoesNotLeakAcrossClones) std::vector no_ann_storage_name_and_type(4); std::unordered_map> no_ann_index_status; segment_without_ann_ctx->set_index_context(create_index_context( - idx_to_cid, no_ann_index_iterators, no_ann_storage_name_and_type, no_ann_index_status)); + no_ann_index_iterators, no_ann_storage_name_and_type, no_ann_index_status)); roaring::Roaring no_ann_row_bitmap; segment_v2::AnnIndexStats no_ann_stats; bool no_ann_range_search_executed = true; - ASSERT_TRUE( - segment_without_ann_ctx - ->evaluate_ann_range_search(no_ann_index_iterators, idx_to_cid, - no_ann_column_iterators, common_expr_to_slotref_map, - no_ann_row_bitmap.cardinality(), no_ann_row_bitmap, - no_ann_stats, false, &no_ann_range_search_executed) - .ok()); + ASSERT_TRUE(segment_without_ann_ctx + ->evaluate_ann_range_search(no_ann_index_iterators, no_ann_column_iterators, + common_expr_to_slotref_map, + no_ann_row_bitmap.cardinality(), + no_ann_row_bitmap, no_ann_stats, false, + &no_ann_range_search_executed) + .ok()); EXPECT_FALSE(no_ann_range_search_executed); EXPECT_FALSE(segment_without_ann_ctx->get_index_context()->has_index_result_for_expr( segment_without_ann_ctx->root().get())); @@ -433,12 +417,10 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearchUsesSourceColumnIndexForSlotM ASSERT_EQ(range_search_ctx->_ann_range_search_runtime.src_col_idx, 1); ASSERT_EQ(range_search_ctx->_ann_range_search_runtime.dst_col_idx, 3); - std::vector idx_to_cid = {0, 5, 6, 7}; - std::vector> cid_to_index_iterators(8); - cid_to_index_iterators[5] = - std::make_unique(); + std::vector> index_iterators(4); + index_iterators[1] = std::make_unique(); auto* mock_ann_index_iter = dynamic_cast( - cid_to_index_iterators[5].get()); + index_iterators[1].get()); std::map properties; properties["index_type"] = "hnsw"; properties["metric_type"] = "l2_distance"; @@ -446,13 +428,13 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearchUsesSourceColumnIndexForSlotM auto pair = vector_search_utils::create_tmp_ann_index_reader(properties); mock_ann_index_iter->_ann_reader = pair.second; - std::vector> column_iterators(8); - column_iterators[7] = std::make_unique(); - std::vector storage_name_and_type(8); + std::vector> column_iterators(4); + column_iterators[3] = std::make_unique(); + std::vector storage_name_and_type(4); std::unordered_map> common_expr_index_status; - common_expr_index_status[5][range_search_ctx->root().get()] = false; - range_search_ctx->set_index_context(create_index_context( - idx_to_cid, cid_to_index_iterators, storage_name_and_type, common_expr_index_status)); + common_expr_index_status[1][range_search_ctx->root().get()] = false; + range_search_ctx->set_index_context( + create_index_context(index_iterators, storage_name_and_type, common_expr_index_status)); std::unordered_map> common_expr_to_slotref_map; @@ -479,13 +461,13 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearchUsesSourceColumnIndexForSlotM segment_v2::AnnIndexStats stats; bool ann_range_search_executed = false; ASSERT_TRUE(range_search_ctx - ->evaluate_ann_range_search(cid_to_index_iterators, idx_to_cid, - column_iterators, common_expr_to_slotref_map, + ->evaluate_ann_range_search(index_iterators, column_iterators, + common_expr_to_slotref_map, row_bitmap.cardinality(), row_bitmap, stats, false, &ann_range_search_executed) .ok()); EXPECT_TRUE(ann_range_search_executed); - EXPECT_TRUE(common_expr_index_status[5][range_search_ctx->root().get()]); + EXPECT_TRUE(common_expr_index_status[1][range_search_ctx->root().get()]); const auto* result = range_search_ctx->get_index_context()->get_index_result_for_expr( range_search_ctx->root().get()); ASSERT_NE(result, nullptr); @@ -845,13 +827,7 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearch_DimensionMismatch) { range_search_ctx->prepare_ann_range_search(user_params); ASSERT_TRUE(range_search_ctx->_ann_range_search_runtime.is_ann_range_search); // Force a dimension mismatch: query dim is 8 in thrift; set index dim to 4 - std::vector idx_to_cid(4); - idx_to_cid[0] = 0; - idx_to_cid[1] = 1; // embedding - idx_to_cid[2] = 2; - idx_to_cid[3] = 3; // virtual dist - - std::vector> cid_to_index_iterators(4); + std::vector> index_iterators(4); auto mock_iter = std::make_unique(); // Back its reader with a real AnnIndexReader but with dim=4 @@ -861,7 +837,7 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearch_DimensionMismatch) { properties["dim"] = "4"; // mismatch auto pair = vector_search_utils::create_tmp_ann_index_reader(properties); mock_iter->_ann_reader = pair.second; - cid_to_index_iterators[1] = std::move(mock_iter); + index_iterators[1] = std::move(mock_iter); std::vector> column_iterators(4); column_iterators[3] = std::make_unique(); @@ -872,8 +848,8 @@ TEST_F(VectorSearchTest, TestEvaluateAnnRangeSearch_DimensionMismatch) { common_expr_to_slotref_map; auto st = range_search_ctx->evaluate_ann_range_search( - cid_to_index_iterators, idx_to_cid, column_iterators, common_expr_to_slotref_map, - row_bitmap.cardinality(), row_bitmap, stats, false, nullptr); + index_iterators, column_iterators, common_expr_to_slotref_map, row_bitmap.cardinality(), + row_bitmap, stats, false, nullptr); EXPECT_FALSE(st.ok()); EXPECT_TRUE(st.is()); } diff --git a/be/test/storage/index/date_bloom_filter_test.cpp b/be/test/storage/index/date_bloom_filter_test.cpp index 263f2a44f0177d..742c1632f056d0 100644 --- a/be/test/storage/index/date_bloom_filter_test.cpp +++ b/be/test/storage/index/date_bloom_filter_test.cpp @@ -130,7 +130,7 @@ TEST_F(DateBloomFilterTest, query_index_test) { EXPECT_TRUE(res.has_value()) << res.error(); const auto& rowset_writer = res.value(); - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); auto date = timestamp_from_date("2024-11-08"); @@ -224,7 +224,7 @@ TEST_F(DateBloomFilterTest, in_list_predicate_test) { EXPECT_TRUE(res.has_value()) << res.error(); const auto& rowset_writer = res.value(); - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Insert test data diff --git a/be/test/storage/index/index_builder_test.cpp b/be/test/storage/index/index_builder_test.cpp index 4d5d890c887bb5..7435f48030757e 100644 --- a/be/test/storage/index/index_builder_test.cpp +++ b/be/test/storage/index/index_builder_test.cpp @@ -244,7 +244,7 @@ TEST_F(IndexBuilderTest, DropInvertedIndexTest) { // 5. Write data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -534,7 +534,7 @@ TEST_F(IndexBuilderTest, BuildInvertedIndexAfterWritingDataTest) { // 4. Write data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns according to the schema @@ -865,7 +865,7 @@ TEST_F(IndexBuilderTest, AddIndexWhenOneExistsTest) { // 5. Write data to rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -1035,7 +1035,7 @@ TEST_F(IndexBuilderTest, AddIndexWhenOneExistsTestV1) { // 8. Write data to rowset { - Block block = v1_schema->create_block(); + Block block = v1_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -1186,7 +1186,7 @@ TEST_F(IndexBuilderTest, MultiSegmentBuildIndexTest) { // 4. Write data to the rowset in multiple batches to ensure we get multiple segments for (int segment = 0; segment < num_segments; segment++) { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -1338,7 +1338,7 @@ TEST_F(IndexBuilderTest, NonExistentColumnIndexTest) { // 4. Write data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -1514,7 +1514,7 @@ TEST_F(IndexBuilderTest, RenameColumnIndexTest) { // 5. Write data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -1670,7 +1670,7 @@ TEST_F(IndexBuilderTest, AddNonExistentColumnIndexWhenOneExistsTest) { // 5. Write data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -1844,7 +1844,7 @@ TEST_F(IndexBuilderTest, AddNonExistentColumnIndexWhenOneExistsTestV1) { // 9. Write data to rowset { - Block block = v1_schema->create_block(); + Block block = v1_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -1996,7 +1996,7 @@ TEST_F(IndexBuilderTest, NonNullIndexDataTest) { // 4. Write non-null data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns with no null values @@ -2122,7 +2122,7 @@ TEST_F(IndexBuilderTest, NonExistentColumnUniqueIdTest) { // 4. Write data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -2255,7 +2255,7 @@ TEST_F(IndexBuilderTest, DropIndexV1FormatTest) { // 9. Write data to the rowset { - Block block = v1_schema->create_block(); + Block block = v1_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -2381,7 +2381,7 @@ TEST_F(IndexBuilderTest, ResourceCleanupTest) { // 4. Write data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -2538,7 +2538,7 @@ TEST_F(IndexBuilderTest, ArrayTypeIndexTest) { // 7. Create data block and write data { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Prepare columns for k1 and array_col @@ -2646,7 +2646,7 @@ TEST_F(IndexBuilderTest, UniqueKeysTableIndexTest) { auto rowset_writer = std::move(res).value(); { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -2806,7 +2806,7 @@ TEST_F(IndexBuilderTest, HandleSingleRowsetErrorTest) { auto rowset_writer = std::move(result).value(); { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -2928,7 +2928,7 @@ TEST_F(IndexBuilderTest, UpdateInvertedIndexInfoErrorTest) { // Write data { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns @@ -3045,7 +3045,7 @@ TEST_F(IndexBuilderTest, DropOneIndexNotAffectOtherIndexesOnSameColumnTest) { // 5. Write data to the rowset { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); // Add data for k1 and k2 columns diff --git a/be/test/storage/index/inverted/common/inverted_index_gc_binlogs_test.cpp b/be/test/storage/index/inverted/common/inverted_index_gc_binlogs_test.cpp index 4ae64d8da7dab3..b830a486ed8757 100644 --- a/be/test/storage/index/inverted/common/inverted_index_gc_binlogs_test.cpp +++ b/be/test/storage/index/inverted/common/inverted_index_gc_binlogs_test.cpp @@ -148,7 +148,7 @@ TEST_F(IndexGcBinglogsTest, gc_binlogs_test) { EXPECT_TRUE(res.has_value()) << res.error(); const auto& rowset_writer = res.value(); - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); Field key = Field::create_field(10); diff --git a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp index 6ca726770cfb5d..b210407fc937e9 100644 --- a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp +++ b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp @@ -657,7 +657,7 @@ class IndexCompactionUtils { EXPECT_TRUE(res.has_value()) << res.error(); const auto& rowset_writer = res.value(); - Block block = schema->create_block(); + Block block = schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (const auto& row : data[i]) { if constexpr (std::is_same_v) { diff --git a/be/test/storage/iterator/binlog_block_reader_utils_test.cpp b/be/test/storage/iterator/binlog_block_reader_utils_test.cpp index 06e4b19c8ffcba..9d099fe1c92698 100644 --- a/be/test/storage/iterator/binlog_block_reader_utils_test.cpp +++ b/be/test/storage/iterator/binlog_block_reader_utils_test.cpp @@ -19,9 +19,6 @@ #include -#include "core/block/column_with_type_and_name.h" -#include "core/data_type/data_type_number.h" - namespace doris { class BinlogBlockReaderUtilsTest : public testing::Test {}; @@ -29,38 +26,4 @@ TEST_F(BinlogBlockReaderUtilsTest, BuildBeforeColumnName) { EXPECT_EQ(binlog::build_before_column_name("v1"), "__BEFORE__v1__"); } -TEST_F(BinlogBlockReaderUtilsTest, ResolveBeforeColumnIndex) { - auto int_type = std::make_shared(); - auto col_key = ColumnInt64::create(); - auto col_val = ColumnInt64::create(); - auto col_before_val = ColumnInt64::create(); - auto col_op = ColumnInt64::create(); - - Block block; - block.insert({std::move(col_key), int_type, "k1"}); - block.insert({std::move(col_val), int_type, "v1"}); - block.insert({std::move(col_before_val), int_type, "__BEFORE__v1__"}); - block.insert({std::move(col_op), int_type, BINLOG_OP_COL}); - - EXPECT_EQ(binlog::resolve_before_column_index(block, 1, 3), 2); - EXPECT_EQ(binlog::resolve_before_column_index(block, 3, 3), 3); -} - -TEST_F(BinlogBlockReaderUtilsTest, ResolveBeforeColumnIndexFallbackWhenMissing) { - auto int_type = std::make_shared(); - auto col_key = ColumnInt64::create(); - auto col_val = ColumnInt64::create(); - auto col_op = ColumnInt64::create(); - - Block block; - block.insert({std::move(col_key), int_type, "k1"}); - block.insert({std::move(col_val), int_type, "v1"}); - block.insert({std::move(col_op), int_type, BINLOG_OP_COL}); - - // If __BEFORE__v1__ is missing, fall back to the current column to avoid out-of-bounds. - EXPECT_EQ(binlog::resolve_before_column_index(block, 1, 2), 1); - // Non-op columns should return themselves. - EXPECT_EQ(binlog::resolve_before_column_index(block, 0, 2), 0); -} - } // namespace doris diff --git a/be/test/storage/iterator/block_reader_agg_flush_test.cpp b/be/test/storage/iterator/block_reader_agg_flush_test.cpp index 77376285198fce..40aead29623297 100644 --- a/be/test/storage/iterator/block_reader_agg_flush_test.cpp +++ b/be/test/storage/iterator/block_reader_agg_flush_test.cpp @@ -92,9 +92,7 @@ void configure_reader_for_int64_sum(BlockReader& reader, const Block& src_block, config::enable_adaptive_batch_size = false; // Column layout: [0]=key, [1]=agg value. Output layout matches input. - reader._normal_columns_idx = {0}; reader._agg_columns_idx = {1}; - reader._return_columns_loc = {0, 1}; reader._stored_data_columns = make_stored_columns(src_block, batch_max_rows); reader._stored_has_null_tag.assign(reader._stored_data_columns.size(), false); diff --git a/be/test/storage/iterator/block_reader_change_next_block_test.cpp b/be/test/storage/iterator/block_reader_change_next_block_test.cpp index bfa774c325cec9..91bb95e0d8bb60 100644 --- a/be/test/storage/iterator/block_reader_change_next_block_test.cpp +++ b/be/test/storage/iterator/block_reader_change_next_block_test.cpp @@ -50,6 +50,7 @@ #include "core/data_type/data_type_number.h" #include "storage/binlog.h" #include "storage/iterator/binlog_block_reader_utils.h" +#include "storage/schema.h" #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" @@ -69,9 +70,6 @@ namespace { // 5: __DORIS_BINLOG_OP__ (Int64, one of ROW_BINLOG_APPEND/UPDATE/DELETE) constexpr int KEY_IDX = 0; constexpr int VAL_IDX = 1; -constexpr int BEFORE_VAL_IDX = 2; -constexpr int TSO_IDX = 3; -constexpr int LSN_IDX = 4; constexpr int OP_IDX = 5; struct Row { @@ -159,19 +157,41 @@ class FakeLevelIterator : public VCollectIterator::LevelIterator { std::shared_ptr _source; }; +// Read schema mirroring the merged binlog block layout above. +ReadSchemaSPtr make_read_schema(const std::vector& names = { + "key", "val", binlog::build_before_column_name("val"), + BINLOG_TSO_COL, BINLOG_LSN_COL, BINLOG_OP_COL}) { + std::vector cols; + auto add_bigint = [&](const std::string& name) { + auto col = std::make_shared(); + col->set_name(name); + col->set_type(FieldType::OLAP_FIELD_TYPE_BIGINT); + cols.push_back(std::move(col)); + }; + for (const auto& name : names) { + add_bigint(name); + } + return std::make_shared(std::move(cols)); +} + +TabletSchemaSPtr make_tablet_schema(const ReadSchemaSPtr& schema) { + auto tablet_schema = std::make_shared(); + for (const auto& column : schema->columns()) { + tablet_schema->append_column(*column); + } + return tablet_schema; +} + // Wire a BlockReader as if init() had already completed for a row-binlog change // scan over the fixed 6-column schema, then plug in the fake merge iterator. void configure_reader(BlockReader& reader, std::shared_ptr source, size_t batch_size) { config::enable_adaptive_batch_size = false; reader._reader_context.batch_size = batch_size; - // The fake LevelIterator base ctor dereferences reader->tablet_schema(), so a - // schema must exist even though its contents are unused by these code paths. - reader._tablet_schema = std::make_shared(); - - // All 6 columns are "normal" columns and are returned in-place. - reader._normal_columns_idx = {KEY_IDX, VAL_IDX, BEFORE_VAL_IDX, TSO_IDX, LSN_IDX, OP_IDX}; - reader._return_columns_loc = {0, 1, 2, 3, 4, 5}; + // Must be set before constructing the fake LevelIterator: its base ctor + // snapshots reader->_read_schema. + reader._read_schema = make_read_schema(); + reader._tablet_schema = make_tablet_schema(reader._read_schema); reader._next_row.block = source; reader._next_row.row_pos = 0; @@ -236,6 +256,20 @@ class BlockReaderChangeNextBlockTest : public testing::Test { bool _saved_adaptive = false; }; +TEST_F(BlockReaderChangeNextBlockTest, BinlogSchemaWithoutBeforeUsesCurrentColumn) { + auto read_schema = make_read_schema({"key", "val", BINLOG_TSO_COL, BINLOG_OP_COL}); + + EXPECT_EQ(VAL_IDX, read_schema->before_column_ordinal(VAL_IDX)); +} + +TEST_F(BlockReaderChangeNextBlockTest, BinlogSchemaDoesNotRequireLsn) { + auto read_schema = make_read_schema( + {"key", "val", binlog::build_before_column_name("val"), BINLOG_TSO_COL, BINLOG_OP_COL}); + + EXPECT_EQ(-1, read_schema->lsn_ordinal()); + EXPECT_EQ(2, read_schema->before_column_ordinal(VAL_IDX)); +} + // ============================================================================ // _min_delta_next_block branch coverage // ============================================================================ diff --git a/be/test/storage/key/row_key_encoder_test.cpp b/be/test/storage/key/row_key_encoder_test.cpp index 2a330c934e8334..ff4b184c598e38 100644 --- a/be/test/storage/key/row_key_encoder_test.cpp +++ b/be/test/storage/key/row_key_encoder_test.cpp @@ -552,7 +552,7 @@ class RowKeyEncoderTest : public testing::Test { void build(const TabletSchemaSPtr& schema, size_t num_rows, const std::function& fill) { _schema = schema; - _block = schema->create_block(); + _block = schema->create_storage_block(); { auto guard = _block.mutate_columns_scoped(); fill(guard.mutable_columns()); @@ -694,7 +694,7 @@ TEST_F(RowKeyEncoderTest, AllKeyTypesTable) { // 3. Fill the block from kAllKeyData. Cluster-key schemas keep the sort // columns in table order and reverse the primary-key source rows. _schema = schema; - _block = schema->create_block(); + _block = schema->create_storage_block(); auto timezone = cctz::utc_time_zone(); DataTypeSerDe::FormatOptions format_options; format_options.timezone = &timezone; diff --git a/be/test/storage/mow/historical_row_fetcher_test.cpp b/be/test/storage/mow/historical_row_fetcher_test.cpp index 2b8d1c7da9ec8c..3a931cbe33361d 100644 --- a/be/test/storage/mow/historical_row_fetcher_test.cpp +++ b/be/test/storage/mow/historical_row_fetcher_test.cpp @@ -44,7 +44,7 @@ class HistoricalRowFetcherTest : public MowTransformTestBase { // A narrow input block holding just the key column. static Block key_block(const TabletSchemaSPtr& schema, const std::vector& keys) { - Block block = schema->create_block_by_cids({0}); + Block block = schema->create_storage_block({0}); auto* column = block.get_by_position(0).column->assert_mutable().get(); for (int32_t k : keys) { column->insert_data(reinterpret_cast(&k), sizeof(int32_t)); @@ -68,7 +68,7 @@ TEST_F(HistoricalRowFetcherTest, ReadColumnsReturnsThePlannedRows) { ASSERT_EQ(fetcher.pinned_rowsets().size(), 1); std::vector cids {1}; - Block old_values = schema->create_block_by_cids(cids); + Block old_values = schema->create_storage_block(cids); std::map read_index; auto st = fetcher.read_columns(*schema, cids, old_values, &read_index, /*force_read_old_delete_signs=*/false); @@ -95,7 +95,7 @@ TEST_F(HistoricalRowFetcherTest, FillMissingColumnsMixesHistoryAndDefaults) { fetcher.plan_fixed_read(RowLocation {rowset->rowset_id(), 0, 2}, /*dst_pos=*/1); Block input = key_block(schema, {1, 3, 99}); - Block full_block = schema->create_block(); + Block full_block = schema->create_storage_block(); full_block.replace_by_position(0, input.get_by_position(0).column); std::vector use_default_or_null_flag {false, false, true}; @@ -125,7 +125,7 @@ TEST_F(HistoricalRowFetcherTest, FillMissingColumnsSkipsDeletedHistory) { fetcher.plan_fixed_read(RowLocation {rowset->rowset_id(), 0, 1}, /*dst_pos=*/1); Block input = key_block(schema, {1, 2}); - Block full_block = schema->create_block(); + Block full_block = schema->create_storage_block(); full_block.replace_by_position(0, input.get_by_position(0).column); std::vector use_default_or_null_flag {false, false}; diff --git a/be/test/storage/mow/key_probe_test.cpp b/be/test/storage/mow/key_probe_test.cpp index ee91ac2d2514ed..06582f396187d0 100644 --- a/be/test/storage/mow/key_probe_test.cpp +++ b/be/test/storage/mow/key_probe_test.cpp @@ -502,8 +502,8 @@ class RowCacheProbeTest : public KeyProbeTest { int32_t k, bool row_has_seq, int32_t seq, DataWriteType write_type) { const auto seq_idx = static_cast(schema->sequence_col_idx()); - Block block = row_has_seq ? schema->create_block_by_cids({0, seq_idx}) - : schema->create_block_by_cids({0}); + Block block = row_has_seq ? schema->create_storage_block({0, seq_idx}) + : schema->create_storage_block({0}); block.get_by_position(0).column->assert_mutable()->insert_data( reinterpret_cast(&k), sizeof(int32_t)); OlapBlockDataConvertor convertor; diff --git a/be/test/storage/mow/mow_transform_test_base.h b/be/test/storage/mow/mow_transform_test_base.h index 4b49552d9e441c..0aba583a34496e 100644 --- a/be/test/storage/mow/mow_transform_test_base.h +++ b/be/test/storage/mow/mow_transform_test_base.h @@ -327,7 +327,7 @@ class MowTransformTestBase : public testing::Test { EXPECT_TRUE(rw.has_value()) << rw.error(); auto writer = std::move(rw).value(); - Block block = schema->create_block(); + Block block = schema->create_storage_block(); std::vector mcols; for (size_t i = 0; i < block.columns(); ++i) { mcols.push_back(block.get_by_position(i).column->assert_mutable().get()); @@ -369,7 +369,7 @@ class MowTransformTestBase : public testing::Test { // a 1-row block then RowKeyEncoder::full_encode_primary_keys. std::string encode_key(const TabletSchemaSPtr& schema, const RowKeyEncoder& encoder, int32_t k) { - Block block = schema->create_block_by_cids({0}); + Block block = schema->create_storage_block({0}); block.get_by_position(0).column->assert_mutable()->insert_data( reinterpret_cast(&k), sizeof(int32_t)); OlapBlockDataConvertor convertor; @@ -386,7 +386,7 @@ class MowTransformTestBase : public testing::Test { std::string encode_key_with_seq(const TabletSchemaSPtr& schema, const RowKeyEncoder& encoder, int32_t k, int32_t seq) { const auto seq_idx = static_cast(schema->sequence_col_idx()); - Block block = schema->create_block_by_cids({0, seq_idx}); + Block block = schema->create_storage_block({0, seq_idx}); block.get_by_position(0).column->assert_mutable()->insert_data( reinterpret_cast(&k), sizeof(int32_t)); block.get_by_position(1).column->assert_mutable()->insert_data( diff --git a/be/test/storage/read_schema_test.cpp b/be/test/storage/read_schema_test.cpp new file mode 100644 index 00000000000000..2a1c37c094d70a --- /dev/null +++ b/be/test/storage/read_schema_test.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "storage/schema.h" + +namespace doris { +namespace { + +TabletColumnPtr create_int_column(int32_t unique_id, std::string name, bool is_key = false) { + auto column = std::make_shared( + FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, FieldType::OLAP_FIELD_TYPE_INT, + false, unique_id, sizeof(int32_t)); + column->set_name(std::move(name)); + column->set_is_key(is_key); + return column; +} + +TabletColumnPtr create_struct_column(int32_t unique_id) { + auto column = std::make_shared(); + column->set_unique_id(unique_id); + column->set_name("s"); + column->set_type(FieldType::OLAP_FIELD_TYPE_STRUCT); + column->set_is_nullable(false); + + auto first_child = create_int_column(unique_id + 100, "a"); + auto second_child = create_int_column(unique_id + 101, "b"); + column->add_sub_column(*first_child); + column->add_sub_column(*second_child); + return column; +} + +TEST(ReadSchemaTest, DefaultColumnsAreVisible) { + std::vector storage_columns {create_int_column(10, "k", true), + create_int_column(11, "dropped"), + create_struct_column(12)}; + + ReadSchema read_schema(storage_columns); + ASSERT_EQ(3, read_schema.num_block_columns()); + ASSERT_EQ(3, read_schema.num_read_columns()); + for (size_t ordinal = 0; ordinal < storage_columns.size(); ++ordinal) { + EXPECT_EQ(storage_columns[ordinal].get(), read_schema.column(ordinal)); + EXPECT_TRUE( + read_schema.data_type(ordinal)->equals(*storage_columns[ordinal]->get_vec_type())); + EXPECT_EQ(static_cast(ordinal), + read_schema.ordinal_by_uid(storage_columns[ordinal]->unique_id())); + } +} + +TEST(ReadSchemaTest, ProjectionPreservesRequestedOrder) { + std::vector storage_columns {create_int_column(10, "k", true), + create_int_column(11, "dropped"), + create_struct_column(12)}; + + ReadSchema read_schema(storage_columns, std::vector {2, 0}); + ASSERT_EQ(2, read_schema.num_block_columns()); + ASSERT_EQ(2, read_schema.num_read_columns()); + EXPECT_EQ("s", read_schema.column(0)->name()); + EXPECT_EQ("k", read_schema.column(1)->name()); + EXPECT_EQ(0, read_schema.ordinal_by_uid(12)); + EXPECT_EQ(1, read_schema.ordinal_by_uid(10)); + EXPECT_EQ(-1, read_schema.ordinal_by_uid(11)); +} + +TEST(ReadSchemaTest, ExpectedTypesDefineReadBlock) { + std::vector read_columns {create_struct_column(12), + create_int_column(10, "k", true)}; + + auto pruned_struct_type = std::make_shared( + DataTypes {std::make_shared()}, Strings {"a"}); + auto int_type = std::make_shared(); + ReadSchema read_schema(std::move(read_columns), + std::vector {pruned_struct_type, int_type}); + EXPECT_FALSE(read_schema.column(0)->get_vec_type()->equals(*pruned_struct_type)); + + Block block = read_schema.create_read_block(); + ASSERT_EQ(2, block.columns()); + EXPECT_EQ("s", block.get_by_position(0).name); + EXPECT_EQ("k", block.get_by_position(1).name); + EXPECT_TRUE(block.get_by_position(0).type->equals(*pruned_struct_type)); + EXPECT_TRUE(block.get_by_position(1).type->equals(*int_type)); +} + +TEST(ReadSchemaTest, AppendedStorageColumnDoesNotExtendReadBlock) { + std::vector read_columns {create_int_column(10, "k", true), + create_struct_column(12)}; + auto dropped_column = create_int_column(11, "dropped"); + ReadSchema read_schema(std::move(read_columns)); + + ColumnId suffix_ordinal = read_schema.append_column(dropped_column); + EXPECT_EQ(2, suffix_ordinal); + EXPECT_EQ(2, read_schema.num_block_columns()); + EXPECT_EQ(3, read_schema.num_read_columns()); + EXPECT_EQ(suffix_ordinal, read_schema.ordinal_by_uid(11)); + EXPECT_TRUE(read_schema.data_type(suffix_ordinal)->equals(*dropped_column->get_vec_type())); + EXPECT_EQ(2, read_schema.create_read_block().columns()); +} + +} // namespace +} // namespace doris diff --git a/be/test/storage/rowid_conversion_test.cpp b/be/test/storage/rowid_conversion_test.cpp index c8b3bad9336a04..d1b9156fb3f509 100644 --- a/be/test/storage/rowid_conversion_test.cpp +++ b/be/test/storage/rowid_conversion_test.cpp @@ -54,6 +54,7 @@ #include "storage/rowset/rowset_reader_context.h" #include "storage/rowset/rowset_writer.h" #include "storage/rowset/rowset_writer_context.h" +#include "storage/schema.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_meta.h" @@ -191,7 +192,7 @@ class TestRowIdConversion : public testing::TestWithParamcreate_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (int rid = 0; rid < rowset_data[i].size(); ++rid) { int32_t c1 = std::get<0>(rowset_data[i][rid]); @@ -363,15 +364,16 @@ class TestRowIdConversion : public testing::TestWithParam return_columns = {0, 1}; - reader_context.return_columns = &return_columns; + auto read_schema = std::make_shared(tablet_schema->columns(), + std::vector {0, 1}); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); // read output rowset data std::vector> output_data; do { - Block output_block = tablet_schema->create_block(); + Block output_block = read_schema->create_read_block(); s = output_rs_reader->next_batch(&output_block); auto columns = output_block.get_columns_with_type_and_name(); EXPECT_EQ(columns.size(), 2); diff --git a/be/test/storage/rowset/beta_rowset_reader_test.cpp b/be/test/storage/rowset/beta_rowset_reader_test.cpp deleted file mode 100644 index a85c8cadfa34af..00000000000000 --- a/be/test/storage/rowset/beta_rowset_reader_test.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/rowset/beta_rowset_reader.h" - -#include -#include - -#include -#include - -#include "storage/predicate/column_predicate.h" -#include "storage/rowset/beta_rowset.h" -#include "storage/rowset/rowset_meta.h" -#include "storage/rowset/rowset_reader_context.h" -#include "storage/tablet/tablet_schema.h" - -namespace doris { - -// Verifies the forced TSO range pushdown in BetaRowsetReader::get_segment_iterators (introduced -// for binlog/snapshot incremental read). The (start_tso, end_tso] comparison predicates and the -// tso predicate column must be injected directly onto read options, and the tso column must be -// appended to read_columns only when it is not already in return_columns. -// -// The rowset here has zero segments, so get_segment_iterators skips segment iterator creation -// and returns after the injection logic, which is exactly the code path under test. -class BetaRowsetReaderTsoInjectionTest : public testing::Test { -protected: - // Schema: (k1 int key, __binlog_tso__ bigint). Column index 1 is used as the tso column. - static constexpr ColumnId kTsoColumnId = 1; - - void SetUp() override { - _tablet_schema = std::make_shared(); - TabletSchemaPB schema_pb; - schema_pb.set_keys_type(DUP_KEYS); - schema_pb.set_num_short_key_columns(1); - schema_pb.set_num_rows_per_row_block(1024); - schema_pb.set_next_column_unique_id(3); - - ColumnPB* key = schema_pb.add_column(); - key->set_unique_id(1); - key->set_name("k1"); - key->set_type("INT"); - key->set_is_key(true); - key->set_length(4); - key->set_index_length(4); - key->set_is_nullable(false); - - ColumnPB* tso = schema_pb.add_column(); - tso->set_unique_id(2); - tso->set_name("__binlog_tso__"); - tso->set_type("BIGINT"); - tso->set_is_key(false); - tso->set_length(8); - tso->set_is_nullable(false); - - _tablet_schema->init_from_pb(schema_pb); - - _rowset_meta = std::make_shared(); - RowsetMetaPB rowset_meta_pb; - rowset_meta_pb.set_rowset_id(10001); - rowset_meta_pb.set_tablet_id(1); - rowset_meta_pb.set_rowset_type(BETA_ROWSET); - rowset_meta_pb.set_rowset_state(VISIBLE); - rowset_meta_pb.set_start_version(2); - rowset_meta_pb.set_end_version(2); - rowset_meta_pb.set_num_segments(0); - rowset_meta_pb.set_num_rows(0); - rowset_meta_pb.set_empty(true); - _rowset_meta->init_from_pb(rowset_meta_pb); - - _rowset = std::make_shared(_tablet_schema, _rowset_meta, ""); - _reader = std::make_shared(_rowset); - } - - RowsetReaderContext make_context() { - RowsetReaderContext ctx; - ctx.tablet_schema = _tablet_schema; - ctx.return_columns = &_return_columns; - ctx.need_ordered_result = false; - return ctx; - } - - // Counts predicates that act on the tso column in the injected read options. - static int count_tso_predicates(const BetaRowsetReader& reader) { - int count = 0; - for (const auto& pred : reader._read_options.column_predicates) { - if (pred->column_id() == kTsoColumnId) { - ++count; - } - } - return count; - } - - TabletSchemaSPtr _tablet_schema; - RowsetMetaSharedPtr _rowset_meta; - BetaRowsetSharedPtr _rowset; - std::shared_ptr _reader; - std::vector _return_columns = {0}; -}; - -TEST_F(BetaRowsetReaderTsoInjectionTest, InjectBothStartAndEndTso) { - RowsetReaderContext ctx = make_context(); - ctx.tso_predicate_column_id = kTsoColumnId; - ctx.start_tso = 100; - ctx.end_tso = 200; - - std::vector iters; - ASSERT_TRUE(_reader->get_segment_iterators(&ctx, &iters).ok()); - - // Both GT(start) and LE(end) predicates are injected on the tso column. - EXPECT_EQ(count_tso_predicates(*_reader), 2); - EXPECT_EQ(_reader->_read_options.col_id_to_predicates.count(kTsoColumnId), 1); -} - -TEST_F(BetaRowsetReaderTsoInjectionTest, InjectOnlyStartTso) { - RowsetReaderContext ctx = make_context(); - ctx.tso_predicate_column_id = kTsoColumnId; - ctx.start_tso = 100; - - std::vector iters; - ASSERT_TRUE(_reader->get_segment_iterators(&ctx, &iters).ok()); - - EXPECT_EQ(count_tso_predicates(*_reader), 1); - EXPECT_EQ(_reader->_read_options.col_id_to_predicates.count(kTsoColumnId), 1); -} - -TEST_F(BetaRowsetReaderTsoInjectionTest, InjectOnlyEndTso) { - RowsetReaderContext ctx = make_context(); - ctx.tso_predicate_column_id = kTsoColumnId; - ctx.end_tso = 200; - - std::vector iters; - ASSERT_TRUE(_reader->get_segment_iterators(&ctx, &iters).ok()); - - EXPECT_EQ(count_tso_predicates(*_reader), 1); - EXPECT_EQ(_reader->_read_options.col_id_to_predicates.count(kTsoColumnId), 1); -} - -TEST_F(BetaRowsetReaderTsoInjectionTest, NoInjectionWhenTsoRangeAbsent) { - RowsetReaderContext ctx = make_context(); - // Column id present but neither start nor end tso set: nothing should be injected. - ctx.tso_predicate_column_id = kTsoColumnId; - - std::vector iters; - ASSERT_TRUE(_reader->get_segment_iterators(&ctx, &iters).ok()); - - EXPECT_EQ(count_tso_predicates(*_reader), 0); -} - -TEST_F(BetaRowsetReaderTsoInjectionTest, AppendTsoColumnWhenNotInReturnColumns) { - RowsetReaderContext ctx = make_context(); - // return_columns only has k1 (id 0), tso column (id 1) must be appended to read columns. - ctx.tso_predicate_column_id = kTsoColumnId; - ctx.start_tso = 100; - - std::vector iters; - ASSERT_TRUE(_reader->get_segment_iterators(&ctx, &iters).ok()); - - // input schema (read columns) = return_columns + appended tso column = 2 columns. - EXPECT_EQ(_reader->_input_schema->num_column_ids(), 2); -} - -TEST_F(BetaRowsetReaderTsoInjectionTest, NotAppendTsoColumnWhenAlreadyInReturnColumns) { - RowsetReaderContext ctx = make_context(); - // tso column already selected in return_columns: it must not be appended twice. - std::vector return_columns = {0, kTsoColumnId}; - ctx.return_columns = &return_columns; - ctx.tso_predicate_column_id = kTsoColumnId; - ctx.start_tso = 100; - - std::vector iters; - ASSERT_TRUE(_reader->get_segment_iterators(&ctx, &iters).ok()); - - EXPECT_EQ(_reader->_input_schema->num_column_ids(), 2); -} - -} // namespace doris diff --git a/be/test/storage/rowset/segment_flusher_format_test.cpp b/be/test/storage/rowset/segment_flusher_format_test.cpp index bb53fe0031f4b4..e93a9e4e49c3f9 100644 --- a/be/test/storage/rowset/segment_flusher_format_test.cpp +++ b/be/test/storage/rowset/segment_flusher_format_test.cpp @@ -458,7 +458,7 @@ const TypeCase& key_type_from_column_name(std::string_view name) { Result create_wide_key_block(const TabletSchemaSPtr& schema, const WideKeySchemaOptions& options, int segment_ordinal) { - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (size_t row = 0; row < 3; ++row) { const size_t aggregate_value_index = options.keys_type == AGG_KEYS && row == 1 ? 0 : row; for (size_t column_index = 0; column_index < block.columns(); ++column_index) { @@ -527,7 +527,7 @@ TabletSchemaSPtr create_all_scalar_value_schema(TabletStorageFormatPB storage_fo } Result create_all_scalar_value_block(const TabletSchemaSPtr& schema, int segment_ordinal) { - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (size_t row = 0; row < 3; ++row) { RETURN_IF_ERROR_RESULT(append_text_value( &block, 0, std::to_string(segment_ordinal * 10 + static_cast(row)))); @@ -593,7 +593,7 @@ TabletSchemaSPtr create_embedded_index_schema(TabletStorageFormatPB storage_form Result create_embedded_index_block(const TabletSchemaSPtr& schema, int segment_ordinal) { constexpr size_t kRows = 180; constexpr size_t kPayloadSize = 96; - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (size_t row = 0; row < kRows; ++row) { RETURN_IF_ERROR_RESULT( append_text_value(&block, 0, std::to_string(segment_ordinal * kRows + row))); @@ -643,7 +643,7 @@ TabletSchemaSPtr create_external_inverted_index_schema(InvertedIndexStorageForma Result create_external_inverted_index_block(const TabletSchemaSPtr& schema, int segment_ordinal) { - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (size_t row = 0; row < kExternalIndexRows; ++row) { RETURN_IF_ERROR_RESULT(append_text_value( &block, 0, std::to_string(segment_ordinal * kExternalIndexRows + row))); @@ -697,7 +697,7 @@ std::array ann_vector(int segment_ordinal, size_t row) { } Result create_ann_index_block(const TabletSchemaSPtr& schema, int segment_ordinal) { - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (size_t row = 0; row < kExternalIndexRows; ++row) { RETURN_IF_ERROR_RESULT(append_text_value( &block, 0, std::to_string(segment_ordinal * kExternalIndexRows + row))); @@ -800,7 +800,7 @@ Result create_complex_value_block(const TabletSchemaSPtr& schema, int seg R"({"a":1,"nested":{"x":"one"}})", R"({"a":2,"array":[1,2]})", R"({"different":true,"nested":{"x":"three","y":3.5}})"}; - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (size_t row = 0; row < 3; ++row) { const auto value_index = (row + segment_ordinal) % 3; RETURN_IF_ERROR_RESULT(append_text_value( @@ -962,8 +962,8 @@ Result create_complex_row_binlog_block( const std::array structs {R"({"f_int":10,"f_text":"ten"})", "{}", R"({"f_int":30,"f_text":"thirty"})"}; Block block = partial_update_info == nullptr - ? schema->create_block() - : schema->create_block_by_cids(partial_update_info->update_cids); + ? schema->create_storage_block() + : schema->create_storage_block(partial_update_info->update_cids); for (size_t row = 0; row < 3; ++row) { const auto value_index = (row + segment_ordinal) % arrays.size(); for (size_t column_index = 0; column_index < block.columns(); ++column_index) { @@ -1020,7 +1020,7 @@ TabletSchemaSPtr create_row_store_schema(KeysType keys_type = DUP_KEYS, bool ena Result create_row_store_block(const TabletSchemaSPtr& schema, int segment_ordinal, size_t rows = 3, size_t value_size = 40) { - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (size_t row = 0; row < rows; ++row) { for (size_t column_index = 0; column_index < block.columns(); ++column_index) { const auto& name = block.get_by_position(column_index).name; @@ -1074,7 +1074,7 @@ Result create_variant_row_store_block(const TabletSchemaSPtr& schema, int const std::array variants { R"({"a":1,"nested":{"x":"one"}})", R"({"a":2,"array":[1,2]})", R"({"different":true,"nested":{"x":"three","y":3.5}})"}; - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (size_t row = 0; row < rows; ++row) { RETURN_IF_ERROR_RESULT(append_text_value( &block, 0, std::to_string(segment_ordinal * 100 + static_cast(row)))); @@ -1174,7 +1174,7 @@ Result create_fixed_partial_update_block(const TabletSchemaSPtr& schema, const std::array variants { R"({"a":1,"nested":{"x":"one"}})", R"({"a":2,"array":[1,2]})", R"({"different":true,"nested":{"x":"three","y":3.5}})"}; - Block block = schema->create_block_by_cids(partial_update_info.update_cids); + Block block = schema->create_storage_block(partial_update_info.update_cids); for (int row = 0; row < 3; ++row) { for (size_t column_index = 0; column_index < block.columns(); ++column_index) { const auto& name = block.get_by_position(column_index).name; @@ -1204,7 +1204,7 @@ Result create_fixed_partial_update_block(const TabletSchemaSPtr& schema, Result create_flexible_partial_update_block(const TabletSchemaSPtr& schema, int segment_ordinal) { DORIS_CHECK_EQ(schema->num_variant_columns(), 0); - Block block = schema->create_block(); + Block block = schema->create_storage_block(); auto* skip_bitmap = assert_cast( block.get_by_position(schema->skip_bitmap_col_idx()).column->assert_mutable().get()); constexpr std::array key_offsets {0, 0, 1, 2}; @@ -1261,7 +1261,7 @@ struct IntegerTabletBlockOptions { Result create_integer_tablet_block(const TabletSchemaSPtr& schema, int segment_ordinal, IntegerTabletBlockOptions options = {}) { - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (int row = 0; row < 3; ++row) { for (size_t column_index = 0; column_index < block.columns(); ++column_index) { const auto& name = block.get_by_position(column_index).name; @@ -1307,7 +1307,7 @@ Result create_binlog_partial_update_block(const TabletSchemaSPtr& schema, const PartialUpdateInfo& partial_update_info, int segment_ordinal, bool stale_first_existing_sequence = false) { - Block block = schema->create_block_by_cids(partial_update_info.update_cids); + Block block = schema->create_storage_block(partial_update_info.update_cids); for (int row = 0; row < 3; ++row) { for (size_t column_index = 0; column_index < block.columns(); ++column_index) { const auto& name = block.get_by_position(column_index).name; @@ -1333,7 +1333,7 @@ Result create_binlog_partial_update_block(const TabletSchemaSPtr& schema, Result create_mow_history_block(const TabletSchemaSPtr& schema) { constexpr std::array primary_key_order {0, 1, 10, 11}; - Block block = schema->create_block(); + Block block = schema->create_storage_block(); for (const int key : primary_key_order) { for (size_t column_index = 0; column_index < block.columns(); ++column_index) { const auto& name = block.get_by_position(column_index).name; @@ -2139,9 +2139,7 @@ Result read_logical_segment(const std::string& path, uin })); std::sort(physical_column_metadata.begin(), physical_column_metadata.end()); - std::vector column_ids(schema->num_columns()); - std::iota(column_ids.begin(), column_ids.end(), 0); - auto read_schema = std::make_shared(schema->columns(), column_ids); + auto read_schema = std::make_shared(schema->columns()); OlapReaderStatistics stats; StorageReadOptions read_options; read_options.stats = &stats; @@ -2149,9 +2147,9 @@ Result read_logical_segment(const std::string& path, uin std::unique_ptr iterator; RETURN_IF_ERROR_RESULT(segment->new_iterator(read_schema, read_options, &iterator)); - MutableBlock contents(schema->create_block()); + MutableBlock contents(schema->create_storage_block()); while (true) { - Block batch = schema->create_block(); + Block batch = schema->create_storage_block(); auto status = iterator->next_batch(&batch); if (status.is()) { break; diff --git a/be/test/storage/segment/column_reader_writer_test.cpp b/be/test/storage/segment/column_reader_writer_test.cpp index 98e6b74b266da3..5ea417c282a04e 100644 --- a/be/test/storage/segment/column_reader_writer_test.cpp +++ b/be/test/storage/segment/column_reader_writer_test.cpp @@ -135,7 +135,7 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, EXPECT_TRUE(st.ok()); ColumnIteratorUPtr iter; - st = reader->new_iterator(&iter); + st = reader->new_iterator(&iter, nullptr); EXPECT_TRUE(st.ok()); ColumnIteratorOptions iter_opts; @@ -186,7 +186,7 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, EXPECT_TRUE(st.ok()); ColumnIteratorUPtr iter; - st = reader->new_iterator(&iter); + st = reader->new_iterator(&iter, nullptr); EXPECT_TRUE(st.ok()); EXPECT_TRUE(st.ok()); diff --git a/be/test/storage/segment/constant_column_iterator_test.cpp b/be/test/storage/segment/constant_column_iterator_test.cpp index cde4063fd81754..0c596199312252 100644 --- a/be/test/storage/segment/constant_column_iterator_test.cpp +++ b/be/test/storage/segment/constant_column_iterator_test.cpp @@ -58,7 +58,7 @@ TEST_F(ConstantColumnIteratorTest, ColumnReaderCreateWithConstValueReturnsConsta TabletColumn column; column.set_type(FieldType::OLAP_FIELD_TYPE_BIGINT); ColumnIteratorUPtr iter; - st = reader->new_iterator(&iter, &column, nullptr); + st = reader->new_iterator(&iter, &column); ASSERT_TRUE(st.ok()) << st; MutableColumnPtr dst = ColumnVector::create(); diff --git a/be/test/storage/segment/historical_row_retriever_test.cpp b/be/test/storage/segment/historical_row_retriever_test.cpp index eedc24c4fa2830..305dfbcbd41956 100644 --- a/be/test/storage/segment/historical_row_retriever_test.cpp +++ b/be/test/storage/segment/historical_row_retriever_test.cpp @@ -95,7 +95,7 @@ class HistoricalRowRetrieverTest : public MowTransformTestBase { // A block holding the key column and the sequence column, in that order. static Block key_seq_block(const TabletSchemaSPtr& schema, int32_t key, int32_t seq) { - Block block = schema->create_block_by_cids( + Block block = schema->create_storage_block( {0, static_cast(schema->sequence_col_idx())}); block.get_by_position(0).column->assert_mutable()->insert_data( reinterpret_cast(&key), sizeof(int32_t)); @@ -106,7 +106,7 @@ class HistoricalRowRetrieverTest : public MowTransformTestBase { // A block holding only the key column, plus the full-width block the AFTER image is built into. static Block key_block(const TabletSchemaSPtr& schema, const std::vector& keys) { - Block block = schema->create_block_by_cids({0}); + Block block = schema->create_storage_block({0}); auto* column = block.get_by_position(0).column->assert_mutable().get(); for (int32_t k : keys) { column->insert_data(reinterpret_cast(&k), sizeof(int32_t)); @@ -142,7 +142,7 @@ TEST_F(HistoricalRowRetrieverTest, UpdateReadsHistoryAndAppendTakesDefault) { // a read-only probe: the load's delete bitmap must stay untouched EXPECT_EQ(mow->delete_bitmap->cardinality(), 0U); - Block after_block = schema->create_block(); + Block after_block = schema->create_storage_block(); after_block.replace_by_position(0, input.get_by_position(0).column); st = retriever.build_after_block(&after_block, 0, 2); ASSERT_TRUE(st.ok()) << st; @@ -177,7 +177,7 @@ TEST_F(HistoricalRowRetrieverTest, DeleteReadsHistoryOnlyWhenBeforeImageIsWanted ASSERT_EQ(retriever.get_operators().size(), 1); EXPECT_EQ(retriever.get_operators()[0], ROW_BINLOG_DELETE); - Block before_block = schema->create_block_by_cids({1}); + Block before_block = schema->create_storage_block({1}); st = retriever.build_before_block(&before_block, {1}, 0, 1); ASSERT_TRUE(st.ok()) << st; ASSERT_EQ(before_block.rows(), 1); @@ -217,7 +217,7 @@ TEST_F(HistoricalRowRetrieverTest, RowLosingOnSequenceStillReadsTheStoredRow) { EXPECT_EQ(retriever.get_operators()[0], ROW_BINLOG_UPDATE); EXPECT_EQ(mow->delete_bitmap->cardinality(), 0U); // MarkDeleted::NONE: nothing is marked - Block before_block = schema->create_block_by_cids({1}); + Block before_block = schema->create_storage_block({1}); ASSERT_TRUE(retriever.build_before_block(&before_block, {1}, 0, 1).ok()); ASSERT_EQ(before_block.rows(), 1); EXPECT_FALSE(read_is_null(before_block, 0, 0)); @@ -250,12 +250,12 @@ TEST_F(HistoricalRowRetrieverTest, ClearResetsThePerBlockState) { ASSERT_EQ(retriever.get_operators().size(), 1); EXPECT_EQ(retriever.get_operators()[0], ROW_BINLOG_UPDATE); - Block after_block = schema->create_block(); + Block after_block = schema->create_storage_block(); after_block.replace_by_position(0, first.get_by_position(0).column); ASSERT_TRUE(retriever.build_after_block(&after_block, 0, 1).ok()); EXPECT_EQ(read_int(after_block, 1, 0), 11); // the same plan still serves the BEFORE image after the AFTER image consumed it - Block before_block = schema->create_block_by_cids({1}); + Block before_block = schema->create_storage_block({1}); ASSERT_TRUE(retriever.build_before_block(&before_block, {1}, 0, 1).ok()); ASSERT_EQ(before_block.rows(), 1); EXPECT_FALSE(read_is_null(before_block, 0, 0)); @@ -271,13 +271,13 @@ TEST_F(HistoricalRowRetrieverTest, ClearResetsThePerBlockState) { ASSERT_EQ(retriever.get_operators().size(), 1); EXPECT_EQ(retriever.get_operators()[0], ROW_BINLOG_APPEND); - Block before_block = schema->create_block_by_cids({1}); + Block before_block = schema->create_storage_block({1}); ASSERT_TRUE(retriever.build_before_block(&before_block, {1}, 0, 1).ok()); ASSERT_EQ(before_block.rows(), 1); // key 1's retained plan entry would land here, since both blocks plan destination position 0 EXPECT_TRUE(read_is_null(before_block, 0, 0)); - Block after_block = schema->create_block(); + Block after_block = schema->create_storage_block(); after_block.replace_by_position(0, second.get_by_position(0).column); ASSERT_TRUE(retriever.build_after_block(&after_block, 0, 1).ok()); ASSERT_EQ(after_block.rows(), 1); diff --git a/be/test/storage/segment/row_binlog_segment_writer_test.cpp b/be/test/storage/segment/row_binlog_segment_writer_test.cpp index 0c0c4b9972f591..c01cac62dd3961 100644 --- a/be/test/storage/segment/row_binlog_segment_writer_test.cpp +++ b/be/test/storage/segment/row_binlog_segment_writer_test.cpp @@ -58,7 +58,7 @@ TabletSchemaSPtr create_source_schema() { } Block create_source_block(const TabletSchemaSPtr& schema) { - Block block = schema->create_block(); + Block block = schema->create_storage_block(); auto columns_guard = block.mutate_columns_scoped(); auto& columns = columns_guard.mutable_columns(); for (int i = 0; i < 2; ++i) { @@ -82,7 +82,7 @@ TEST(RowBinlogSegmentWriterTest, collectHiddenKeyInSourceDataWriter) { EXPECT_EQ(3, writer.normal_column_count()); Block block = create_source_block(source_schema); - Block full_block = source_schema->create_block(); + Block full_block = source_schema->create_storage_block(); std::vector partial_source_cids; ASSERT_TRUE( writer.prepare_by_source_block(&block, 0, 2, partial_source_cids, &full_block).ok()); diff --git a/be/test/storage/segment/segment_cache_test.cpp b/be/test/storage/segment/segment_cache_test.cpp index 9b116c1fe6b80f..40d7899772266f 100644 --- a/be/test/storage/segment/segment_cache_test.cpp +++ b/be/test/storage/segment/segment_cache_test.cpp @@ -70,17 +70,15 @@ class OlapMeta; static const uint32_t MAX_PATH_LEN = 1024; static StorageEngine* engine_ref = nullptr; -static std::shared_ptr create_full_schema(const TabletSchemaSPtr& tablet_schema) { +static std::shared_ptr create_full_schema(const TabletSchemaSPtr& tablet_schema) { size_t num_columns = tablet_schema->num_columns(); if (num_columns > 0 && tablet_schema->columns().back()->name() == BeConsts::ROW_STORE_COL) { --num_columns; } - std::vector column_ids(num_columns); - for (uint32_t cid = 0; cid < num_columns; ++cid) { - column_ids[cid] = cid; - } - return std::make_shared(tablet_schema->columns(), column_ids); + std::vector columns(tablet_schema->columns().begin(), + tablet_schema->columns().begin() + num_columns); + return std::make_shared(std::move(columns)); } static void set_up() { @@ -365,10 +363,10 @@ TEST_F(SegmentCacheTest, vec_sequence_col) { opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; - std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); + std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); auto s = segments[0]->new_iterator(schema, opts, &iter); ASSERT_TRUE(s.ok()); - auto read_block = rowset->tablet_schema()->create_block(); + auto read_block = schema->create_read_block(); res = iter->next_batch(&read_block); ASSERT_TRUE(res.ok()) << res; ASSERT_EQ(1, read_block.rows()); diff --git a/be/test/storage/segment/segment_iterator_apply_index_expr_test.cpp b/be/test/storage/segment/segment_iterator_apply_index_expr_test.cpp index 9d550bb4b8f67b..3ea91b996c6c5a 100644 --- a/be/test/storage/segment/segment_iterator_apply_index_expr_test.cpp +++ b/be/test/storage/segment/segment_iterator_apply_index_expr_test.cpp @@ -100,14 +100,12 @@ VExprContextSPtr make_mock_ctx(Status eval_status, bool with_index_context = tru expr->set_evaluate_status(std::move(eval_status)); auto ctx = std::make_shared(expr); if (with_index_context) { - std::vector col_ids; std::vector> index_iters; std::vector storage_types; std::unordered_map> status_map; ColumnIteratorOptions column_iter_opts; - auto index_ctx = - std::make_shared(col_ids, index_iters, storage_types, status_map, - nullptr, nullptr, column_iter_opts); + auto index_ctx = std::make_shared(index_iters, storage_types, status_map, + nullptr, nullptr, column_iter_opts); ctx->set_index_context(index_ctx); } return ctx; @@ -121,11 +119,8 @@ class SegmentIteratorApplyIndexExprTest : public testing::Test { _tablet_schema = make_tablet_schema(); _segment = make_stub_segment(100, _tablet_schema); - std::vector read_column_ids(_tablet_schema->num_columns()); - for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) { - read_column_ids[cid] = cid; - } - _read_schema = std::make_shared(_tablet_schema->columns(), read_column_ids); + // Read schema covers all tablet columns in order, so ordinal == tablet cid. + _read_schema = std::make_shared(_tablet_schema->columns()); _iter = std::make_unique(_segment, _read_schema); // Set up RuntimeState with fallback enabled so _downgrade_without_index works @@ -139,7 +134,7 @@ class SegmentIteratorApplyIndexExprTest : public testing::Test { std::shared_ptr _segment; std::shared_ptr _tablet_schema; - SchemaSPtr _read_schema; + ReadSchemaSPtr _read_schema; std::unique_ptr _iter; RuntimeState _runtime_state; OlapReaderStatistics _stats; diff --git a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp index 172ab8fdafc1ad..dcb270062e7b86 100644 --- a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp +++ b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp @@ -123,12 +123,9 @@ std::shared_ptr make_commit_tso_gt_predicate(int32_t co return predicates; } -SchemaSPtr make_read_schema(const TabletSchemaSPtr& tablet_schema) { - std::vector read_column_ids(tablet_schema->num_columns()); - for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) { - read_column_ids[cid] = cid; - } - return std::make_shared(tablet_schema->columns(), read_column_ids); +// Read schema covers all tablet columns in order, so ordinal == tablet cid. +ReadSchemaSPtr make_read_schema(const TabletSchemaSPtr& tablet_schema) { + return std::make_shared(tablet_schema->columns()); } } // namespace diff --git a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp index b990201b49544c..bf340a207c65d2 100644 --- a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp +++ b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp @@ -98,12 +98,9 @@ TabletSchemaSPtr make_tablet_schema() { return tablet_schema; } -SchemaSPtr make_read_schema(const TabletSchemaSPtr& tablet_schema) { - std::vector read_column_ids(tablet_schema->num_columns()); - for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) { - read_column_ids[cid] = cid; - } - return std::make_shared(tablet_schema->columns(), read_column_ids); +// Read schema covers all tablet columns in order, so ordinal == tablet cid. +ReadSchemaSPtr make_read_schema(const TabletSchemaSPtr& tablet_schema) { + return std::make_shared(tablet_schema->columns()); } Block make_int_block() { @@ -125,7 +122,7 @@ class SegmentIteratorLazyPrunedTest : public ::testing::Test { auto iter = std::make_unique(nullptr, _read_schema); iter->_opts.tablet_schema = _tablet_schema; iter->_opts.stats = &_stats; - iter->_support_lazy_read_pruned_columns.insert(0); + iter->_lazy_pruned_ordinals.push_back(0); iter->_column_iterators.resize(1); auto column_iter = std::make_unique(); @@ -135,7 +132,7 @@ class SegmentIteratorLazyPrunedTest : public ::testing::Test { } TabletSchemaSPtr _tablet_schema; - SchemaSPtr _read_schema; + ReadSchemaSPtr _read_schema; OlapReaderStatistics _stats; }; diff --git a/be/test/storage/segment/segment_iterator_limit_opt_test.cpp b/be/test/storage/segment/segment_iterator_limit_opt_test.cpp index 858947436a8965..b1adde1d122268 100644 --- a/be/test/storage/segment/segment_iterator_limit_opt_test.cpp +++ b/be/test/storage/segment/segment_iterator_limit_opt_test.cpp @@ -74,11 +74,8 @@ class SegmentIteratorLimitOptTest : public ::testing::Test { _tablet_schema = std::make_shared(); _tablet_schema->init_from_pb(schema_pb); - std::vector read_column_ids(_tablet_schema->num_columns()); - for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) { - read_column_ids[cid] = cid; - } - _read_schema = std::make_shared(_tablet_schema->columns(), read_column_ids); + // Read schema covers all tablet columns in order, so ordinal == tablet cid. + _read_schema = std::make_shared(_tablet_schema->columns()); } // Build a SegmentIterator with minimal opts for _can_opt_limit_reads() testing. @@ -92,7 +89,7 @@ class SegmentIteratorLimitOptTest : public ::testing::Test { } std::shared_ptr _tablet_schema; - SchemaSPtr _read_schema; + ReadSchemaSPtr _read_schema; OlapReaderStatistics _stats; }; diff --git a/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp b/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp index 640060e2083f94..64627706e39050 100644 --- a/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp +++ b/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp @@ -50,16 +50,13 @@ TEST(SegmentIteratorNoNeedReadDataTest, extracted_variant_count_on_index) { const ColumnId subcol_cid = tablet_schema->field_index(*subcol.path_info_ptr()); ASSERT_GE(subcol_cid, 0); - std::vector read_column_ids(tablet_schema->num_columns()); - for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) { - read_column_ids[cid] = cid; - } - auto read_schema = std::make_shared(tablet_schema->columns(), read_column_ids); + // Read schema covers all tablet columns in order, so ordinal == tablet cid. + auto read_schema = std::make_shared(tablet_schema->columns()); SegmentIterator iter(nullptr, read_schema); iter._opts.tablet_schema = tablet_schema; iter._opts.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; - iter._need_read_data_indices[static_cast(subcol_cid)] = false; - iter._output_columns.emplace(1); + iter._column_states[subcol_cid].need_read_data = false; + iter._output_column_uids.emplace(1); EXPECT_FALSE(iter._need_read_data(subcol_cid)); @@ -81,24 +78,22 @@ TEST(SegmentIteratorNoNeedReadDataTest, zonemap_always_true_predicate_column) { pred_col->set_unique_id(2); pred_col->set_name("event_time"); pred_col->set_type("DATETIMEV2"); + pred_col->set_frac(0); pred_col->set_is_key(false); pred_col->set_is_nullable(false); auto tablet_schema = std::make_shared(); tablet_schema->init_from_pb(schema_pb); - std::vector read_column_ids(tablet_schema->num_columns()); - for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) { - read_column_ids[cid] = cid; - } - auto read_schema = std::make_shared(tablet_schema->columns(), read_column_ids); + // Read schema covers all tablet columns in order, so ordinal == tablet cid. + auto read_schema = std::make_shared(tablet_schema->columns()); SegmentIterator iter(nullptr, read_schema); iter._opts.tablet_schema = tablet_schema; iter._opts.zonemap_always_true_pred_cols.emplace(1); EXPECT_FALSE(iter._need_read_data(1)); - iter._output_columns.emplace(2); + iter._output_column_uids.emplace(2); EXPECT_TRUE(iter._need_read_data(1)); iter._opts.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; diff --git a/be/test/storage/segment/segments_key_bounds_truncation_test.cpp b/be/test/storage/segment/segments_key_bounds_truncation_test.cpp index 10ee96e1c8e5d1..c781666dd9f98c 100644 --- a/be/test/storage/segment/segments_key_bounds_truncation_test.cpp +++ b/be/test/storage/segment/segments_key_bounds_truncation_test.cpp @@ -181,7 +181,7 @@ class SegmentsKeyBoundsTruncationTest : public testing::Test { std::vector ret; int const_value = 999; for (const auto& segment_rows : data) { - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); for (const auto& row : segment_rows) { columns[0]->insert_data(row.data(), row.size()); diff --git a/be/test/storage/tablet/tablet_schema_test.cpp b/be/test/storage/tablet/tablet_schema_test.cpp index e39c1733444453..6e1640f0b843e5 100644 --- a/be/test/storage/tablet/tablet_schema_test.cpp +++ b/be/test/storage/tablet/tablet_schema_test.cpp @@ -49,9 +49,8 @@ TEST_F(TabletSchemaTest, test_commit_tso_col_idx_from_append_column) { tablet_schema.append_column(*create_commit_tso_column(1)); EXPECT_EQ(1, tablet_schema.commit_tso_col_idx()); - std::vector column_ids {0, 1}; - Schema read_schema(tablet_schema.columns(), column_ids); - EXPECT_EQ(1, read_schema.commit_tso_col_idx()); + ReadSchema read_schema(tablet_schema.columns(), {0, 1}); + EXPECT_EQ(1, read_schema.commit_tso_ordinal()); } TEST_F(TabletSchemaTest, test_commit_tso_col_idx_from_current_index_schema) { diff --git a/be/test/storage/tablet_reader_test.cpp b/be/test/storage/tablet_reader_test.cpp index ef8e9454122c87..fe0132f1993666 100644 --- a/be/test/storage/tablet_reader_test.cpp +++ b/be/test/storage/tablet_reader_test.cpp @@ -28,6 +28,7 @@ #include "storage/delete/delete_handler.h" #include "storage/rowset/rowset_meta.h" +#include "storage/schema.h" #include "storage/tablet/tablet_schema.h" namespace doris { @@ -55,16 +56,22 @@ class TabletReaderTest : public testing::Test { // Build a DeleteHandler initialized with a single delete-predicate rowset. static void init_delete_handler(DeleteHandler& handler, const TabletSchemaSPtr& schema, - const DeletePredicatePB& delete_predicate) { + const DeletePredicatePB& delete_predicate, + ReadSchemaSPtr* read_schema) { auto rs_meta = std::make_shared(); rs_meta->set_tablet_schema(schema); rs_meta->set_version(Version(2, 2)); rs_meta->set_delete_predicate(delete_predicate); - ASSERT_TRUE(handler.init(schema, {rs_meta}, /*version=*/100).ok()); + *read_schema = std::make_shared(schema->columns()); + std::vector dropped_columns; + ASSERT_TRUE(handler.init({rs_meta}, /*version=*/100, *read_schema, dropped_columns).ok()); + for (auto& column : dropped_columns) { + (*read_schema)->append_column(std::make_shared(std::move(column))); + } } }; -// The delete columns (resolved by the delete handler to field ids) are mapped back to their +// The delete columns (resolved by the delete handler to read-schema ordinals) are mapped to their // column unique ids and stripped from all_access_paths; unrelated columns keep their paths. TEST_F(TabletReaderTest, remove_delete_columns_from_access_paths) { auto schema = create_schema({{"k1", 10}, {"k2", 11}, {"v1", 12}, {"v2", 13}, {"v4", 15}}); @@ -88,14 +95,15 @@ TEST_F(TabletReaderTest, remove_delete_columns_from_access_paths) { in1->add_values("4"); DeleteHandler handler; - init_delete_handler(handler, schema, delete_predicate); + ReadSchemaSPtr read_schema; + init_delete_handler(handler, schema, delete_predicate, &read_schema); std::map access_paths; for (int32_t uid : {10, 11, 12, 13, 15}) { access_paths[uid] = TColumnAccessPaths {}; } - TabletReader::remove_delete_columns_from_access_paths(handler, *schema, access_paths); + TabletReader::remove_delete_columns_from_access_paths(handler, *read_schema, access_paths); EXPECT_EQ(size_t(2), access_paths.size()); EXPECT_EQ(size_t(1), access_paths.count(13)) << "non-delete column must keep its access path"; @@ -117,68 +125,16 @@ TEST_F(TabletReaderTest, remove_delete_columns_keeps_unrelated_paths) { p1->set_cond_value("1"); DeleteHandler handler; - init_delete_handler(handler, schema, delete_predicate); + ReadSchemaSPtr read_schema; + init_delete_handler(handler, schema, delete_predicate, &read_schema); std::map access_paths; access_paths[12] = TColumnAccessPaths {}; access_paths[15] = TColumnAccessPaths {}; - TabletReader::remove_delete_columns_from_access_paths(handler, *schema, access_paths); + TabletReader::remove_delete_columns_from_access_paths(handler, *read_schema, access_paths); EXPECT_EQ(size_t(2), access_paths.size()); } -// Contract test for the binlog/snapshot incremental-read TSO range forwarding added to -// TabletReader::_capture_rs_readers (tablet_reader.cpp:184-186): -// _reader_context.start_tso = read_params.start_tso; -// _reader_context.end_tso = read_params.end_tso; -// Exercising _capture_rs_readers end to end would require a fully constructed Tablet + rowset -// readers (it unconditionally dereferences _tablet), which is far too heavy and brittle for a -// unit test. Instead we pin down the field contract on both sides: both must be -// std::optional defaulting to nullopt, and the forwarding must preserve the optional -// state (both set / only one / none). This guards against the fields being dropped or their -// type changed, which would silently break the forwarding. -TEST_F(TabletReaderTest, forward_tso_range_field_contract) { - // Both sides default to nullopt. - TabletReader::ReaderParams params; - EXPECT_FALSE(params.start_tso.has_value()); - EXPECT_FALSE(params.end_tso.has_value()); - - RowsetReaderContext default_ctx; - EXPECT_FALSE(default_ctx.start_tso.has_value()); - EXPECT_FALSE(default_ctx.end_tso.has_value()); - - // Equivalent of the forwarding assignments; the optional state must be preserved verbatim. - auto forward = [](const TabletReader::ReaderParams& p) { - RowsetReaderContext ctx; - ctx.start_tso = p.start_tso; - ctx.end_tso = p.end_tso; - return ctx; - }; - - // both set - params.start_tso = 100; - params.end_tso = 200; - RowsetReaderContext ctx = forward(params); - ASSERT_TRUE(ctx.start_tso.has_value()); - ASSERT_TRUE(ctx.end_tso.has_value()); - EXPECT_EQ(*ctx.start_tso, 100); - EXPECT_EQ(*ctx.end_tso, 200); - - // only start set - params.start_tso = 100; - params.end_tso = std::nullopt; - ctx = forward(params); - ASSERT_TRUE(ctx.start_tso.has_value()); - EXPECT_EQ(*ctx.start_tso, 100); - EXPECT_FALSE(ctx.end_tso.has_value()); - - // none set - params.start_tso = std::nullopt; - params.end_tso = std::nullopt; - ctx = forward(params); - EXPECT_FALSE(ctx.start_tso.has_value()); - EXPECT_FALSE(ctx.end_tso.has_value()); -} - } // namespace doris diff --git a/be/test/storage/transform/validate_stage_test.cpp b/be/test/storage/transform/validate_stage_test.cpp index 793614eedec450..ae9ef24c619df5 100644 --- a/be/test/storage/transform/validate_stage_test.cpp +++ b/be/test/storage/transform/validate_stage_test.cpp @@ -175,7 +175,7 @@ TEST_F(ValidateStageTest, V1_DirectAcceptsGoodWidth) { auto chain = build_transform_chain(c); TransformExecContext ctx = exec_ctx(schema, &c); - Block block = schema->create_block(); // full width, 0 rows + Block block = schema->create_storage_block(); // full width, 0 rows EXPECT_TRUE(chain.apply(ctx, &block).ok()); } @@ -186,7 +186,7 @@ TEST_F(ValidateStageTest, V2_DirectRejectsBadWidth) { auto chain = build_transform_chain(c); TransformExecContext ctx = exec_ctx(schema, &c); - Block block = schema->create_block_by_cids({0}); // 1 column != num_columns(3) + Block block = schema->create_storage_block({0}); // 1 column != num_columns(3) auto st = chain.apply(ctx, &block); EXPECT_FALSE(st.ok()); EXPECT_EQ(st.code(), ErrorCode::INVALID_ARGUMENT) << st; @@ -210,7 +210,7 @@ TEST_F(ValidateStageTest, V3_PartialUpdateRejectsNoTabletContext) { ctx.tablet = nullptr; // no tablet context ctx.mow_context = nullptr; - Block block = schema->create_block_by_cids({0}); + Block block = schema->create_storage_block({0}); block.get_by_position(0).column->assert_mutable()->insert_default(); auto st = chain.apply(ctx, &block); EXPECT_FALSE(st.ok()); @@ -246,7 +246,7 @@ TEST_F(ValidateStageTest, V4_PartialUpdateRejectsWithoutSegmentId) { ctx.rowset_ctx = &rwc; ctx.segment_id = -1; // add_block seam: no segment id - Block block = schema->create_block_by_cids({0}); + Block block = schema->create_storage_block({0}); IColumn* kc = block.get_by_position(0).column->assert_mutable().get(); int32_t k = 1; kc->insert_data(reinterpret_cast(&k), sizeof(int32_t)); @@ -293,7 +293,7 @@ TEST_F(ValidateStageTest, V6V7_FixedPartialUpdateRejectsBadWidth) { // V6 too wide: full width (3 == num_columns) is not a partial update block. { TransformExecContext ctx = make_ctx(); - Block block = schema->create_block(); + Block block = schema->create_storage_block(); auto st = chain.apply(ctx, &block); EXPECT_FALSE(st.ok()); EXPECT_EQ(st.code(), ErrorCode::INVALID_ARGUMENT) << st; @@ -303,7 +303,7 @@ TEST_F(ValidateStageTest, V6V7_FixedPartialUpdateRejectsBadWidth) { // V7 too narrow: fewer columns than the key (0 < 1 key column). { TransformExecContext ctx = make_ctx(); - Block block = schema->create_block_by_cids({}); + Block block = schema->create_storage_block({}); auto st = chain.apply(ctx, &block); EXPECT_FALSE(st.ok()); EXPECT_EQ(st.code(), ErrorCode::INVALID_ARGUMENT) << st; @@ -331,7 +331,7 @@ TEST_F(ValidateStageTest, V8_FlexiblePartialUpdateRejectsBadWidth) { ctx.mow_context = mow; ctx.partial_update_info = pui; - Block block = schema->create_block_by_cids({0}); // 1 col != num_columns(4) + Block block = schema->create_storage_block({0}); // 1 col != num_columns(4) auto st = chain.apply(ctx, &block); EXPECT_FALSE(st.ok()); EXPECT_EQ(st.code(), ErrorCode::INVALID_ARGUMENT) << st; @@ -359,14 +359,14 @@ TEST_F(ValidateStageTest, V9_TransientPartialUpdateValidatedAsDirect) { { TransformExecContext ctx = exec_ctx(schema, &rwc); ctx.partial_update_info = pui; - Block block = schema->create_block(); // 3 cols == num_columns + Block block = schema->create_storage_block(); // 3 cols == num_columns EXPECT_TRUE(chain.apply(ctx, &block).ok()); } // a narrow block is rejected with the non-PU width error -- not the PU one { TransformExecContext ctx = exec_ctx(schema, &rwc); ctx.partial_update_info = pui; - Block block = schema->create_block_by_cids({0}); // 1 col != num_columns(3) + Block block = schema->create_storage_block({0}); // 1 col != num_columns(3) auto st = chain.apply(ctx, &block); EXPECT_FALSE(st.ok()); EXPECT_EQ(st.code(), ErrorCode::INVALID_ARGUMENT) << st; @@ -388,7 +388,7 @@ TEST_F(ValidateStageTest, FlushSeamRejectsBeforeCreatingAWriter) { InvertedIndexFileCollection index_files; SegmentFlusher flusher(rwc, segment_files, index_files); - Block block = schema->create_block_by_cids({0}); // 1 column != num_columns(3) + Block block = schema->create_storage_block({0}); // 1 column != num_columns(3) block.get_by_position(0).column->assert_mutable()->insert_default(); auto st = flusher.flush_single_block(&block, /*segment_id=*/0); EXPECT_FALSE(st.ok()); diff --git a/be/test/storage/transform/variant_rowstore_test.cpp b/be/test/storage/transform/variant_rowstore_test.cpp index 4e4c5bd7b70756..a16de1cfcdb536 100644 --- a/be/test/storage/transform/variant_rowstore_test.cpp +++ b/be/test/storage/transform/variant_rowstore_test.cpp @@ -116,7 +116,7 @@ class VariantRowStoreTest : public MowTransformTestBase { cids.push_back(static_cast(i)); } } - Block dst = schema->create_block_by_cids(cids); + Block dst = schema->create_storage_block(cids); DataTypeSerDeSPtrs serdes = create_data_type_serdes(dst.get_data_types()); std::unordered_map col_uid_to_idx; std::vector default_values(cids.size()); @@ -145,7 +145,7 @@ TEST_F(VariantRowStoreTest, VariantParseNoVariantPassThrough) { auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(schema, &rwc); - Block block = schema->create_block(); // full width, 2 rows + Block block = schema->create_storage_block(); // full width, 2 rows IColumn* k = block.get_by_position(0).column->assert_mutable().get(); IColumn* v = block.get_by_position(1).column->assert_mutable().get(); IColumn* ds = block.get_by_position(2).column->assert_mutable().get(); @@ -181,7 +181,7 @@ TEST_F(VariantRowStoreTest, VariantParseDirectSingleRow) { auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(schema, &rwc); - Block block = schema->create_block(); + Block block = schema->create_storage_block(); int32_t k = 1; int8_t z = 0; block.get_by_position(0).column->assert_mutable()->insert_data( @@ -208,7 +208,7 @@ TEST_F(VariantRowStoreTest, VariantParseDirectMultiRow) { auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(schema, &rwc); - Block block = schema->create_block(); + Block block = schema->create_storage_block(); IColumn* k = block.get_by_position(0).column->assert_mutable().get(); IColumn* ds = block.get_by_position(2).column->assert_mutable().get(); int32_t ks[] = {1, 2}; @@ -243,7 +243,7 @@ TEST_F(VariantRowStoreTest, VariantParseEmptyBlock) { auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(schema, &rwc); - Block block = schema->create_block(); // typed columns, 0 rows + Block block = schema->create_storage_block(); // typed columns, 0 rows ASSERT_EQ(block.rows(), 0); ASSERT_TRUE(chain.apply(ctx, &block).ok()); EXPECT_EQ(block.columns(), schema->num_columns()); @@ -257,7 +257,7 @@ TEST_F(VariantRowStoreTest, VariantParseEmptyBlock) { // Builds a full-width row-store block of `rows` rows (k = i+1, v = base+10*i), // row-store column left as a placeholder default for the generator to overwrite. static Block make_row_store_block(const TabletSchemaSPtr& schema, int num_rows, int32_t base) { - Block block = schema->create_block(); + Block block = schema->create_storage_block(); IColumn* k = block.get_by_position(0).column->assert_mutable().get(); IColumn* v = block.get_by_position(1).column->assert_mutable().get(); IColumn* ds = block.get_by_position(2).column->assert_mutable().get(); @@ -282,7 +282,7 @@ TEST_F(VariantRowStoreTest, RowStoreFillRowsZeroNoGenerator) { auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(schema, &rwc); - Block block = schema->create_block(); // 0 rows + Block block = schema->create_storage_block(); // 0 rows ASSERT_EQ(block.rows(), 0); ASSERT_TRUE(chain.apply(ctx, &block).ok()); EXPECT_EQ(ctx.derived_column.second, nullptr); @@ -336,7 +336,7 @@ TEST_F(VariantRowStoreTest, RowStoreFillMaterializeContent) { TransformExecContext ctx = exec_ctx(schema, &rwc); // one row: k=42, v=-5, delete_sign=0 - Block block = schema->create_block(); + Block block = schema->create_storage_block(); int32_t k = 42; int32_t v = -5; int8_t z = 0; @@ -391,7 +391,7 @@ TEST_F(VariantRowStoreTest, RowStoreSnapshotsVariantBeforeParse) { (std::vector {"Validate", "RowStoreFill", "VariantParse"})); TransformExecContext ctx = exec_ctx(schema, &rwc); - Block block = schema->create_block(); + Block block = schema->create_storage_block(); int32_t key = 1; int8_t delete_sign = 0; block.get_by_position(0).column->assert_mutable()->insert_data( diff --git a/be/test/storage/variant/index_storage_variant_io_context_test.cpp b/be/test/storage/variant/index_storage_variant_io_context_test.cpp index fa5ea32508c490..38baa657375526 100644 --- a/be/test/storage/variant/index_storage_variant_io_context_test.cpp +++ b/be/test/storage/variant/index_storage_variant_io_context_test.cpp @@ -64,8 +64,8 @@ TEST_F(IndexStorageVariantIoContextTest, auto reader_schema = build_schema_with_variant_path_column(*tablet_schema(), kVariantUid, "hot", FieldType::OLAP_FIELD_TYPE_INT); const ColumnId path_column_id = static_cast(reader_schema->num_columns() - 1); - auto scan_schema = std::make_shared(reader_schema->columns(), - std::vector {path_column_id}); + auto scan_schema = std::make_shared(reader_schema->columns(), + std::vector {path_column_id}); TUniqueId query_id; query_id.hi = 101; diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index 6ed78e24324ea9..dacbe860637b01 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -899,7 +899,7 @@ class VariantColumnWriterReaderTest : public testing::Test { auto rowset_writer = std::move(res).value(); for (const auto& batch : batches) { - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); auto variant_col = ColumnVariant::create( _tablet_schema->column(0).variant_max_subcolumns_count(), false); @@ -940,7 +940,7 @@ class VariantColumnWriterReaderTest : public testing::Test { return Status::InvalidArgument("writer is null"); } - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); auto variant_col = ColumnVariant::create( _tablet_schema->column(0).variant_max_subcolumns_count(), false); @@ -1421,7 +1421,7 @@ class VariantColumnWriterReaderTest : public testing::Test { RETURN_IF_ERROR(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer)); RETURN_IF_ERROR(writer->init()); - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); auto scalar_variant = ColumnVariant::create(0, parent_column.variant_enable_doc_mode()); for (const auto& json : jsons) { @@ -3497,7 +3497,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { // 5. write data auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); std::unordered_map inserted_jsonstr; auto path_with_size = @@ -4121,7 +4121,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_and_read_hierarchical_doc) // 5. write doc-value-only data into variant auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); std::unordered_map inserted_jsonstr; fill_variant_column_with_doc_value_only(column_object, kRows, &inserted_jsonstr); @@ -4312,7 +4312,7 @@ TEST_F(VariantColumnWriterReaderTest, EXPECT_TRUE(writer->init().ok()); auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); std::unordered_map inserted_jsonstr; fill_variant_column_with_doc_value_only(column_object, kRows, &inserted_jsonstr); @@ -4477,7 +4477,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_materialized_v3_uses_v3_enc EXPECT_TRUE(writer->init().ok()); auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); std::unordered_map inserted_jsonstr; fill_variant_column_with_doc_value_only(column_object, kRows, &inserted_jsonstr); @@ -4586,7 +4586,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucke EXPECT_TRUE(writer->init().ok()); auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); std::unordered_map inserted_jsonstr; fill_variant_column_with_doc_value_only(column_object, kRows, &inserted_jsonstr); @@ -5232,7 +5232,7 @@ TEST_F(VariantColumnWriterReaderTest, test_storage_parse_kv_write_materialized_a R"({"hot":4,"warm":40,"cold3":103})", }; - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); auto scalar_variant = ColumnVariant::create(0, false); for (const auto& json : jsons) { @@ -5892,7 +5892,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) { // 5. write data auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); std::unordered_map inserted_jsonstr; auto path_with_size = VariantUtil::fill_object_column_with_nested_test_data(column_object, 1000, @@ -6451,7 +6451,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) { // 5. write data auto olap_data_convertor = std::make_unique(); // here is nullable variant - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); std::unordered_map inserted_jsonstr; variant_util::PathToNoneNullValues path_with_size; fill_nullable_variant_block(&block, &inserted_jsonstr, &path_with_size); @@ -6672,7 +6672,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable_without_finalize) // 5. write data auto olap_data_convertor = std::make_unique(); // here is nullable variant - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); std::unordered_map inserted_jsonstr; variant_util::PathToNoneNullValues path_with_size; fill_nullable_variant_block(&block, &inserted_jsonstr, &path_with_size); @@ -6752,7 +6752,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bm_with_finalize) { // 5. write data auto olap_data_convertor = std::make_unique(); // here is nullable variant - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); std::unordered_map inserted_jsonstr; variant_util::PathToNoneNullValues path_with_size; fill_nullable_variant_block(&block, &inserted_jsonstr, &path_with_size); @@ -6832,7 +6832,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bf_with_finalize) { // 5. write data auto olap_data_convertor = std::make_unique(); // here is nullable variant - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); std::unordered_map inserted_jsonstr; variant_util::PathToNoneNullValues path_with_size; fill_nullable_variant_block(&block, &inserted_jsonstr, &path_with_size); @@ -6914,7 +6914,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_zm_with_finalize) { // 5. write data auto olap_data_convertor = std::make_unique(); // here is nullable variant - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); std::unordered_map inserted_jsonstr; variant_util::PathToNoneNullValues path_with_size; fill_nullable_variant_block(&block, &inserted_jsonstr, &path_with_size); @@ -6996,7 +6996,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_inverted_with_finalize) { // 5. write data auto olap_data_convertor = std::make_unique(); // here is nullable variant - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); std::unordered_map inserted_jsonstr; variant_util::PathToNoneNullValues path_with_size; fill_nullable_variant_block(&block, &inserted_jsonstr, &path_with_size); @@ -7075,7 +7075,7 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) { // 5. write data auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); auto type_string = std::make_shared(); auto json_column = type_string->create_column(); @@ -7212,7 +7212,7 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) { // 5. write data auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); auto type_string = std::make_shared(); auto json_column = type_string->create_column(); @@ -7369,7 +7369,7 @@ void test_write_variant_column(StorageEngine* _engine_ref, std::string _absolute // 5. make test data for column_object auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); VariantUtil::VariantStringCreator simple_column_object = [](ColumnString* column_string, size_t size) { @@ -7786,7 +7786,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) { // 5. write data auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); variant_util::PathToNoneNullValues path_with_size; std::unordered_map inserted_jsonstr; @@ -7945,7 +7945,7 @@ TEST_F(VariantColumnWriterReaderTest, test_concurrent_load_external_meta_and_get // 5. write a small amount of data to build some subcolumns auto olap_data_convertor = std::make_unique(); - auto block = _tablet_schema->create_block(); + auto block = _tablet_schema->create_storage_block(); auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); std::unordered_map inserted_jsonstr; auto path_with_size = diff --git a/be/test/storage/variant/variant_doc_mode_compaction_test.cpp b/be/test/storage/variant/variant_doc_mode_compaction_test.cpp index f234cf8e34cc89..4c7d280b378098 100644 --- a/be/test/storage/variant/variant_doc_mode_compaction_test.cpp +++ b/be/test/storage/variant/variant_doc_mode_compaction_test.cpp @@ -250,7 +250,7 @@ class VariantDocModeCompactionTest : public ::testing::Test { EXPECT_TRUE(res.has_value()) << res.error(); auto rowset_writer = std::move(res).value(); - Block block = tablet_schema->create_block(); + Block block = tablet_schema->create_storage_block(); auto columns = std::move(block).mutate_columns(); auto* variant_col = assert_cast(columns[1].get()); auto raw_json_column = ColumnString::create(); @@ -427,11 +427,13 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen input_reader_context.tablet_schema = tablet_schema; input_reader_context.need_ordered_result = false; std::vector input_return_columns = {1}; - input_reader_context.return_columns = &input_return_columns; + auto input_read_schema = + std::make_shared(tablet_schema->columns(), input_return_columns); + input_reader_context.read_schema = input_read_schema; RowsetReaderSharedPtr input_rs_reader; create_and_init_rowset_reader(rowset.get(), input_reader_context, &input_rs_reader); - Block input_block = tablet_schema->create_block_by_cids(input_return_columns); + Block input_block = input_read_schema->create_read_block(); auto st = input_rs_reader->next_batch(&input_block); ASSERT_TRUE(st.ok()) << st.to_json(); ASSERT_EQ(1, input_block.columns()); @@ -474,14 +476,15 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; std::vector return_columns = {0}; - reader_context.return_columns = &return_columns; + auto read_schema = std::make_shared(tablet_schema->columns(), return_columns); + reader_context.read_schema = read_schema; RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); int64_t total_rows = 0; Status s = Status::OK(); while (s.ok()) { - Block output_block = tablet_schema->create_block_by_cids(return_columns); + Block output_block = read_schema->create_read_block(); s = output_rs_reader->next_batch(&output_block); if (s.ok()) { total_rows += output_block.rows(); @@ -493,4 +496,4 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen } // namespace doris -#endif \ No newline at end of file +#endif diff --git a/be/test/testutil/index_storage_test_util.cpp b/be/test/testutil/index_storage_test_util.cpp index 88dee85a2ebb53..cad64a0ce6ac0f 100644 --- a/be/test/testutil/index_storage_test_util.cpp +++ b/be/test/testutil/index_storage_test_util.cpp @@ -55,6 +55,7 @@ #include "storage/rowset/rowset_reader.h" #include "storage/rowset/rowset_writer.h" #include "storage/rowset/rowset_writer_context.h" +#include "storage/schema.h" #include "storage/segment/segment.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_meta.h" @@ -1236,7 +1237,7 @@ Result IndexStorageTestFixture::write_rowset(const IndexRowsetS tablet_options.variant_columns.size(), batch.variant_columns_by_column.size())); } - Block block = _tablet_schema->create_block(); + Block block = _tablet_schema->create_storage_block(); RETURN_RESULT_IF_ERROR( fill_block(*_tablet_schema, tablet_options, batch, &next_key, &block)); RETURN_RESULT_IF_ERROR(rowset_writer->add_block(&block)); @@ -1277,17 +1278,31 @@ Result IndexStorageTestFixture::read_rowsets( return_columns.resize(_tablet_schema->num_columns()); std::iota(return_columns.begin(), return_columns.end(), 0); } - - TabletSchemaSPtr read_schema = _tablet_schema; + TabletSchemaSPtr tablet_schema = _tablet_schema; if (options.use_variant_v2) { - read_schema = std::make_shared(*_tablet_schema); - for (int32_t column_id = 0; column_id < read_schema->num_columns(); ++column_id) { - auto& column = read_schema->mutable_column(column_id); + tablet_schema = std::make_shared(*_tablet_schema); + for (int32_t column_id = 0; column_id < tablet_schema->num_columns(); ++column_id) { + auto& column = tablet_schema->mutable_column(column_id); if (column.is_variant_type()) { column.set_variant_is_v2(true); } } } + auto read_schema = std::make_shared(tablet_schema->columns(), return_columns); + // Test specs express predicate columns as tablet cids; the read path wants + // ordinals into the read schema, so rebase them here (the role TabletReader + // plays for real queries). Predicate columns must be read columns. + std::vector> predicates; + predicates.reserve(options.predicates.size()); + for (const auto& pred : options.predicates) { + auto pos = std::find(return_columns.begin(), return_columns.end(), pred->column_id()); + if (pos == return_columns.end()) { + return ResultError(Status::InvalidArgument("predicate column {} not in return columns", + pred->column_id())); + } + auto ordinal = static_cast(std::distance(return_columns.begin(), pos)); + predicates.push_back(ordinal == pred->column_id() ? pred : pred->clone(ordinal)); + } RuntimeState runtime_state; runtime_state.set_exec_env(ExecEnv::GetInstance()); @@ -1305,10 +1320,10 @@ Result IndexStorageTestFixture::read_rowsets( RowsetReaderContext context; context.reader_type = options.reader_type; - context.tablet_schema = read_schema; + context.tablet_schema = tablet_schema; context.need_ordered_result = options.need_ordered_result; - context.return_columns = &return_columns; - context.predicates = &options.predicates; + context.read_schema = read_schema; + context.predicates = &predicates; context.stats = &result.stats; context.target_cast_type_for_variants = options.target_cast_type_for_variants; context.all_access_paths = options.all_access_paths; @@ -1319,17 +1334,17 @@ Result IndexStorageTestFixture::read_rowsets( RETURN_RESULT_IF_ERROR(reader->init(&context)); while (true) { - Block block = read_schema->create_block_by_cids(return_columns); + Block block = read_schema->create_read_block(); auto status = reader->next_batch(&block); if (status.is()) { break; } RETURN_RESULT_IF_ERROR(status); if (options.collect_string_values) { - collect_string_values_from_block(*read_schema, return_columns, block, &result); + collect_string_values_from_block(*tablet_schema, return_columns, block, &result); } if (options.collect_variant_values) { - collect_variant_values_from_block(*read_schema, return_columns, block, &result); + collect_variant_values_from_block(*tablet_schema, return_columns, block, &result); } result.rows_read += block.rows(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 09c8bc2462835e..49f1e2cdc0673b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -38,6 +38,8 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTableWrapper; +import org.apache.doris.catalog.RowBinlogTableWrapper; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.common.util.Util; @@ -213,6 +215,7 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.StatisticConstants; import org.apache.doris.tablefunction.TableValuedFunctionIf; +import org.apache.doris.thrift.TBinlogScanType; import org.apache.doris.thrift.TPartitionType; import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TResultSinkType; @@ -2978,6 +2981,11 @@ public PlanFragment visitPhysicalLazyMaterializeOlapScan(PhysicalLazyMaterialize .map(context::findSlotRef).filter(Objects::nonNull).map(SlotRef::getSlotId) .collect(Collectors.toSet()); + olapScanNode.updateRequiredSlots(context, scanIds); + // The deferred fetch uses one row-id, so key-only eager output must still select the + // winning physical row by the table's sequence column. + preserveStorageSemanticSlots(olapScanNode, scanIds, true); + preserveExtraStorageKeySlots(olapScanNode, scanIds); olapScanNode.getTupleDesc().getSlots().removeIf(slot -> !scanIds.contains(slot.getId())); context.createSlotDesc(olapScanNode.getTupleDesc(), lazyScan.getRowId()); for (Slot slot : lazyScan.getOutput()) { @@ -3061,6 +3069,7 @@ private void updateScanSlotsMaterialization(ScanNode scanNode, requiredWithVirtualColumns.addAll(virtualColumnInputSlotIds); } if (scanNode instanceof OlapScanNode) { + preserveStorageSemanticSlots((OlapScanNode) scanNode, requiredWithVirtualColumns, false); preserveExtraStorageKeySlots((OlapScanNode) scanNode, requiredWithVirtualColumns); } else if (scanNode instanceof PluginDrivenScanNode) { preserveConnectorMustReadSlots((PluginDrivenScanNode) scanNode, requiredWithVirtualColumns); @@ -3138,13 +3147,146 @@ static void preserveConnectorMustReadSlots(PluginDrivenScanNode scanNode, Set requiredSlotIds, + boolean requireSequenceForLazyMaterialization) { + if (scanNode.getOlapTable() instanceof RowBinlogTableWrapper + && scanNode.getScanParams() != null + && scanNode.getScanParams().incrementalRead()) { + preserveRowBinlogSemanticSlots(scanNode, requiredSlotIds); + return; + } + if (scanNode.getOlapTable() instanceof OlapTableWrapper + && !(scanNode.getOlapTable() instanceof RowBinlogTableWrapper)) { + preserveSnapshotCommitTsoSlot(scanNode, requiredSlotIds); + } + preserveMergeSequenceSlots(scanNode, requiredSlotIds, requireSequenceForLazyMaterialization); + } + + private void preserveSnapshotCommitTsoSlot(OlapScanNode scanNode, Set requiredSlotIds) { + SlotDescriptor commitTsoSlot = scanNode.getTupleDesc().getSlots().stream() + .filter(slot -> slot.getColumn() != null + && slot.getColumn().getName().equals(Column.COMMIT_TSO_COL)) + .findFirst() + .orElse(null); + Preconditions.checkState(commitTsoSlot != null, + "commit TSO slot is missing from snapshot scan tuple"); + preserveStorageSlot(commitTsoSlot, requiredSlotIds); + } + + private void preserveRowBinlogSemanticSlots(OlapScanNode scanNode, Set requiredSlotIds) { + RowBinlogTableWrapper wrapper = (RowBinlogTableWrapper) scanNode.getOlapTable(); + TBinlogScanType scanType = OlapScanNode.parseBinlogScanType( + scanNode.getScanParams(), wrapper.getOriginTable()); + if (scanType == TBinlogScanType.NONE) { + return; + } + + List scanSlots = scanNode.getTupleDesc().getSlots(); + Map slotByName = scanSlots.stream() + .filter(slot -> slot.getColumn() != null) + .collect(Collectors.toMap( + slot -> slot.getColumn().getName(), slot -> slot, (left, right) -> left)); + + preserveStorageSlot(slotByName.get(Column.BINLOG_TSO_COL), requiredSlotIds); + preserveStorageSlot(slotByName.get(Column.BINLOG_OPERATION_COL), requiredSlotIds); + if (scanType == TBinlogScanType.APPEND_ONLY) { + return; + } + + Set requestedSlots = Sets.newHashSet(requiredSlotIds); + for (SlotDescriptor slot : scanSlots) { + Column column = slot.getColumn(); + if (column != null && column.isKey()) { + preserveStorageSlot(slot, requiredSlotIds); + } + } + + if (!wrapper.getOriginTable().getBinlogConfig().getNeedHistoricalValue()) { + // Row-binlog schemas without historical values have no before-image columns, so there + // are no before-image storage dependencies to preserve. + return; + } + for (SlotDescriptor slot : scanSlots) { + Column column = slot.getColumn(); + if (column == null || column.isKey() || !requestedSlots.contains(slot.getId()) + || isRowBinlogInternalColumn(column)) { + continue; + } + preserveStorageSlot(slotByName.get(Column.generateBeforeColName(column.getName())), + requiredSlotIds); + } + } + + private boolean isRowBinlogInternalColumn(Column column) { + String columnName = column.getName(); + return columnName.startsWith(Column.BINLOG_BEFORE_PREFIX) + || columnName.equals(Column.BINLOG_LSN_COL) + || columnName.equals(Column.BINLOG_OPERATION_COL) + || columnName.equals(Column.BINLOG_TSO_COL); + } + + private void preserveMergeSequenceSlots(OlapScanNode scanNode, Set requiredSlotIds, + boolean requireSequenceForLazyMaterialization) { + if (!shouldPreserveStorageKeySlots(scanNode)) { + return; + } + + List scanSlots = scanNode.getTupleDesc().getSlots(); + Set sequenceColumnNames = Sets.newHashSet(); + boolean requiresSequenceColumn = requireSequenceForLazyMaterialization; + Map> columnSeqMapping = getStorageSemanticTable(scanNode).getColumnSeqMapping(); + for (SlotDescriptor slot : scanSlots) { + Column column = slot.getColumn(); + if (column == null || column.isKey() || column.isSequenceColumn() + || !requiredSlotIds.contains(slot.getId())) { + continue; + } + requiresSequenceColumn = true; + for (Map.Entry> entry : columnSeqMapping.entrySet()) { + if (entry.getValue().stream().anyMatch(column.getName()::equalsIgnoreCase)) { + sequenceColumnNames.add(entry.getKey()); + } + } + } + + for (SlotDescriptor slot : scanSlots) { + Column column = slot.getColumn(); + if (column == null) { + continue; + } + if ((requiresSequenceColumn && column.isSequenceColumn()) + || sequenceColumnNames.stream().anyMatch(column.getName()::equalsIgnoreCase)) { + preserveStorageSlot(slot, requiredSlotIds); + } + } + } + + private void preserveStorageSlot(SlotDescriptor slot, Set requiredSlotIds) { + Preconditions.checkNotNull(slot, "storage semantic dependency slot is missing"); + requiredSlotIds.add(slot.getId()); + } + private boolean shouldPreserveStorageKeySlots(OlapScanNode scanNode) { + OlapTable storageSemanticTable = getStorageSemanticTable(scanNode); long selectedIndexId = scanNode.getSelectedIndexId() == -1 ? scanNode.getOlapTable().getBaseIndexId() : scanNode.getSelectedIndexId(); KeysType keysType = scanNode.getOlapTable().getIndexMetaByIndexId(selectedIndexId).getKeysType(); return keysType == KeysType.AGG_KEYS - || (keysType == KeysType.UNIQUE_KEYS && !scanNode.getOlapTable().getEnableUniqueKeyMergeOnWrite()); + || (keysType == KeysType.UNIQUE_KEYS + && !storageSemanticTable.getEnableUniqueKeyMergeOnWrite()); + } + + private OlapTable getStorageSemanticTable(OlapScanNode scanNode) { + OlapTable scanTable = scanNode.getOlapTable(); + return scanTable instanceof OlapTableWrapper + ? ((OlapTableWrapper) scanTable).getOriginTable() + : scanTable; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java index 3a99d0b4ada5df..b112595dcefd62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java @@ -81,7 +81,7 @@ public ProbeContext(SlotReference slot, Set requiredMaterializedSlots) { public Optional visitPhysicalFilter(PhysicalFilter filter, ProbeContext context) { if (SessionVariable.getTopNLazyMaterializationUsingIndex() && filter.child() instanceof PhysicalOlapScan) { - // agg table / non-light-schema-change table do not support lazy materialize + // Reject OLAP tables whose storage semantics cannot be reconstructed from one row-id. OlapTable table = ((PhysicalOlapScan) filter.child()).getTable(); if (!supportOlapTopnLazyMaterialize(table)) { return Optional.empty(); @@ -138,7 +138,7 @@ boolean checkRelationTableSupportedType(PhysicalCatalogRelation relation) { /** * Whether an OLAP table can perform topn lazy materialization. * - *

Two hard requirements: + *

Three hard requirements: *

    *
  • Not an AGG_KEYS table: aggregate tables cannot locate a single source row for a value.
    • *
  • light_schema_change is enabled: lazy materialization appends a synthetic global row-id @@ -148,6 +148,8 @@ boolean checkRelationTableSupportedType(PhysicalCatalogRelation relation) { * synthetic row-id column. That path either fails with "field name is invalid" during the * scan or silently returns NULL for the lazily-fetched columns. Disable the optimization * for such tables and fall back to normal topn.
    • + *
  • No sequence-map: different value groups can come from different physical rows, while + * topn lazy materialization keeps only one row-id for each relation.
    • *
*/ private boolean supportOlapTopnLazyMaterialize(OlapTable table) { @@ -157,6 +159,9 @@ private boolean supportOlapTopnLazyMaterialize(OlapTable table) { if (!table.getEnableLightSchemaChange()) { return false; } + if (table.hasColumnSeqMapping()) { + return false; + } return true; } @@ -179,7 +184,7 @@ public Optional visitPhysicalOlapScan(PhysicalOlapScan scan, if (scan.getSelectedIndexId() != scan.getTable().getBaseIndexId()) { return Optional.empty(); } - // agg table / non-light-schema-change table do not support lazy materialize + // Reject OLAP tables whose storage semantics cannot be reconstructed from one row-id. OlapTable table = scan.getTable(); if (!supportOlapTopnLazyMaterialize(table)) { return Optional.empty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 94bed913bcb8a6..c14ecb72c26921 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -804,17 +804,13 @@ public List computeOutput() { return getOutputByIndex(selectedIndexId); } List baseSchema = table.getBaseSchema(true); - boolean skipBinlogBeforeColumn = scanParams.isPresent() && scanParams.get().incrementalRead(); - List slotFromColumn = createSlotsVectorized(baseSchema, skipBinlogBeforeColumn); + List slotFromColumn = createSlotsVectorized(baseSchema); Builder slots = ImmutableList.builder(); IdGenerator exprIdGenerator = StatementScopeIdGenerator.getExprIdGenerator(); for (int i = 0; i < baseSchema.size(); i++) { final int index = i; Column col = baseSchema.get(i); - if (skipBinlogBeforeColumn && col.getName().startsWith(Column.BINLOG_BEFORE_PREFIX)) { - continue; - } Pair key = Pair.of(selectedIndexId, col.getName()); Slot slot = cacheSlotWithSlotName.computeIfAbsent(key, k -> slotFromColumn.get(index)); slots.add(slot); @@ -932,22 +928,25 @@ public Optional getScoreRangeInfo() { return scoreRangeInfo; } - private List createSlotsVectorized(List columns, boolean skipBinlogBeforeColumn) { + protected List createSlotsVectorized(List columns) { List qualified = qualified(); SlotReference[] slots = new SlotReference[columns.size()]; IdGenerator exprIdGenerator = StatementScopeIdGenerator.getExprIdGenerator(); for (int i = 0; i < columns.size(); i++) { - if (skipBinlogBeforeColumn && columns.get(i).getName().startsWith(Column.BINLOG_BEFORE_PREFIX)) { - continue; - } ExprId nextId = exprIdGenerator.getNextId(); - slots[i] = SlotReference.fromColumn(nextId, table, columns.get(i), qualified); + slots[i] = SlotReference.fromColumn(nextId, table, getOutputColumn(columns.get(i)), qualified); } return Arrays.asList(slots); } - protected List createSlotsVectorized(List columns) { - return createSlotsVectorized(columns, false); + private Column getOutputColumn(Column column) { + if (scanParams.isPresent() && scanParams.get().incrementalRead() + && column.getName().startsWith(Column.BINLOG_BEFORE_PREFIX)) { + Column outputColumn = new Column(column); + outputColumn.setIsVisible(false); + return outputColumn; + } + return column; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index d17c8b7dd63ba2..75e8a6e8250664 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1756,6 +1756,10 @@ public void setScanParams(TableScanParams scanParams) { this.scanParams = scanParams; } + public TableScanParams getScanParams() { + return scanParams; + } + public long getIncrementalScanEndTime() { if (scanParams != null && scanParams.incrementalRead() && scanParams.getMapParams().containsKey(OLAP_END_TIMESTAMP)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index 918f9313c2881d..eb885887ded140 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -19,13 +19,25 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.GroupingInfo; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.SortInfo; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTableWrapper; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.RowBinlogTableWrapper; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.properties.DataTrait; @@ -48,6 +60,7 @@ import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.planner.AggregationNode; +import org.apache.doris.planner.MaterializationNode; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PartitionSortNode; import org.apache.doris.planner.PlanFragment; @@ -83,8 +96,13 @@ public class PhysicalPlanTranslatorTest extends TestWithFeService { + private boolean originalEnableFeatureBinlog; + private boolean originalEnableTableStream; + @Override protected void runBeforeAll() throws Exception { + originalEnableFeatureBinlog = Config.enable_feature_binlog; + originalEnableTableStream = Config.enable_table_stream; createDatabase("test_db"); createTable("create table test_db.t(a int, b int) distributed by hash(a) buckets 3 " + "properties('replication_num' = '1');"); @@ -98,9 +116,45 @@ protected void runBeforeAll() throws Exception { + ")\n" + "distributed by hash(k1) buckets 3\n" + "properties('replication_num' = '1');"); + Config.enable_feature_binlog = true; + Config.enable_table_stream = true; + createTable("create table test_db.binlog_scan_schema_t(\n" + + "k1 int, k2 int, v1 int, v2 int)\n" + + "unique key(k1, k2)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1'," + + "'enable_unique_key_merge_on_write' = 'true'," + + "'binlog.enable' = 'true','binlog.format' = 'ROW'," + + "'binlog.need_historical_value' = 'true');"); + createTable("create table test_db.sequence_scan_schema_t(\n" + + "k1 int, k2 int, v1 int, v2 int)\n" + + "unique key(k1, k2)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1'," + + "'enable_unique_key_merge_on_write' = 'false'," + + "'function_column.sequence_type' = 'int');"); + createTable("create table test_db.sequence_map_scan_schema_t(\n" + + "k1 bigint, k2 int, v1 int, v2 int, s1 bigint, s2 bigint)\n" + + "unique key(k1, k2)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1'," + + "'enable_unique_key_merge_on_write' = 'false'," + + "'sequence_mapping.s1' = 'v1'," + + "'sequence_mapping.s2' = 'v2');"); + createTable("create stream test_db.binlog_scan_schema_stream " + + "on table test_db.binlog_scan_schema_t properties('type' = 'append_only')"); + Database database = (Database) Env.getCurrentInternalCatalog().getDbOrMetaException("test_db"); + bumpPartitionsAndReplicas( + (OlapTable) database.getTableOrMetaException("binlog_scan_schema_t"), 2L); connectContext.getSessionVariable().setDisableNereidsRules("prune_empty_partition"); } + @Override + protected void runAfterAll() { + Config.enable_feature_binlog = originalEnableFeatureBinlog; + Config.enable_table_stream = originalEnableTableStream; + } + @Test public void testOlapPrune() throws Exception { LogicalProperties placeHolder = Mockito.mock(LogicalProperties.class); @@ -173,6 +227,127 @@ public DataTrait get() { thriftScanNode.olap_scan_node.getExtraKeyColumnSlotIds()); } + @Test + public void testBinlogPhysicalScanSchemaDependencies() throws Exception { + OlapScanNode scanNode = getFirstOlapScanNode( + "select v1 from test_db.binlog_scan_schema_t" + + "@incr(\"incrementType\" = \"MIN_DELTA\")"); + List scanColumns = scanNode.getTupleDesc().getSlots().stream() + .map(slot -> slot.getColumn().getName()) + .collect(Collectors.toList()); + + Assertions.assertTrue(scanColumns.containsAll(ImmutableList.of( + "k1", "k2", "v1", Column.generateBeforeColName("v1"), + Column.BINLOG_OPERATION_COL, Column.BINLOG_TSO_COL))); + Assertions.assertFalse(scanColumns.contains("v2")); + Assertions.assertFalse(scanColumns.contains(Column.generateBeforeColName("v2"))); + Assertions.assertFalse(scanColumns.contains(Column.BINLOG_LSN_COL)); + + Assertions.assertTrue(scanNode.getExtraKeyColumnSlotIds().isEmpty()); + Assertions.assertEquals(ImmutableList.of("v1"), scanNode.getOutputTupleDesc().getSlots().stream() + .map(slot -> slot.getColumn().getName()) + .collect(Collectors.toList())); + + TPlanNode thriftScanNode = scanNode.treeToThrift().getNodes().get(0); + Set dependencyUniqueIds = scanNode.getTupleDesc().getSlots().stream() + .filter(slot -> !slot.getColumn().getName().equals("v1")) + .map(slot -> slot.getColumn().getUniqueId()) + .collect(Collectors.toSet()); + Assertions.assertTrue(dependencyUniqueIds.stream() + .noneMatch(thriftScanNode.olap_scan_node.getOutputColumnUniqueIds()::contains)); + } + + @Test + public void testAppendOnlyBinlogPhysicalScanSchemaDependencies() throws Exception { + OlapScanNode scanNode = getFirstOlapScanNode( + "select v1 from test_db.binlog_scan_schema_t" + + "@incr(\"incrementType\" = \"APPEND_ONLY\")"); + Set scanColumns = scanNode.getTupleDesc().getSlots().stream() + .map(slot -> slot.getColumn().getName()) + .collect(Collectors.toSet()); + Assertions.assertEquals(ImmutableSet.of( + "v1", Column.BINLOG_OPERATION_COL, Column.BINLOG_TSO_COL), scanColumns); + + Assertions.assertTrue(scanNode.getExtraKeyColumnSlotIds().isEmpty()); + } + + @Test + public void testMergeSequencePhysicalScanSchemaDependencies() throws Exception { + OlapScanNode sequenceScan = getFirstOlapScanNode( + "select v1 from test_db.sequence_scan_schema_t"); + Assertions.assertTrue(sequenceScan.getTupleDesc().getSlots().stream() + .anyMatch(slot -> slot.getColumn().getName().equals(Column.SEQUENCE_COL))); + + OlapScanNode sequenceMapScan = getFirstOlapScanNode( + "select v1 from test_db.sequence_map_scan_schema_t"); + Set sequenceMapScanColumns = sequenceMapScan.getTupleDesc().getSlots().stream() + .map(slot -> slot.getColumn().getName()) + .collect(Collectors.toSet()); + Assertions.assertTrue(sequenceMapScanColumns.containsAll(ImmutableSet.of("k1", "k2", "v1", "s1"))); + Assertions.assertFalse(sequenceMapScanColumns.contains("v2")); + Assertions.assertFalse(sequenceMapScanColumns.contains("s2")); + } + + @Test + public void testSnapshotPhysicalScanSchemaIncludesCommitTso() throws Exception { + OlapScanNode snapshotMowScan = getOlapScanNodes( + "select v1 + 1 as projected_v1 from test_db.binlog_scan_schema_stream@snapshot()").stream() + .filter(scan -> scan.getOlapTable() instanceof OlapTableWrapper + && !(scan.getOlapTable() instanceof RowBinlogTableWrapper)) + .findFirst() + .orElseThrow(() -> new AssertionError("snapshot base scan not found")); + Set physicalSlotIds = snapshotMowScan.getTupleDesc().getSlots().stream() + .map(SlotDescriptor::getId) + .collect(Collectors.toSet()); + Assertions.assertTrue(snapshotMowScan.getTupleDesc().getSlots().stream() + .map(slot -> slot.getColumn().getName()) + .anyMatch(Column.COMMIT_TSO_COL::equals)); + Assertions.assertTrue(snapshotMowScan.getOutputTupleDesc().getSlots().stream() + .noneMatch(slot -> slot.getColumn() != null + && slot.getColumn().getName().equals(Column.COMMIT_TSO_COL))); + + Set projectionInputSlotIds = Sets.newHashSet(); + for (Expr projection : snapshotMowScan.getProjectList()) { + Expr.extractSlots(projection, projectionInputSlotIds); + } + Assertions.assertTrue(physicalSlotIds.containsAll(projectionInputSlotIds)); + } + + @Test + public void testLazyMaterializedScanPreservesStorageDependencies() throws Exception { + int originalThreshold = connectContext.getSessionVariable().topNLazyMaterializationThreshold; + connectContext.getSessionVariable().topNLazyMaterializationThreshold = 1024; + try { + Planner planner = getSQLPlanner( + "select v2 from test_db.sequence_scan_schema_t order by k1 limit 1"); + Set materializationNodes = Sets.newHashSet(); + Set scanNodes = Sets.newHashSet(); + for (PlanFragment fragment : planner.getFragments()) { + PlanNode root = fragment.getPlanRoot(); + if (root != null) { + root.collect(MaterializationNode.class, materializationNodes); + root.collect(OlapScanNode.class, scanNodes); + } + } + + Assertions.assertEquals(1, materializationNodes.size()); + Assertions.assertEquals(1, scanNodes.size()); + OlapScanNode scanNode = scanNodes.iterator().next(); + Assertions.assertTrue(scanNode.getTupleDesc().getSlots().stream() + .noneMatch(slot -> slot.getColumn().getName().equals("v2"))); + Set dependencyUniqueIds = scanNode.getTupleDesc().getSlots().stream() + .filter(slot -> slot.getColumn().getName().equals(Column.SEQUENCE_COL)) + .map(slot -> slot.getColumn().getUniqueId()) + .collect(Collectors.toSet()); + Assertions.assertEquals(1, dependencyUniqueIds.size()); + TPlanNode thriftScanNode = scanNode.treeToThrift().getNodes().get(0); + Assertions.assertTrue(dependencyUniqueIds.stream() + .noneMatch(thriftScanNode.olap_scan_node.getOutputColumnUniqueIds()::contains)); + } finally { + connectContext.getSessionVariable().topNLazyMaterializationThreshold = originalThreshold; + } + } + @Test public void testAggNeedsFinalize() throws Exception { String querySql = "select b from test_db.t group by b"; @@ -347,6 +522,10 @@ public void testPartitionTopNPrunesPartitionKeyFromSortInfo() throws Exception { } private OlapScanNode getFirstOlapScanNode(String sql) throws Exception { + return getOlapScanNodes(sql).get(0); + } + + private List getOlapScanNodes(String sql) throws Exception { Planner planner = getSQLPlanner(sql); Assertions.assertNotNull(planner); List scanNodes = new ArrayList<>(); @@ -357,7 +536,22 @@ private OlapScanNode getFirstOlapScanNode(String sql) throws Exception { } } Assertions.assertFalse(scanNodes.isEmpty()); - return scanNodes.get(0); + return scanNodes; + } + + private static void bumpPartitionsAndReplicas(OlapTable table, long newVersion) { + for (Partition partition : table.getPartitions()) { + long timestamp = System.currentTimeMillis(); + partition.setVisibleVersionAndTime(newVersion, timestamp, timestamp); + partition.setNextVersion(newVersion + 1); + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE, true)) { + for (Tablet tablet : index.getTablets()) { + for (Replica replica : tablet.getReplicas()) { + replica.updateVersion(newVersion); + } + } + } + } } private static final class TestScanNode extends ScanNode { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java index 0f0dc6775f8f28..c2fca54dd6ee92 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java @@ -60,6 +60,19 @@ public void testOlapScanRejectsRequiredMaterializedSlots() { Assertions.assertFalse(source.isPresent()); } + @Test + public void testOlapScanRejectsSequenceMapTable() { + SlotReference baseSlot = new SlotReference("a", IntegerType.INSTANCE); + PhysicalOlapScan scan = mockBaseOlapScan(baseSlot); + Mockito.when(scan.getTable().hasColumnSeqMapping()).thenReturn(true); + + MaterializeProbeVisitor.ProbeContext context = new MaterializeProbeVisitor.ProbeContext(baseSlot); + Optional source = + new MaterializeProbeVisitor().visitPhysicalOlapScan(scan, context); + + Assertions.assertFalse(source.isPresent()); + } + @Test public void testOlapScanUsesRelationSlotWithAccessPaths() { SlotReference contextSlot = new SlotReference("a", IntegerType.INSTANCE); diff --git a/regression-test/data/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.out b/regression-test/data/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.out new file mode 100644 index 00000000000000..65e98091bdab12 --- /dev/null +++ b/regression-test/data/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.out @@ -0,0 +1,73 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dict_default_session -- +1 s1 +2 s2 +3 s3 +4 s4 +5 s5 +6 s6 +7 s7 +8 s8 +9 s9 +10 s10 + +-- !agg_default_session -- +1 s1 +2 s2 +3 s3 +4 s4 +5 s5 +6 s6 +7 s7 +8 s8 +9 s9 +10 s10 + +-- !agg_second_rowset -- +s1501 +s1502 +s1503 +s1504 +s1505 +s1506 +s1507 +s1508 +s1509 +s1510 + +-- !dict_inline_topn -- +1 s1 +2 s2 +3 s3 +4 s4 +5 s5 +6 s6 +7 s7 +8 s8 +9 s9 +10 s10 + +-- !dict_inline_topn_single_child -- +s1 +s2 +s3 +s4 +s5 +s6 +s7 +s8 +s9 +s10 + +-- !int_inline_topn_last_child -- +3000001 +3000002 +3000003 +3000004 +3000005 +3000006 +3000007 +3000008 +3000009 +3000010 + diff --git a/regression-test/data/row_binlog_p0/test_binlog_changes_syntax.out b/regression-test/data/row_binlog_p0/test_binlog_changes_syntax.out index 5f056221642ba8..801a324ecebb45 100644 --- a/regression-test/data/row_binlog_p0/test_binlog_changes_syntax.out +++ b/regression-test/data/row_binlog_p0/test_binlog_changes_syntax.out @@ -69,6 +69,12 @@ 4 40 d 0 5 50 e 0 +-- !mow_append_value_only -- +20 +21 +40 +50 + -- !mow_min_delta_range -- 1 10 a 2 1 13 a3 3 @@ -76,6 +82,20 @@ 3 30 c 1 4 40 d 0 +-- !mow_min_delta_value_only -- +10 +13 +21 +30 +40 + +-- !mow_min_delta_select_star -- +1 10 a +1 13 a3 +2 21 b1 +3 30 c +4 40 d + -- !mow_detail_range -- 1 10 a 2 1 11 a1 2 @@ -91,6 +111,21 @@ 5 50 e 0 5 50 e 1 +-- !mow_detail_value_only -- +10 +11 +11 +12 +12 +13 +20 +20 +21 +30 +40 +50 +50 + -- !mow_detail_start -- 1 10 2 1 11 2 diff --git a/regression-test/data/schema_change/test_alter_table_column_with_delete_drop_column_dup_key.out b/regression-test/data/schema_change/test_alter_table_column_with_delete_drop_column_dup_key.out index 963c2f9820100e..b246e0c4b41e7f 100644 --- a/regression-test/data/schema_change/test_alter_table_column_with_delete_drop_column_dup_key.out +++ b/regression-test/data/schema_change/test_alter_table_column_with_delete_drop_column_dup_key.out @@ -21,7 +21,7 @@ 5 5 5 B -- !sql -- -2 2 2 2 +20 200 2000 2 -- !sql -- 3 3 3 3 @@ -32,14 +32,31 @@ -- !sql -- 4 4 4 A +-- !project_other_after_delete_drop_readd -- +1 +3 +4 + -- !sql -- 1 1 1 A 3 3 3 A 4 4 4 A 5 5 5 B +-- !project_other_after_delete_drop_readd_drop_again -- +1 +3 +4 +5 + -- !sql -- -4 A 4 4 A +4 A 4 4 + +-- !project_other_after_delete_drop_readd_drop_again_full_schema_change -- +1 +3 +4 +5 -- !sql -- 1 1 1 1 diff --git a/regression-test/data/unique_seq_map_p0/test_unique_seq_map_hidden_dependency.out b/regression-test/data/unique_seq_map_p0/test_unique_seq_map_hidden_dependency.out new file mode 100644 index 00000000000000..8e66a7b8f53ced --- /dev/null +++ b/regression-test/data/unique_seq_map_p0/test_unique_seq_map_hidden_dependency.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !hidden_s1 -- +20 +70 + +-- !hidden_s2 -- +300 +600 + +-- !topn_sequence_map_by_value -- +70 600 + +-- !topn_sequence_map_by_key -- +2 70 600 diff --git a/regression-test/suites/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.groovy b/regression-test/suites/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.groovy new file mode 100644 index 00000000000000..e00662c46eb095 --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.groovy @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Regression case for the pruned-struct read misalignment on the inline topn scan path. +// +// Root cause: when a topn ordered read (order by key limit N) hits a rowset with +// multiple OVERLAPPING segments, BetaRowsetReader picks VMergeIterator, whose +// VMergeIteratorContext::init() eagerly loads the first block of every segment. +// That block is created from the storage Schema (FULL struct type, all children), +// while the struct sub-column iterators have been filtered down to the pruned +// subset by set_access_paths()/remove_pruned_sub_iterators(). The pairing loop in +// StructFileColumnIterator::next_batch() indexes _sub_column_iterators[i] by the +// dst tuple position, so the two ordinal spaces disagree: +// - text/dict child data decoded into a TINYINT child column +// -> [E-3110] insert_many_dict_data is not supported for TINYINT +// - same-typed children shift silently, and the loop runs past the pruned +// iterator vector -> out-of-bounds -> BE SIGSEGV +// +// The FE TopN lazy materialization (VMaterializeNode) usually shields this path +// on master; setting topn_lazy_materialization_threshold=-1 (or any state where +// the rule bails, e.g. light_schema_change=false tables) exposes it. +// +// Layout construction: MemTable.need_flush debug point forces one segment per +// inserted block (batch_size=500, 2000 rows -> 4 segments, below the +// segcompaction_batch_size=10 threshold so segcompaction never merges them), and +// the NULL k1 rows in every block make every segment's key lower bound NULL, so +// the segments overlap and the rowset is marked OVERLAPPING. +suite("test_topn_pruned_struct_overlapping_segments", "nonConcurrent") { + def dictTable = "test_topn_pruned_struct_ovlp_dict" + def intTable = "test_topn_pruned_struct_ovlp_int" + def aggTable = "test_topn_pruned_struct_ovlp_agg" + + sql "DROP TABLE IF EXISTS ${dictTable}" + sql "DROP TABLE IF EXISTS ${intTable}" + sql "DROP TABLE IF EXISTS ${aggTable}" + sql """ + CREATE TABLE ${dictTable} ( + k1 BIGINT, + c ARRAY>> + ) DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + CREATE TABLE ${intTable} ( + k1 BIGINT, + c ARRAY>> + ) DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + + try { + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush") + // 4 blocks of 500 rows -> 4 segments per tablet; k1 NULL every 20 rows so + // every segment starts at NULL -> overlapping key ranges -> OVERLAPPING rowset. + sql "SET batch_size = 500" + sql """ + INSERT INTO ${dictTable} + SELECT if(number % 20 = 0, NULL, number), + array(array(named_struct( + 'col1', CAST(number AS INT), + 'col2', CAST(number % 127 AS TINYINT), + 'col17', concat('s', number)))) + FROM numbers('number' = '2000') + """ + sql """ + INSERT INTO ${intTable} + SELECT if(number % 20 = 0, NULL, number), + array(array(named_struct( + 'col1', CAST(number + 1000000 AS INT), + 'col2', CAST(number + 2000000 AS INT), + 'col3', CAST(number + 3000000 AS INT)))) + FROM numbers('number' = '2000') + """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush") + } + + // Self-check the storage layout: the test is void unless the rowset really is + // a multi-segment OVERLAPPING one. BetaRowsetReader::is_merge_iterator() needs + // both the OVERLAPPING flag and num_segments > 1, so assert both. + for (def tbl : [dictTable, intTable]) { + def tablets = sql_return_maparray("SHOW TABLETS FROM ${tbl}") + def (code, out, err) = curl("GET", tablets[0].MetaUrl) + assertEquals(0, code) + def jsonMeta = parseJson(out.trim()) + def hasMergeLayout = jsonMeta.rs_metas.any { meta -> + meta.segments_overlap_pb == "OVERLAPPING" && meta.num_segments > 1 + } + assertTrue(hasMergeLayout, + "expected an OVERLAPPING rowset with num_segments > 1 in ${tbl}, " + + "check the MemTable.need_flush debug point") + } + + // AGG-table variant: the same ordinal-space mismatch also hits the aggregate + // merge path (BlockReader::_init_agg_state used to build the reader_replace + // argument type from the full TabletColumn while the stored block uses the + // pruned type). It needs no debug point, no topn and no special session vars: + // two ordinary inserts (two rowsets) + a pruned read used to fail with + // "Aggregate function reader_replace argument 0 type check failed". + sql """ + CREATE TABLE ${aggTable} ( + k1 BIGINT, + c ARRAY>> REPLACE + ) AGGREGATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + INSERT INTO ${aggTable} + SELECT number, array(array(named_struct( + 'col1', CAST(number AS INT), + 'col2', CAST(number % 127 AS TINYINT), + 'col17', concat('s', number)))) + FROM numbers('number' = '1000') + """ + sql """ + INSERT INTO ${aggTable} + SELECT number + 1000, array(array(named_struct( + 'col1', CAST(number + 1000 AS INT), + 'col2', CAST(number % 127 AS TINYINT), + 'col17', concat('s', number + 1000)))) + FROM numbers('number' = '1000') + """ + + // 1. Default session: on master this plans VMaterializeNode (TopN lazy + // materialization + rowid fetch); guards that path stays correct too. + // Expected: k1 1..10 -> col1 = k1, col17 = 's' + k1. + qt_dict_default_session """ + SELECT element_at(c[1][1], 1), element_at(c[1][1], 'col17') + FROM ${dictTable} WHERE k1 IS NOT NULL ORDER BY k1 LIMIT 10 + """ + + // AGG merge path, pure default session (no topn involved at all). + qt_agg_default_session """ + SELECT element_at(c[1][1], 1), element_at(c[1][1], 'col17') + FROM ${aggTable} WHERE k1 >= 1 AND k1 <= 10 ORDER BY k1 + """ + // Rows from the second rowset must merge and read correctly too. + qt_agg_second_rowset """ + SELECT element_at(c[1][1], 'col17') + FROM ${aggTable} WHERE k1 >= 1501 AND k1 <= 1510 ORDER BY k1 + """ + + // 2. Force the inline topn scan path (no VMaterializeNode). This is the exact + // path that used to fail with + // "[E-3110] Method insert_many_dict_data is not supported for TINYINT". + sql "SET topn_lazy_materialization_threshold = -1" + // The storage ordered-key read (read_orderby_key -> need_ordered_result -> + // VMergeIterator) is additionally gated by enable_segment_limit_pushdown in + // OlapScanner. The variable is randomized in fuzzy regression runs; pin it so + // this section always exercises the merge path this fix targets. + sql "SET enable_segment_limit_pushdown = true" + + qt_dict_inline_topn """ + SELECT element_at(c[1][1], 1), element_at(c[1][1], 'col17') + FROM ${dictTable} WHERE k1 IS NOT NULL ORDER BY k1 LIMIT 10 + """ + + // Single pruned child: the misalignment shifts col17 data onto col1 (INT), + // which used to fail with "... is not supported for INT". + qt_dict_inline_topn_single_child """ + SELECT element_at(c[1][1], 'col17') + FROM ${dictTable} WHERE k1 IS NOT NULL ORDER BY k1 LIMIT 10 + """ + + // Same-typed children: the shifted decode does not error out, so the pairing + // loop used to run past the pruned iterator vector and SIGSEGV the BE; a + // partial regression could also return col1's values (1000001..) instead of + // col3's (3000001..) here. + qt_int_inline_topn_last_child """ + SELECT element_at(c[1][1], 'col3') + FROM ${intTable} WHERE k1 IS NOT NULL ORDER BY k1 LIMIT 10 + """ +} diff --git a/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy b/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy index 6775599d79204c..2ce56948450d84 100644 --- a/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy +++ b/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy @@ -268,6 +268,14 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { ORDER BY id, __DORIS_BINLOG_LSN__ """ + order_qt_mow_append_value_only """ + SELECT v1 + FROM ${mowTable}@incr('startTimestamp' = '${mowT0}', + "endTimestamp" = "${mowT1}", + "incrementType" = "APPEND_ONLY") + ORDER BY v1 + """ + // 2.2 MIN_DELTA [mowT0, mowT1]: per-key folded result. order_qt_mow_min_delta_range """ SELECT id, v1, v2, __DORIS_BINLOG_OP__ @@ -276,6 +284,26 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { "incrementType" = "MIN_DELTA") ORDER BY id, __DORIS_BINLOG_OP__, v1 """ + + // Read only a value column. Key, TSO, OP and the matching before-image + // column are storage dependencies and must not appear in the result. + order_qt_mow_min_delta_value_only """ + SELECT v1 + FROM ${mowTable}@incr('startTimestamp' = '${mowT0}', + "endTimestamp" = "${mowT1}", + "incrementType" = "MIN_DELTA") + ORDER BY v1 + """ + + // SELECT * expands to the visible base-table columns only. Internal + // binlog columns and generated before-image columns must stay hidden. + order_qt_mow_min_delta_select_star """ + SELECT * + FROM ${mowTable}@incr('startTimestamp' = '${mowT0}', + "endTimestamp" = "${mowT1}", + "incrementType" = "MIN_DELTA") + ORDER BY id, v1, v2 + """ // MIN_DELTA op codes (from __DORIS_BINLOG_OP__): // 0 = INSERT/APPEND, 1 = DELETE, 2 = UPDATE_BEFORE, 3 = UPDATE_AFTER // MIN_DELTA folds the start vs. end snapshot of the window, regardless @@ -305,6 +333,14 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { ORDER BY __DORIS_BINLOG_TSO__, __DORIS_BINLOG_LSN__ """ + order_qt_mow_detail_value_only """ + SELECT v1 + FROM ${mowTable}@incr('startTimestamp' = '${mowT0}', + "endTimestamp" = "${mowT1}", + "incrementType" = "DETAIL") + ORDER BY v1 + """ + // 2.4 startTimestamp only: includes the late (6,60). order_qt_mow_detail_start """ SELECT id, v1, __DORIS_BINLOG_OP__ diff --git a/regression-test/suites/schema_change/test_alter_table_column_with_delete_drop_column_dup_key.groovy b/regression-test/suites/schema_change/test_alter_table_column_with_delete_drop_column_dup_key.groovy index 1f9336ae3b3492..ff17dadf5f96a2 100644 --- a/regression-test/suites/schema_change/test_alter_table_column_with_delete_drop_column_dup_key.groovy +++ b/regression-test/suites/schema_change/test_alter_table_column_with_delete_drop_column_dup_key.groovy @@ -94,8 +94,8 @@ suite("test_alter_table_column_with_delete_drop_column_dup_key", "schema_change" """ sql "insert into ${tbName1} values(1,1,1,1);" - sql "insert into ${tbName1} values(2,2,2,2);" - qt_sql "select * from ${tbName1} where value2=2 order by k1;" + sql "insert into ${tbName1} values(20,200,2000,2);" + qt_sql "select * from ${tbName1} where value2=2000 order by k1;" // test alter light schema change by the way if (!isCloudMode()) { @@ -123,7 +123,7 @@ suite("test_alter_table_column_with_delete_drop_column_dup_key", "schema_change" }); qt_sql "select * from ${tbName1} where value1=3 order by k1;" - // drop value3 + // re-add value3 with a different type and unique ID sql """ ALTER TABLE ${tbName1} ADD COLUMN value3 CHAR(100) DEFAULT 'A'; @@ -138,9 +138,35 @@ suite("test_alter_table_column_with_delete_drop_column_dup_key", "schema_change" }); qt_sql "select * from ${tbName1} where value1=4 order by k1;" + order_qt_project_other_after_delete_drop_readd """ + SELECT value1 + FROM ${tbName1} + ORDER BY value1 + """ + sql "insert into ${tbName1} values(5,5,5,'B');" qt_sql "select * from ${tbName1} order by k1;" + // Keep the old delete predicate through light schema changes, then drop the same-name + // replacement so the following full schema change must resolve value3 historically. + sql """ + ALTER TABLE ${tbName1} + DROP COLUMN value3; + """ + Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { + res = getJobState(tbName1) + if (res == "FINISHED" || res == "CANCELLED") { + assertEquals("FINISHED", res) + return true; + } + return false; + }); + order_qt_project_other_after_delete_drop_readd_drop_again """ + SELECT value1 + FROM ${tbName1} + ORDER BY value1 + """ + // Do schema change that not do light weight schema change sql """ ALTER TABLE ${tbName1} @@ -155,6 +181,12 @@ suite("test_alter_table_column_with_delete_drop_column_dup_key", "schema_change" return false; }); qt_sql "select * from ${tbName1} where value1=4 order by k1;" + order_qt_project_other_after_delete_drop_readd_drop_again_full_schema_change """ + SELECT value1 + FROM ${tbName1} + ORDER BY value1 + """ + sql "DROP TABLE ${tbName1} FORCE;" //======================= Test Light Weight Schema Change with Compaction diff --git a/regression-test/suites/unique_seq_map_p0/test_unique_seq_map_hidden_dependency.groovy b/regression-test/suites/unique_seq_map_p0/test_unique_seq_map_hidden_dependency.groovy new file mode 100644 index 00000000000000..402a7ade9a3e3a --- /dev/null +++ b/regression-test/suites/unique_seq_map_p0/test_unique_seq_map_hidden_dependency.groovy @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_unique_seq_map_hidden_dependency") { + sql "DROP TABLE IF EXISTS test_unique_seq_map_hidden_dependency" + sql """ + CREATE TABLE test_unique_seq_map_hidden_dependency ( + k BIGINT NOT NULL, + v1 INT NULL, + v2 INT NULL, + s1 BIGINT NULL, + s2 BIGINT NULL + ) ENGINE=OLAP + UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false", + "light_schema_change" = "true", + "disable_auto_compaction" = "true", + "sequence_mapping.s1" = "v1", + "sequence_mapping.s2" = "v2" + ) + """ + + // Keep each write in a separate rowset. For key 1, v1 comes from the + // second rowset while v2 comes from the third. For key 2 the sources are + // reversed, so neither query can accidentally use the newest whole row. + sql "INSERT INTO test_unique_seq_map_hidden_dependency VALUES (1, 10, 100, 10, 10), (2, 50, 500, 50, 50)" + sql "INSERT INTO test_unique_seq_map_hidden_dependency VALUES (1, 20, 200, 20, 5), (2, 60, 600, 40, 60)" + sql "INSERT INTO test_unique_seq_map_hidden_dependency VALUES (1, 30, 300, 15, 30), (2, 70, 700, 70, 55)" + sql "sync" + + // s1 and the key are internal merge dependencies for this projection. + order_qt_hidden_s1 """ + SELECT v1 + FROM test_unique_seq_map_hidden_dependency + ORDER BY v1 + """ + + // s2 and the key are internal merge dependencies for this projection. + order_qt_hidden_s2 """ + SELECT v2 + FROM test_unique_seq_map_hidden_dependency + ORDER BY v2 + """ + + sql "SET topn_lazy_materialization_threshold = 1024" + def topnByValueQuery = """ + SELECT v1, v2 + FROM test_unique_seq_map_hidden_dependency + ORDER BY v1 DESC + LIMIT 1 + """ + explain { + sql "SHAPE PLAN ${topnByValueQuery}" + contains("PhysicalTopN") + notContains("PhysicalLazyMaterialize") + } + order_qt_topn_sequence_map_by_value topnByValueQuery + + def topnByKeyQuery = """ + SELECT k, v1, v2 + FROM test_unique_seq_map_hidden_dependency + ORDER BY k DESC + LIMIT 1 + """ + explain { + sql "SHAPE PLAN ${topnByKeyQuery}" + contains("PhysicalTopN") + notContains("PhysicalLazyMaterialize") + } + order_qt_topn_sequence_map_by_key topnByKeyQuery +}