diff --git a/be/src/exec/common/variant_util.cpp b/be/src/exec/common/variant_util.cpp index 0aa1172b5fc9ca..a4447523e1f6cb 100644 --- a/be/src/exec/common/variant_util.cpp +++ b/be/src/exec/common/variant_util.cpp @@ -280,6 +280,10 @@ bool should_materialize_nested_group_regular_subcolumns( (info_it != uid_to_variant_extended_info.end() && info_it->second.has_nested_group); } +bool can_skip_missing_physical_column(const TabletColumn& column) { + return column.has_default_value() || column.is_nullable(); +} + std::unordered_set collect_nested_group_compaction_root_uids( const TabletSchemaSPtr& target, const std::unordered_map& uid_to_variant_extended_info) { @@ -865,8 +869,14 @@ Status VariantCompactionUtil::aggregate_path_to_stats( for (const auto& segment : segment_cache.get_segments()) { std::shared_ptr column_reader; OlapReaderStatistics stats; - RETURN_IF_ERROR( - segment->get_column_reader(column->unique_id(), &column_reader, &stats)); + auto st = segment->get_column_reader(column->unique_id(), &column_reader, &stats); + if (st.is()) { + if (!can_skip_missing_physical_column(*column)) { + return st; + } + continue; + } + RETURN_IF_ERROR(st); if (!column_reader) { continue; } @@ -910,8 +920,14 @@ Status VariantCompactionUtil::aggregate_variant_extended_info( for (const auto& segment : segment_cache.get_segments()) { std::shared_ptr column_reader; OlapReaderStatistics stats; - RETURN_IF_ERROR( - segment->get_column_reader(column->unique_id(), &column_reader, &stats)); + auto st = segment->get_column_reader(column->unique_id(), &column_reader, &stats); + if (st.is()) { + if (!can_skip_missing_physical_column(*column)) { + return st; + } + continue; + } + RETURN_IF_ERROR(st); if (!column_reader) { continue; } diff --git a/be/src/storage/index/inverted/inverted_index_iterator.cpp b/be/src/storage/index/inverted/inverted_index_iterator.cpp index 936b82d5d56abe..df14673adcfac8 100644 --- a/be/src/storage/index/inverted/inverted_index_iterator.cpp +++ b/be/src/storage/index/inverted/inverted_index_iterator.cpp @@ -55,6 +55,7 @@ void InvertedIndexIterator::add_reader(InvertedIndexReaderType type, } Status InvertedIndexIterator::read_from_index(const IndexParam& param) { + _last_read_index_id = -1; const auto* i_param_ptr = std::get_if(¶m); if (i_param_ptr == nullptr) { return Status::Error( @@ -79,6 +80,7 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { return Status::Error( "inverted index reader is null"); } + _last_read_index_id = reader->get_index_id(); auto* runtime_state = _context->runtime_state; if (!i_param->skip_try && reader->type() == InvertedIndexReaderType::BKD) { if (runtime_state != nullptr && diff --git a/be/src/storage/index/inverted/inverted_index_iterator.h b/be/src/storage/index/inverted/inverted_index_iterator.h index afc4a663670633..c1e846d21a0541 100644 --- a/be/src/storage/index/inverted/inverted_index_iterator.h +++ b/be/src/storage/index/inverted/inverted_index_iterator.h @@ -70,6 +70,8 @@ class InvertedIndexIterator : public IndexIterator { [[nodiscard]] Result select_best_reader( const std::string& analyzer_key); + [[nodiscard]] int64_t last_read_index_id() const { return _last_read_index_id; } + private: ENABLE_FACTORY_CREATOR(InvertedIndexIterator); @@ -98,10 +100,11 @@ class InvertedIndexIterator : public IndexIterator { // These two phases are guaranteed not to overlap, so no synchronization is needed. // Do NOT call add_reader() after any read_from_index() call on the same iterator. std::vector _reader_entries; + int64_t _last_read_index_id = -1; // Index for O(1) lookup by analyzer_key. Maps normalized key to indices in _reader_entries. // Built incrementally in add_reader(). std::unordered_map> _key_to_entries; }; -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 445d40230e8e4c..134057e8e69dff 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -27,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +36,7 @@ #include #include #include +#include #include "common/cast_set.h" #include "common/config.h" @@ -112,6 +115,43 @@ struct TabletInfo { UniqueId tablet_uid; }; +enum class IndexProbeSource { + COLUMN_PREDICATE, + EXPR_PUSHDOWN, + SEARCH_FUNCTION, +}; + +enum class IndexProbeState { + APPLIED, + FALLBACK, + NOT_ATTEMPTED, +}; + +enum class IndexFallbackReason { + NONE, + MISSING_INDEX, + BYPASS, + NO_TERMS, + CORRUPTED, + EVALUATE_SKIPPED, + QUERY_DISABLED, + NOT_SUPPORTED, +}; + +struct IndexProbeEvent { + int32_t column_uid = -1; + std::optional variant_path; + int64_t index_id = -1; + int32_t segment_id = -1; + InvertedIndexStorageFormatPB storage_format = InvertedIndexStorageFormatPB::V1; + IndexProbeSource source = IndexProbeSource::COLUMN_PREDICATE; + IndexProbeState state = IndexProbeState::NOT_ATTEMPTED; + IndexFallbackReason reason = IndexFallbackReason::NONE; + int64_t input_rows = 0; + int64_t output_rows = 0; + int64_t filtered_rows = 0; +}; + struct TabletSize { TabletSize(TTabletId in_tablet_id, size_t in_tablet_size) : tablet_id(in_tablet_id), tablet_size(in_tablet_size) {} @@ -388,6 +428,8 @@ struct OlapReaderStatistics { int64_t inverted_index_analyzer_timer = 0; int64_t inverted_index_lookup_timer = 0; InvertedIndexStatistics inverted_index_stats; + bool collect_index_probe_events = false; + std::vector index_probe_events; int64_t ann_index_load_ns = 0; int64_t ann_topn_search_ns = 0; diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index dc6930777f5508..66279cba40eeb0 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -86,6 +86,7 @@ #include "storage/index/index_query_context.h" #include "storage/index/index_reader_helper.h" #include "storage/index/indexed_column_reader.h" +#include "storage/index/inverted/inverted_index_iterator.h" #include "storage/index/inverted/inverted_index_reader.h" #include "storage/index/ordinal_page_index.h" #include "storage/index/primary_key_index.h" @@ -231,6 +232,34 @@ Status rebind_storage_exprs_to_reader_schema(const StorageReadOptions& opts, con } // namespace +namespace { + +IndexFallbackReason index_fallback_reason(const Status& status, bool need_remaining) { + if (status.is()) { + return IndexFallbackReason::MISSING_INDEX; + } + if (status.is()) { + return IndexFallbackReason::BYPASS; + } + if (status.is() && need_remaining) { + return IndexFallbackReason::NO_TERMS; + } + if (status.is()) { + return IndexFallbackReason::CORRUPTED; + } + if (status.is()) { + return IndexFallbackReason::EVALUATE_SKIPPED; + } + return IndexFallbackReason::NOT_SUPPORTED; +} + +int64_t last_inverted_index_id(const std::unique_ptr& iterator) { + auto* inverted_index_iterator = dynamic_cast(iterator.get()); + return inverted_index_iterator == nullptr ? -1 : inverted_index_iterator->last_read_index_id(); +} + +} // namespace + SegmentIterator::~SegmentIterator() = default; void SegmentIterator::_init_row_bitmap_by_condition_cache() { @@ -906,19 +935,35 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { } { - if (_opts.runtime_state && - _opts.runtime_state->query_options().enable_inverted_index_query && - (has_index_in_iterators() || !_common_expr_ctxs_push_down.empty())) { + const bool should_collect_not_attempted = _opts.stats != nullptr && + _opts.stats->collect_index_probe_events && + !_col_predicates.empty(); + const bool enable_inverted_index_query = + _opts.runtime_state != nullptr && + _opts.runtime_state->query_options().enable_inverted_index_query; + if (_opts.runtime_state && (enable_inverted_index_query || should_collect_not_attempted) && + (has_index_in_iterators() || !_common_expr_ctxs_push_down.empty() || + should_collect_not_attempted)) { SCOPED_RAW_TIMER(&_opts.stats->inverted_index_filter_timer); size_t input_rows = _row_bitmap.cardinality(); // Only apply column-level inverted index if we have iterators - if (has_index_in_iterators()) { + if (enable_inverted_index_query && has_index_in_iterators()) { RETURN_IF_ERROR(_apply_inverted_index()); + } else { + for (const auto& pred : _col_predicates) { + _record_index_probe_event(pred, IndexProbeState::NOT_ATTEMPTED, + enable_inverted_index_query + ? IndexFallbackReason::MISSING_INDEX + : IndexFallbackReason::QUERY_DISABLED, + input_rows, input_rows); + } } // Always apply expr-level index (e.g., search expressions) if we have common_expr_pushdown // This allows search expressions with variant subcolumns to be evaluated even when // the segment doesn't have all subcolumns - RETURN_IF_ERROR(_apply_index_expr()); + if (enable_inverted_index_query) { + RETURN_IF_ERROR(_apply_index_expr()); + } for (auto it = _common_expr_ctxs_push_down.begin(); it != _common_expr_ctxs_push_down.end();) { if ((*it)->all_expr_inverted_index_evaluated()) { @@ -1445,19 +1490,82 @@ inline bool SegmentIterator::_inverted_index_not_support_pred_type(const Predica return type == PredicateType::BF || type == PredicateType::BITMAP_FILTER; } +void SegmentIterator::_record_index_probe_event(const std::shared_ptr& pred, + IndexProbeState state, IndexFallbackReason reason, + int64_t input_rows, int64_t output_rows, + int64_t index_id) { + if (_opts.stats == nullptr || !_opts.stats->collect_index_probe_events) { + return; + } + const auto pred_column_id = pred->column_id(); + if (pred_column_id >= _opts.tablet_schema->num_columns()) { + return; + } + + const auto& column = _opts.tablet_schema->column(pred_column_id); + int32_t column_uid = column.unique_id(); + if (column_uid < 0) { + column_uid = column.parent_unique_id(); + } + + std::optional variant_path; + if (column.has_path_info()) { + variant_path = column.path_info_ptr()->copy_pop_front().get_path(); + } + + _opts.stats->index_probe_events.push_back(IndexProbeEvent { + .column_uid = column_uid, + .variant_path = std::move(variant_path), + .index_id = index_id, + .segment_id = static_cast(_segment->id()), + .storage_format = _opts.tablet_schema->get_inverted_index_storage_format(), + .source = IndexProbeSource::COLUMN_PREDICATE, + .state = state, + .reason = reason, + .input_rows = input_rows, + .output_rows = output_rows, + .filtered_rows = std::max(0, input_rows - output_rows), + }); +} + Status SegmentIterator::_apply_inverted_index_on_column_predicate( std::shared_ptr pred, std::vector>& remaining_predicates, bool* continue_apply) { + const bool collect_index_probe_events = + _opts.stats != nullptr && _opts.stats->collect_index_probe_events; if (!_check_apply_by_inverted_index(pred)) { remaining_predicates.emplace_back(pred); + if (collect_index_probe_events) { + const auto rows = static_cast(_row_bitmap.cardinality()); + const auto pred_column_id = pred->column_id(); + IndexFallbackReason reason = IndexFallbackReason::NOT_SUPPORTED; + if (_opts.runtime_state != nullptr && + !_opts.runtime_state->query_options().enable_inverted_index_query) { + reason = IndexFallbackReason::QUERY_DISABLED; + } else if (pred_column_id >= _index_iterators.size() || + _index_iterators[pred_column_id] == nullptr) { + reason = IndexFallbackReason::MISSING_INDEX; + } + _record_index_probe_event(pred, IndexProbeState::NOT_ATTEMPTED, reason, rows, rows); + } } else { bool need_remaining_after_evaluate = _column_has_fulltext_index(pred->column_id()) && PredicateTypeTraits::is_equal_or_list(pred->type()); + const auto input_rows = + collect_index_probe_events ? static_cast(_row_bitmap.cardinality()) : 0; Status res = pred->evaluate(_storage_name_and_type[pred->column_id()], _index_iterators[pred->column_id()].get(), num_rows(), &_row_bitmap); if (!res.ok()) { if (_downgrade_without_index(res, need_remaining_after_evaluate)) { + if (collect_index_probe_events) { + const auto index_id = + last_inverted_index_id(_index_iterators[pred->column_id()]); + _record_index_probe_event( + pred, IndexProbeState::FALLBACK, + index_fallback_reason(res, need_remaining_after_evaluate), input_rows, + input_rows, index_id); + } remaining_predicates.emplace_back(pred); return Status::OK(); } @@ -1472,6 +1580,13 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate( *continue_apply = false; } + if (collect_index_probe_events) { + const auto index_id = last_inverted_index_id(_index_iterators[pred->column_id()]); + _record_index_probe_event(pred, IndexProbeState::APPLIED, IndexFallbackReason::NONE, + input_rows, static_cast(_row_bitmap.cardinality()), + index_id); + } + if (need_remaining_after_evaluate) { remaining_predicates.emplace_back(pred); return Status::OK(); @@ -3504,7 +3619,7 @@ bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& col return false; } - if (!_check_all_conditions_passed_inverted_index_for_column(cid)) { + if (!_check_all_conditions_passed_inverted_index_for_column(cid, true)) { return false; } diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index c7faf3fdb51d17..e393c7b52c6bd6 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -194,6 +194,9 @@ class SegmentIterator : public RowwiseIterator { std::shared_ptr pred, std::vector>& remaining_predicates, bool* continue_apply); + void _record_index_probe_event(const std::shared_ptr& pred, + IndexProbeState state, IndexFallbackReason reason, + int64_t input_rows, int64_t output_rows, int64_t index_id = -1); [[nodiscard]] Status _apply_ann_topn_predicate(); [[nodiscard]] Status _apply_index_expr(); diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index b7626d553ede15..0182362a6fa00e 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -17,10 +17,12 @@ #include "storage/task/index_builder.h" +#include #include #include "common/logging.h" #include "common/status.h" +#include "exec/common/variant_util.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" @@ -36,6 +38,99 @@ namespace doris { +namespace { + +std::string alter_index_field_pattern(const TOlapTableIndex& index) { + if (!index.__isset.properties) { + return {}; + } + auto it = index.properties.find("field_pattern"); + if (it == index.properties.end()) { + return {}; + } + return it->second; +} + +bool has_field_pattern_index(const std::vector& indexes) { + return std::any_of(indexes.begin(), indexes.end(), + [](const auto& index) { return !alter_index_field_pattern(index).empty(); }); +} + +int32_t resolve_alter_index_column_idx(const TabletSchemaSPtr& tablet_schema, + const TOlapTableIndex& alter_index) { + DCHECK_EQ(alter_index.columns.size(), 1); + auto column_idx = tablet_schema->field_index(alter_index.columns[0]); + if (column_idx < 0 && alter_index.__isset.column_unique_ids && + !alter_index.column_unique_ids.empty()) { + column_idx = tablet_schema->field_index(alter_index.column_unique_ids[0]); + } + if (column_idx < 0) { + return column_idx; + } + + const auto field_pattern = alter_index_field_pattern(alter_index); + if (field_pattern.empty()) { + return column_idx; + } + + const auto& parent_column = tablet_schema->column(column_idx); + const int32_t parent_unique_id = parent_column.is_extracted_column() + ? parent_column.parent_unique_id() + : parent_column.unique_id(); + for (int32_t i = 0; i < tablet_schema->num_columns(); ++i) { + const auto& column = tablet_schema->column(i); + if (!column.has_path_info() || column.parent_unique_id() != parent_unique_id) { + continue; + } + const auto full_path = column.path_info_ptr()->get_path(); + const auto relative_path = column.path_info_ptr()->copy_pop_front().get_path(); + if (field_pattern == full_path || field_pattern == relative_path) { + return i; + } + } + return column_idx; +} + +std::vector inverted_indexs_for_alter_index( + const TabletSchemaSPtr& tablet_schema, const TOlapTableIndex& alter_index, + const TabletColumn& column, + std::vector>* owned_indexes) { + auto index_metas = tablet_schema->inverted_indexs(column); + if (!index_metas.empty()) { + return index_metas; + } + + const auto field_pattern = alter_index_field_pattern(alter_index); + if (field_pattern.empty()) { + return index_metas; + } + + const int32_t parent_unique_id = + column.is_extracted_column() ? column.parent_unique_id() : column.unique_id(); + if (column.is_extracted_column()) { + TabletSchema::SubColumnInfo sub_column_info; + const std::string relative_path = column.path_info_ptr()->copy_pop_front().get_path(); + if (variant_util::generate_sub_column_info(*tablet_schema, parent_unique_id, relative_path, + &sub_column_info)) { + for (auto& index : sub_column_info.indexes) { + index_metas.push_back(index.get()); + owned_indexes->emplace_back(std::move(index)); + } + if (!index_metas.empty()) { + return index_metas; + } + } + } + + for (const auto& index : + tablet_schema->inverted_index_by_field_pattern(parent_unique_id, field_pattern)) { + index_metas.push_back(index.get()); + } + return index_metas; +} + +} // namespace + IndexBuilder::IndexBuilder(StorageEngine& engine, TabletSharedPtr tablet, const std::vector& columns, const std::vector& alter_inverted_indexes, @@ -97,25 +192,20 @@ Status IndexBuilder::update_inverted_index_info() { if (_is_drop_op) { for (const auto& t_inverted_index : _alter_inverted_indexes) { - DCHECK_EQ(t_inverted_index.columns.size(), 1); - auto column_name = t_inverted_index.columns[0]; - auto column_idx = output_rs_tablet_schema->field_index(column_name); + const auto column_idx = + resolve_alter_index_column_idx(output_rs_tablet_schema, t_inverted_index); if (column_idx < 0) { - if (!t_inverted_index.column_unique_ids.empty()) { - auto column_unique_id = t_inverted_index.column_unique_ids[0]; - column_idx = output_rs_tablet_schema->field_index(column_unique_id); - } - if (column_idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name - << " referenced_column=" << column_idx << "]"; - continue; - } + LOG(WARNING) << "referenced column was missing. " + << "[column=" << t_inverted_index.columns[0] + << " referenced_column=" << column_idx << "]"; + continue; } auto column = output_rs_tablet_schema->column(column_idx); // inverted index - auto index_metas = output_rs_tablet_schema->inverted_indexs(column); + std::vector> owned_index_metas; + auto index_metas = inverted_indexs_for_alter_index( + output_rs_tablet_schema, t_inverted_index, column, &owned_index_metas); for (const auto& index_meta : index_metas) { // Only drop the index that matches the requested index_id, // not all indexes on this column @@ -226,6 +316,14 @@ Status IndexBuilder::update_inverted_index_info() { output_rs_tablet_schema->append_index(std::move(index)); } + if (has_field_pattern_index(_alter_inverted_indexes)) { + auto extended_schema = + variant_util::VariantCompactionUtil::calculate_variant_extended_schema( + {input_rowset}, output_rs_tablet_schema); + if (extended_schema != nullptr) { + output_rs_tablet_schema = std::move(extended_schema); + } + } } // construct input rowset reader RowsetReaderSharedPtr input_rs_reader; @@ -454,25 +552,18 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta nullptr, true /* can_use_ram_dir */, _tablet->tablet_id()); } // create inverted index writer, or ann index writer + std::vector> owned_index_metas; for (auto inverted_index : _alter_inverted_indexes) { DCHECK(inverted_index.index_type == TIndexType::INVERTED || inverted_index.index_type == TIndexType::ANN); - DCHECK_EQ(inverted_index.columns.size(), 1); auto index_id = inverted_index.index_id; - auto column_name = inverted_index.columns[0]; - auto column_idx = output_rowset_schema->field_index(column_name); + const auto column_idx = + resolve_alter_index_column_idx(output_rowset_schema, inverted_index); if (column_idx < 0) { - if (inverted_index.__isset.column_unique_ids && - !inverted_index.column_unique_ids.empty()) { - column_idx = output_rowset_schema->field_index( - inverted_index.column_unique_ids[0]); - } - if (column_idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name - << " referenced_column=" << column_idx << "]"; - continue; - } + LOG(WARNING) << "referenced column was missing. " + << "[column=" << inverted_index.columns[0] + << " referenced_column=" << column_idx << "]"; + continue; } auto column = output_rowset_schema->column(column_idx); // variant column is not support for building index @@ -485,12 +576,12 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta continue; } DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id)); - _olap_data_convertor->add_column_data_convertor(column); - return_columns.emplace_back(column_idx); + bool writer_created = false; if (inverted_index.index_type == TIndexType::INVERTED) { // inverted index - auto index_metas = output_rowset_schema->inverted_indexs(column); + auto index_metas = inverted_indexs_for_alter_index( + output_rowset_schema, inverted_index, column, &owned_index_metas); for (const auto& index_meta : index_metas) { if (index_meta->index_id() != index_id) { continue; @@ -519,6 +610,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta _index_column_writers.insert( std::make_pair(writer_sign, std::move(inverted_index_builder))); inverted_index_writer_signs.emplace_back(writer_sign); + writer_created = true; } } } else if (inverted_index.index_type == TIndexType::ANN) { @@ -548,9 +640,14 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta _index_column_writers.insert( std::make_pair(writer_sign, std::move(index_writer))); inverted_index_writer_signs.emplace_back(writer_sign); + writer_created = true; } } } + if (writer_created) { + _olap_data_convertor->add_column_data_convertor(column); + return_columns.emplace_back(column_idx); + } } // DO NOT forget index_file_writer for the segment, otherwise, original inverted index will be deleted. @@ -666,28 +763,24 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, VLOG_DEBUG << "begin to write inverted/ann index"; // converter block data _olap_data_convertor->set_source_content(block, 0, block->rows()); - for (auto i = 0; i < _alter_inverted_indexes.size(); ++i) { - auto inverted_index = _alter_inverted_indexes[i]; + size_t convertor_idx = 0; + for (const auto& inverted_index : _alter_inverted_indexes) { auto index_id = inverted_index.index_id; - auto column_name = inverted_index.columns[0]; - auto column_idx = tablet_schema->field_index(column_name); + auto column_idx = resolve_alter_index_column_idx(tablet_schema, inverted_index); DBUG_EXECUTE_IF("IndexBuilder::_write_inverted_index_data_column_idx_is_negative", { column_idx = -1; }) if (column_idx < 0) { - if (!inverted_index.column_unique_ids.empty()) { - auto column_unique_id = inverted_index.column_unique_ids[0]; - column_idx = tablet_schema->field_index(column_unique_id); - } - if (column_idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name << " referenced_column=" << column_idx - << "]"; - continue; - } + LOG(WARNING) << "referenced column was missing. " + << "[column=" << inverted_index.columns[0] + << " referenced_column=" << column_idx << "]"; + continue; } const auto& column = tablet_schema->column(column_idx); auto writer_sign = std::make_pair(segment_idx, index_id); - auto converted_result = _olap_data_convertor->convert_column_data(i); + if (_index_column_writers.find(writer_sign) == _index_column_writers.end()) { + continue; + } + auto converted_result = _olap_data_convertor->convert_column_data(convertor_idx++); DBUG_EXECUTE_IF("IndexBuilder::_write_inverted_index_data_convert_column_data_error", { converted_result.first = Status::Error( "debug point: _write_inverted_index_data_convert_column_data_error"); @@ -699,10 +792,10 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, const auto* ptr = (const uint8_t*)converted_result.second->get_data(); const auto* null_map = converted_result.second->get_nullmap(); if (null_map) { - RETURN_IF_ERROR(_add_nullable(column_name, writer_sign, &column, null_map, &ptr, + RETURN_IF_ERROR(_add_nullable(column.name(), writer_sign, &column, null_map, &ptr, block->rows())); } else { - RETURN_IF_ERROR(_add_data(column_name, writer_sign, &column, &ptr, block->rows())); + RETURN_IF_ERROR(_add_data(column.name(), writer_sign, &column, &ptr, block->rows())); } } _olap_data_convertor->clear_source_content(); diff --git a/be/test/storage/index_storage_lifecycle_test.cpp b/be/test/storage/index_storage_lifecycle_test.cpp new file mode 100644 index 00000000000000..17dc0741aa1e98 --- /dev/null +++ b/be/test/storage/index_storage_lifecycle_test.cpp @@ -0,0 +1,1463 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "common/config.h" +#include "core/data_type/data_type_string.h" +#include "exec/common/variant_util.h" +#include "storage/predicate/predicate_creator.h" +#include "testutil/index_storage_test_util.h" +#include "util/debug_points.h" + +namespace doris::index_storage_test { +namespace { + +std::shared_ptr title_equals(std::string value) { + return create_comparison_predicate( + 1, "title", std::make_shared(), + Field::create_field(std::move(value)), false); +} + +std::shared_ptr string_equals(int32_t column_id, std::string column_name, + std::string value) { + return create_comparison_predicate( + column_id, std::move(column_name), std::make_shared(), + Field::create_field(std::move(value)), false); +} + +void expect_applied_title_index(const IndexReadResult& result, int64_t expected_filtered_rows, + int64_t index_id = 20001) { + expect_inverted_index_used(result); + expect_index_filter_stats(result, expected_filtered_rows); + + int64_t filtered_rows = 0; + int64_t applied_events = 0; + for (const auto& event : result.stats.index_probe_events) { + if (event.state != IndexProbeState::APPLIED || event.column_uid != 2 || + event.index_id != index_id) { + continue; + } + ++applied_events; + filtered_rows += event.filtered_rows; + EXPECT_FALSE(event.variant_path.has_value()); + } + EXPECT_GT(applied_events, 0); + EXPECT_EQ(filtered_rows, expected_filtered_rows); +} + +void expect_applied_variant_path_index(const IndexReadResult& result, std::string_view path, + int64_t index_id, int64_t expected_filtered_rows) { + expect_inverted_index_used(result); + expect_index_filter_stats(result, expected_filtered_rows); + + int64_t filtered_rows = 0; + int64_t applied_events = 0; + for (const auto& event : result.stats.index_probe_events) { + if (event.state != IndexProbeState::APPLIED || event.column_uid != 2 || + event.index_id != index_id || !event.variant_path.has_value() || + event.variant_path.value() != path) { + continue; + } + ++applied_events; + filtered_rows += event.filtered_rows; + } + EXPECT_GT(applied_events, 0); + EXPECT_EQ(filtered_rows, expected_filtered_rows); +} + +bool has_doc_value_column(const IndexRowsetProbe& probe) { + return std::any_of(probe.segments.begin(), probe.segments.end(), [](const auto& segment) { + return std::any_of(segment.variant_columns.begin(), segment.variant_columns.end(), + [](const auto& column) { return column.is_doc_value_column; }); + }); +} + +bool has_variant_layout(const IndexRowsetProbe& probe, int32_t parent_unique_id, + std::string_view relative_path) { + return std::any_of(probe.segments.begin(), probe.segments.end(), [&](const auto& segment) { + return std::any_of(segment.variant_columns.begin(), segment.variant_columns.end(), + [&](const auto& column) { + return column.parent_unique_id == parent_unique_id && + column.relative_path == relative_path; + }); + }); +} + +bool has_variant_parent(const IndexRowsetProbe& probe, int32_t parent_unique_id) { + return std::any_of(probe.segments.begin(), probe.segments.end(), [&](const auto& segment) { + return std::any_of( + segment.variant_columns.begin(), segment.variant_columns.end(), + [&](const auto& column) { return column.parent_unique_id == parent_unique_id; }); + }); +} + +bool has_sparse_path_stat(const IndexRowsetProbe& probe, std::string_view relative_path) { + return std::any_of(probe.segments.begin(), probe.segments.end(), [&](const auto& segment) { + return std::any_of(segment.variant_columns.begin(), segment.variant_columns.end(), + [&](const auto& column) { + auto it = + column.sparse_non_null_size.find(std::string(relative_path)); + return it != column.sparse_non_null_size.end() && it->second > 0; + }); + }); +} + +std::string dump_schema_paths(const TabletSchema& schema) { + std::ostringstream out; + for (int32_t i = 0; i < schema.num_columns(); ++i) { + const auto& column = schema.column(i); + out << i << ": uid=" << column.unique_id() << " name=" << column.name() + << " type=" << TabletColumn::get_string_by_field_type(column.type()); + if (column.has_path_info()) { + out << " path=" << column.path_info_ptr()->get_path(); + } + out << '\n'; + } + return out.str(); +} + +class ScopedDebugPoint { +public: + ScopedDebugPoint(std::string name, std::map params) + : _name(std::move(name)), _enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->remove(_name); + DebugPoints::instance()->add_with_params(_name, params); + } + + ~ScopedDebugPoint() { + DebugPoints::instance()->remove(_name); + config::enable_debug_points = _enable_debug_points; + } + +private: + std::string _name; + bool _enable_debug_points = false; +}; + +} // namespace + +class IndexStorageLifecycleTest : public IndexStorageTestFixture { +protected: + void run_deep_sparse_variant_lifecycle(bool external_segment_meta, int64_t tablet_id); + void run_nested_group_variant_lifecycle(bool external_segment_meta, int64_t tablet_id); +}; + +void IndexStorageLifecycleTest::run_deep_sparse_variant_lifecycle(bool external_segment_meta, + int64_t tablet_id) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + variant.max_subcolumns_count = 1; + variant.sparse_hash_shard_count = 2; + + IndexTabletOptions options; + options.tablet_id = tablet_id; + options.external_segment_meta = external_segment_meta; + options.variant_columns = {std::move(variant)}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back(VariantJsonBatch::single_variant( + {R"({"hot": "h0", "deep": {"rare_a": "a0"}, "cold0": "c0"})", + R"({"hot": "h1", "deep": {"rare_b": "b0"}, "cold1": "c1"})"}, + 0)); + auto rowset0_result = write_rowset(rowset0); + ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error(); + + auto rowset0_probe = probe_rowset(rowset0_result.value()); + ASSERT_TRUE(rowset0_probe.has_value()) << rowset0_probe.error(); + EXPECT_TRUE(has_variant_layout(rowset0_probe.value(), 2, "hot")); + EXPECT_TRUE(has_sparse_path_stat(rowset0_probe.value(), "deep.rare_a")); + EXPECT_TRUE(has_sparse_path_stat(rowset0_probe.value(), "deep.rare_b")); + + IndexRowsetSpec rowset1; + rowset1.version = 1; + rowset1.batches.push_back(VariantJsonBatch::single_variant( + {R"({"hot": "h2", "deep": {"rare_a": "a1"}, "cold2": "c2"})", + R"({"hot": "h3", "deep": {"rare_c": "c0"}, "cold3": "c3"})"}, + 100)); + auto rowset1_result = write_rowset(rowset1); + ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error(); + + auto read_result = read_rowsets({rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 4); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, + {rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + + auto compacted_probe = probe_rowset(compacted.value()); + ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error(); + EXPECT_TRUE(has_variant_layout(compacted_probe.value(), 2, "hot")); + EXPECT_TRUE(has_sparse_path_stat(compacted_probe.value(), "deep.rare_a")); + EXPECT_TRUE(has_sparse_path_stat(compacted_probe.value(), "deep.rare_b")); + EXPECT_TRUE(has_sparse_path_stat(compacted_probe.value(), "deep.rare_c")); + + auto compacted_read = read_rowsets({compacted.value()}); + ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error(); + EXPECT_EQ(compacted_read->rows_read, 4); +} + +void IndexStorageLifecycleTest::run_nested_group_variant_lifecycle(bool external_segment_meta, + int64_t tablet_id) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + variant.max_subcolumns_count = 1; + variant.sparse_hash_shard_count = 2; + variant.enable_nested_group = true; + + IndexTabletOptions options; + options.tablet_id = tablet_id; + options.external_segment_meta = external_segment_meta; + options.variant_columns = {std::move(variant)}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back(VariantJsonBatch::single_variant( + {R"({"owner": "alice", "profile": {"region": "us"}, "items": [{"sku": "a", "qty": 1}]})", + R"({"owner": "bob", "profile": {"region": "eu"}, "items": [{"sku": "b", "qty": 2}]})"}, + 0)); + auto rowset0_result = write_rowset(rowset0); + ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error(); + + IndexRowsetSpec rowset1; + rowset1.version = 1; + rowset1.batches.push_back(VariantJsonBatch::single_variant( + {R"({"owner": "carol", "profile": {"region": "apac"}, "items": [{"sku": "c", "qty": 3}]})", + R"({"owner": "dave", "profile": {"region": "us"}, "items": [{"sku": "d", "qty": 4}]})"}, + 100)); + auto rowset1_result = write_rowset(rowset1); + ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error(); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, + {rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + ASSERT_TRUE(compacted.value()->tablet_schema()->has_column_unique_id(2)); + EXPECT_TRUE(compacted.value()->tablet_schema()->column_by_uid(2).variant_enable_nested_group()); + + auto compacted_probe = probe_rowset(compacted.value()); + ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error(); + EXPECT_TRUE(has_variant_parent(compacted_probe.value(), 2)); + EXPECT_TRUE(has_variant_layout(compacted_probe.value(), 2, "owner")); + EXPECT_TRUE(has_variant_layout(compacted_probe.value(), 2, "profile.region")); + + auto compacted_read = read_rowsets({compacted.value()}); + ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error(); + EXPECT_EQ(compacted_read->rows_read, 4); +} + +TEST_F(IndexStorageLifecycleTest, TextIndexHitAfterCumulativeCompaction) { + const auto index_case = + IndexStorageCaseBuilder("text_index_hit_after_cumulative_compaction") + .tablet_id(110001) + .text_column(TextColumnSpec {.unique_id = 2, .name = "title"}) + .inverted_index(IndexSpec::column_index(20001, "idx_title", 2)) + .rowset(0, IndexDataSourceSpec::inline_text({"hello", "other"}, 0)) + .rowset(1, IndexDataSourceSpec::inline_text({"hello", "other"}, 100)) + .build(); + ASSERT_TRUE(create_tablet(index_case.tablet_options).ok()); + auto rowsets = write_rowsets(index_case.rowsets); + ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); + + IndexReadOptions read_options; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets(rowsets.value(), read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 2); + expect_applied_title_index(read_result.value(), 2); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value()); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + + auto reloaded = reload_rowsets({compacted.value()}); + ASSERT_TRUE(reloaded.has_value()) << reloaded.error(); + IndexReadOptions compacted_read_options; + compacted_read_options.predicates.push_back(title_equals("hello")); + auto compacted_read = read_rowsets(reloaded.value(), compacted_read_options); + ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error(); + EXPECT_EQ(compacted_read->rows_read, 2); + expect_applied_title_index(compacted_read.value(), 2); +} + +TEST_F(IndexStorageLifecycleTest, TextIndexHitAfterFullCompaction) { + IndexTabletOptions options; + options.tablet_id = 110007; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back(VariantJsonBatch::single_text({"hello", "other"}, 0)); + auto rowset0_result = write_rowset(rowset0); + ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error(); + + IndexRowsetSpec rowset1; + rowset1.version = 1; + rowset1.batches.push_back(VariantJsonBatch::single_text({"other", "hello"}, 100)); + auto rowset1_result = write_rowset(rowset1); + ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error(); + + auto compacted = compact_rowsets(IndexCompactionKind::FULL, + {rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + + IndexReadOptions read_options; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets({compacted.value()}, read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 2); + expect_applied_title_index(read_result.value(), 2); +} + +TEST_F(IndexStorageLifecycleTest, CountOnIndexPredicateRecordsAppliedEvent) { + IndexTabletOptions options; + options.tablet_id = 110008; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other", "hello"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + IndexReadOptions read_options; + read_options.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets({rowset_result.value()}, read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 2); + expect_applied_title_index(read_result.value(), 1); +} + +TEST_F(IndexStorageLifecycleTest, CountOnIndexSkipsReadingKeyDataWhenIndexApplied) { + IndexTabletOptions options; + options.tablet_id = 110027; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other", "hello"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + ScopedDebugPoint fail_if_key_column_is_read("segment_iterator._read_columns_by_index", + {{"column_name", "k"}}); + IndexReadOptions read_options; + read_options.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets({rowset_result.value()}, read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 2); + EXPECT_EQ(read_result->stats.raw_rows_read, 2); + expect_applied_title_index(read_result.value(), 1); +} + +TEST_F(IndexStorageLifecycleTest, TextPredicateRecordsQueryIndexIdAfterNullBitmapCheck) { + IndexTabletOptions options; + options.tablet_id = 110010; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + options.inverted_indexes.push_back( + IndexSpec::column_index(20001, "idx_title_fulltext", 2, {{"parser", "english"}})); + options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_title_string", 2)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.data_sources.push_back(IndexDataSourceSpec::inline_text({"hello", "other"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + IndexReadOptions read_options; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets({rowset_result.value()}, read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 1); + expect_applied_title_index(read_result.value(), 1, 20002); +} + +TEST_F(IndexStorageLifecycleTest, V1IndexStorageWritesOneFilePerIndex) { + IndexTabletOptions options; + options.tablet_id = 110019; + options.index_storage_format = InvertedIndexStorageFormatPB::V1; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + options.inverted_indexes.push_back( + IndexSpec::column_index(20001, "idx_title_fulltext", 2, {{"parser", "english"}})); + options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_title_string", 2)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + auto probe = probe_rowset(rowset_result.value()); + ASSERT_TRUE(probe.has_value()) << probe.error(); + EXPECT_EQ(probe->num_segments, 1); + EXPECT_EQ(probe->index_files.expected_files, 2); + expect_index_files(probe.value(), true); + + IndexReadOptions read_options; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets({rowset_result.value()}, read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 1); + expect_applied_title_index(read_result.value(), 1, 20002); +} + +TEST_F(IndexStorageLifecycleTest, V2IndexStorageWritesCompoundIndexFile) { + IndexTabletOptions options; + options.tablet_id = 110020; + options.index_storage_format = InvertedIndexStorageFormatPB::V2; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + options.inverted_indexes.push_back( + IndexSpec::column_index(20001, "idx_title_fulltext", 2, {{"parser", "english"}})); + options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_title_string", 2)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + auto probe = probe_rowset(rowset_result.value()); + ASSERT_TRUE(probe.has_value()) << probe.error(); + EXPECT_EQ(probe->num_segments, 1); + EXPECT_EQ(probe->index_files.expected_files, 1); + EXPECT_GT(probe->index_files.rowset_meta_entries, 0); + expect_index_files(probe.value(), true); + + IndexReadOptions read_options; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets({rowset_result.value()}, read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 1); + expect_applied_title_index(read_result.value(), 1, 20002); +} + +TEST_F(IndexStorageLifecycleTest, TextIndexDisabledRecordsNotAttempted) { + IndexTabletOptions options; + options.tablet_id = 110013; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other", "hello"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + IndexReadOptions read_options; + read_options.enable_inverted_index_query = false; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets({rowset_result.value()}, read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 2); + EXPECT_EQ(read_result->stats.rows_inverted_index_filtered, 0); + expect_inverted_index_not_attempted(read_result.value()); + + auto event = std::find_if(read_result->stats.index_probe_events.begin(), + read_result->stats.index_probe_events.end(), [](const auto& e) { + return e.state == IndexProbeState::NOT_ATTEMPTED && + e.reason == IndexFallbackReason::QUERY_DISABLED && + e.column_uid == 2 && e.index_id == -1; + }); + ASSERT_NE(event, read_result->stats.index_probe_events.end()); +} + +TEST_F(IndexStorageLifecycleTest, TextPredicateWithoutIndexRecordsNotAttempted) { + IndexTabletOptions options; + options.tablet_id = 110004; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + IndexReadOptions read_options; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets({rowset_result.value()}, read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 1); + EXPECT_EQ(read_result->stats.rows_inverted_index_filtered, 0); + expect_inverted_index_not_attempted(read_result.value()); + + auto event = std::find_if(read_result->stats.index_probe_events.begin(), + read_result->stats.index_probe_events.end(), [](const auto& e) { + return e.state == IndexProbeState::NOT_ATTEMPTED && + e.reason == IndexFallbackReason::MISSING_INDEX && + e.column_uid == 2; + }); + ASSERT_NE(event, read_result->stats.index_probe_events.end()); +} + +TEST_F(IndexStorageLifecycleTest, DropTextIndexRecordsNotAttempted) { + const auto index_case = + IndexStorageCaseBuilder("drop_text_index_records_not_attempted") + .tablet_id(110005) + .text_column(TextColumnSpec {.unique_id = 2, .name = "title"}) + .inverted_index(IndexSpec::column_index(20001, "idx_title", 2)) + .rowset(0, IndexDataSourceSpec::inline_text({"hello", "other"}, 0)) + .build(); + ASSERT_TRUE(create_tablet(index_case.tablet_options).ok()); + auto rowsets = write_rowsets(index_case.rowsets); + ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); + + IndexReadOptions before_drop_options; + before_drop_options.predicates.push_back(title_equals("hello")); + auto before_drop = read_rowsets(rowsets.value(), before_drop_options); + ASSERT_TRUE(before_drop.has_value()) << before_drop.error(); + EXPECT_EQ(before_drop->rows_read, 1); + expect_applied_title_index(before_drop.value(), 1); + + auto dropped_rowsets = drop_inverted_indexes({IndexSpec::column_index(20001, "idx_title", 2)}); + ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error(); + ASSERT_EQ(dropped_rowsets->size(), 1); + + auto dropped_probe = probe_rowset(dropped_rowsets->front()); + ASSERT_TRUE(dropped_probe.has_value()) << dropped_probe.error(); + expect_index_files(dropped_probe.value(), false); + + auto reloaded_dropped_rowsets = reload_rowsets(dropped_rowsets.value()); + ASSERT_TRUE(reloaded_dropped_rowsets.has_value()) << reloaded_dropped_rowsets.error(); + IndexReadOptions after_drop_options; + after_drop_options.predicates.push_back(title_equals("hello")); + auto after_drop = read_rowsets(reloaded_dropped_rowsets.value(), after_drop_options); + ASSERT_TRUE(after_drop.has_value()) << after_drop.error(); + EXPECT_EQ(after_drop->rows_read, 1); + EXPECT_EQ(after_drop->stats.rows_inverted_index_filtered, 0); + expect_inverted_index_not_attempted(after_drop.value()); + + auto event = std::find_if(after_drop->stats.index_probe_events.begin(), + after_drop->stats.index_probe_events.end(), [](const auto& e) { + return e.state == IndexProbeState::NOT_ATTEMPTED && + e.reason == IndexFallbackReason::MISSING_INDEX && + e.column_uid == 2; + }); + ASSERT_NE(event, after_drop->stats.index_probe_events.end()); +} + +TEST_F(IndexStorageLifecycleTest, BuildTextIndexAfterExistingRowsUsesNewIndex) { + const auto index_case = + IndexStorageCaseBuilder("build_text_index_after_existing_rows_uses_new_index") + .tablet_id(110011) + .text_column(TextColumnSpec {.unique_id = 2, .name = "title"}) + .rowset(0, IndexDataSourceSpec::inline_text({"hello", "other"}, 0)) + .rowset(1, IndexDataSourceSpec::inline_text({"other", "hello"}, 100)) + .build(); + ASSERT_TRUE(create_tablet(index_case.tablet_options).ok()); + auto rowsets = write_rowsets(index_case.rowsets); + ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); + + IndexReadOptions before_build_options; + before_build_options.predicates.push_back(title_equals("hello")); + auto before_build = read_rowsets(rowsets.value(), before_build_options); + ASSERT_TRUE(before_build.has_value()) << before_build.error(); + EXPECT_EQ(before_build->rows_read, 2); + EXPECT_EQ(before_build->stats.rows_inverted_index_filtered, 0); + expect_inverted_index_not_attempted(before_build.value()); + + const auto built_index = IndexSpec::column_index(20003, "idx_title_built", 2); + auto built_rowsets = build_inverted_indexes({built_index}); + ASSERT_TRUE(built_rowsets.has_value()) << built_rowsets.error(); + ASSERT_EQ(built_rowsets->size(), 2); + for (const auto& rowset : built_rowsets.value()) { + auto probe = probe_rowset(rowset); + ASSERT_TRUE(probe.has_value()) << probe.error(); + expect_index_files(probe.value(), true); + } + + auto reloaded_built_rowsets = reload_rowsets(built_rowsets.value()); + ASSERT_TRUE(reloaded_built_rowsets.has_value()) << reloaded_built_rowsets.error(); + IndexReadOptions after_build_options; + after_build_options.predicates.push_back(title_equals("hello")); + auto after_build = read_rowsets(reloaded_built_rowsets.value(), after_build_options); + ASSERT_TRUE(after_build.has_value()) << after_build.error(); + EXPECT_EQ(after_build->rows_read, 2); + expect_applied_title_index(after_build.value(), 2, 20003); +} + +TEST_F(IndexStorageLifecycleTest, BuildAndDropVariantPathIndexAfterExistingRows) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + variant.predefined_paths = { + VariantPathSpec {.path = "b", .type = FieldType::OLAP_FIELD_TYPE_STRING}, + }; + + const auto index_case = + IndexStorageCaseBuilder("build_and_drop_variant_path_index_after_existing_rows") + .tablet_id(110018) + .variant_column(std::move(variant)) + .rowset(0, IndexDataSourceSpec::inline_variant( + {R"({"b": "one"})", R"({"b": "other"})"}, 0)) + .rowset(1, IndexDataSourceSpec::inline_variant( + {R"({"b": "other"})", R"({"b": "one"})"}, 100)) + .build(); + ASSERT_TRUE(create_tablet(index_case.tablet_options).ok()); + auto rowsets = write_rowsets(index_case.rowsets); + ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); + + auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value()); + ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error(); + + const int32_t path_column_id = column_id_by_path("v.b"); + ASSERT_GE(path_column_id, 0); + const auto& path_column = tablet_schema()->column(path_column_id); + + IndexReadOptions read_options; + read_options.return_columns = {0, static_cast(path_column_id)}; + read_options.target_cast_type_for_variants[path_column.name()] = + std::make_shared(); + read_options.predicates.push_back(string_equals(path_column_id, path_column.name(), "one")); + + auto before_build = read_rowsets(readable_rowsets.value(), read_options); + ASSERT_TRUE(before_build.has_value()) << before_build.error(); + EXPECT_EQ(before_build->rows_read, 2); + EXPECT_EQ(before_build->stats.rows_inverted_index_filtered, 0); + expect_inverted_index_not_attempted(before_build.value()); + + auto missing_before = std::find_if( + before_build->stats.index_probe_events.begin(), + before_build->stats.index_probe_events.end(), [](const auto& e) { + return e.state == IndexProbeState::NOT_ATTEMPTED && + e.reason == IndexFallbackReason::MISSING_INDEX && e.column_uid == 2 && + e.variant_path.has_value() && e.variant_path.value() == "b"; + }); + ASSERT_NE(missing_before, before_build->stats.index_probe_events.end()); + + const auto path_index = IndexSpec::field_pattern_index(20004, "idx_v_b_built", 2, "b"); + auto built_rowsets = build_inverted_indexes({path_index}); + ASSERT_TRUE(built_rowsets.has_value()) << built_rowsets.error(); + ASSERT_EQ(built_rowsets->size(), 2); + for (const auto& rowset : built_rowsets.value()) { + auto probe = probe_rowset(rowset); + ASSERT_TRUE(probe.has_value()) << probe.error(); + expect_index_files(probe.value(), true); + } + + auto reloaded_built_rowsets = reload_rowsets(built_rowsets.value()); + ASSERT_TRUE(reloaded_built_rowsets.has_value()) << reloaded_built_rowsets.error(); + auto after_build = read_rowsets(reloaded_built_rowsets.value(), read_options); + ASSERT_TRUE(after_build.has_value()) << after_build.error(); + EXPECT_EQ(after_build->rows_read, 2); + expect_applied_variant_path_index(after_build.value(), "b", 20004, 2); + + auto dropped_rowsets = drop_inverted_indexes({path_index}); + ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error(); + ASSERT_EQ(dropped_rowsets->size(), 2); + for (const auto& rowset : dropped_rowsets.value()) { + auto probe = probe_rowset(rowset); + ASSERT_TRUE(probe.has_value()) << probe.error(); + expect_index_files(probe.value(), false); + } + + auto reloaded_dropped_rowsets = reload_rowsets(dropped_rowsets.value()); + ASSERT_TRUE(reloaded_dropped_rowsets.has_value()) << reloaded_dropped_rowsets.error(); + auto after_drop = read_rowsets(reloaded_dropped_rowsets.value(), read_options); + ASSERT_TRUE(after_drop.has_value()) << after_drop.error(); + EXPECT_EQ(after_drop->rows_read, 2); + EXPECT_EQ(after_drop->stats.rows_inverted_index_filtered, 0); + expect_inverted_index_not_attempted(after_drop.value()); + + auto missing_after = std::find_if( + after_drop->stats.index_probe_events.begin(), + after_drop->stats.index_probe_events.end(), [](const auto& e) { + return e.state == IndexProbeState::NOT_ATTEMPTED && + e.reason == IndexFallbackReason::MISSING_INDEX && e.column_uid == 2 && + e.variant_path.has_value() && e.variant_path.value() == "b"; + }); + ASSERT_NE(missing_after, after_drop->stats.index_probe_events.end()); +} + +TEST_F(IndexStorageLifecycleTest, DropVariantPathIndexWithoutMaterializedPathRemovesMetadata) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + + IndexTabletOptions options; + options.tablet_id = 110021; + options.variant_columns = {std::move(variant)}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back( + VariantJsonBatch::single_variant({R"({"b": "one"})", R"({"b": "other"})"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + const auto missing_path_index = + IndexSpec::field_pattern_index(20005, "idx_v_missing", 2, "missing"); + auto built_rowsets = build_inverted_indexes({missing_path_index}); + ASSERT_TRUE(built_rowsets.has_value()) << built_rowsets.error(); + ASSERT_EQ(built_rowsets->size(), 1); + EXPECT_FALSE(built_rowsets->front() + ->tablet_schema() + ->inverted_index_by_field_pattern(2, "missing") + .empty()); + + auto dropped_rowsets = drop_inverted_indexes({missing_path_index}); + ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error(); + ASSERT_EQ(dropped_rowsets->size(), 1); + EXPECT_TRUE(dropped_rowsets->front() + ->tablet_schema() + ->inverted_index_by_field_pattern(2, "missing") + .empty()); +} + +TEST_F(IndexStorageLifecycleTest, DropOneVariantPathIndexKeepsSiblingPathIndexUsable) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + variant.predefined_paths = { + VariantPathSpec {.path = "b", .type = FieldType::OLAP_FIELD_TYPE_STRING}, + VariantPathSpec {.path = "c", .type = FieldType::OLAP_FIELD_TYPE_STRING}, + }; + + const auto index_case = + IndexStorageCaseBuilder("drop_one_variant_path_index_keeps_sibling_path_index_usable") + .tablet_id(110022) + .variant_column(std::move(variant)) + .rowset(0, + IndexDataSourceSpec::inline_variant({R"({"b": "drop", "c": "keep"})", + R"({"b": "other", "c": "other"})"}, + 0)) + .rowset(1, + IndexDataSourceSpec::inline_variant({R"({"b": "other", "c": "other"})", + R"({"b": "drop", "c": "keep"})"}, + 100)) + .build(); + ASSERT_TRUE(create_tablet(index_case.tablet_options).ok()); + auto rowsets = write_rowsets(index_case.rowsets); + ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); + + auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value()); + ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error(); + + const int32_t b_column_id = column_id_by_path("v.b"); + const int32_t c_column_id = column_id_by_path("v.c"); + ASSERT_GE(b_column_id, 0); + ASSERT_GE(c_column_id, 0); + const auto& b_column = tablet_schema()->column(b_column_id); + const auto& c_column = tablet_schema()->column(c_column_id); + + IndexReadOptions b_read_options; + b_read_options.return_columns = {0, static_cast(b_column_id)}; + b_read_options.target_cast_type_for_variants[b_column.name()] = + std::make_shared(); + b_read_options.predicates.push_back(string_equals(b_column_id, b_column.name(), "drop")); + + IndexReadOptions c_read_options; + c_read_options.return_columns = {0, static_cast(c_column_id)}; + c_read_options.target_cast_type_for_variants[c_column.name()] = + std::make_shared(); + c_read_options.predicates.push_back(string_equals(c_column_id, c_column.name(), "keep")); + + const auto b_index = IndexSpec::field_pattern_index(20006, "idx_v_b_drop", 2, "b"); + const auto c_index = IndexSpec::field_pattern_index(20007, "idx_v_c_keep", 2, "c"); + auto built_rowsets = build_inverted_indexes({b_index, c_index}); + ASSERT_TRUE(built_rowsets.has_value()) << built_rowsets.error(); + ASSERT_EQ(built_rowsets->size(), 2); + for (const auto& rowset : built_rowsets.value()) { + auto probe = probe_rowset(rowset); + ASSERT_TRUE(probe.has_value()) << probe.error(); + expect_index_files(probe.value(), true); + } + + auto reloaded_built_rowsets = reload_rowsets(built_rowsets.value()); + ASSERT_TRUE(reloaded_built_rowsets.has_value()) << reloaded_built_rowsets.error(); + auto b_after_build = read_rowsets(reloaded_built_rowsets.value(), b_read_options); + ASSERT_TRUE(b_after_build.has_value()) << b_after_build.error(); + EXPECT_EQ(b_after_build->rows_read, 2); + expect_applied_variant_path_index(b_after_build.value(), "b", 20006, 2); + auto c_after_build = read_rowsets(reloaded_built_rowsets.value(), c_read_options); + ASSERT_TRUE(c_after_build.has_value()) << c_after_build.error(); + EXPECT_EQ(c_after_build->rows_read, 2); + expect_applied_variant_path_index(c_after_build.value(), "c", 20007, 2); + + auto dropped_rowsets = drop_inverted_indexes({b_index}); + ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error(); + ASSERT_EQ(dropped_rowsets->size(), 2); + for (const auto& rowset : dropped_rowsets.value()) { + EXPECT_TRUE(rowset->tablet_schema()->inverted_index_by_field_pattern(2, "b").empty()); + EXPECT_FALSE(rowset->tablet_schema()->inverted_index_by_field_pattern(2, "c").empty()); + auto probe = probe_rowset(rowset); + ASSERT_TRUE(probe.has_value()) << probe.error(); + expect_index_files(probe.value(), true); + } + + auto reloaded_dropped_rowsets = reload_rowsets(dropped_rowsets.value()); + ASSERT_TRUE(reloaded_dropped_rowsets.has_value()) << reloaded_dropped_rowsets.error(); + auto b_after_drop = read_rowsets(reloaded_dropped_rowsets.value(), b_read_options); + ASSERT_TRUE(b_after_drop.has_value()) << b_after_drop.error(); + EXPECT_EQ(b_after_drop->rows_read, 2); + EXPECT_EQ(b_after_drop->stats.rows_inverted_index_filtered, 0); + expect_inverted_index_not_attempted(b_after_drop.value()); + + auto missing_b = std::find_if(b_after_drop->stats.index_probe_events.begin(), + b_after_drop->stats.index_probe_events.end(), [](const auto& e) { + return e.state == IndexProbeState::NOT_ATTEMPTED && + e.reason == IndexFallbackReason::MISSING_INDEX && + e.column_uid == 2 && e.variant_path.has_value() && + e.variant_path.value() == "b"; + }); + ASSERT_NE(missing_b, b_after_drop->stats.index_probe_events.end()); + + auto c_after_drop = read_rowsets(reloaded_dropped_rowsets.value(), c_read_options); + ASSERT_TRUE(c_after_drop.has_value()) << c_after_drop.error(); + EXPECT_EQ(c_after_drop->rows_read, 2); + expect_applied_variant_path_index(c_after_drop.value(), "c", 20007, 2); +} + +TEST_F(IndexStorageLifecycleTest, PatchedSchemaKeepsExistingTextIndexUsable) { + IndexTabletOptions options; + options.tablet_id = 110015; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + IndexSchemaPatch patch; + patch.add_text_columns.push_back( + TextColumnSpec {.unique_id = 3, .name = "body", .nullable = true}); + auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch); + ASSERT_NE(patched_schema, nullptr); + ASSERT_TRUE(patched_schema->has_column_unique_id(3)); + + auto patched_rowsets = rowsets_with_schema({rowset_result.value()}, std::move(patched_schema)); + ASSERT_TRUE(patched_rowsets.has_value()) << patched_rowsets.error(); + + IndexReadOptions read_options; + read_options.return_columns = {0, 1}; + read_options.predicates.push_back(title_equals("hello")); + auto read_result = read_rowsets(patched_rowsets.value(), read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 1); + expect_applied_title_index(read_result.value(), 1); +} + +TEST_F(IndexStorageLifecycleTest, RowsetsWithSchemaUpdatesTabletRowsetMapForIndexDrop) { + IndexTabletOptions options; + options.tablet_id = 110019; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + auto index = IndexSpec::column_index(20008, "idx_title_patch_drop", 2); + IndexSchemaPatch patch; + patch.add_inverted_indexes.push_back(index); + auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch); + ASSERT_NE(patched_schema, nullptr); + ASSERT_TRUE(patched_schema->has_inverted_index_with_index_id(20008)); + + auto patched_rowsets = rowsets_with_schema({rowset_result.value()}, std::move(patched_schema)); + ASSERT_TRUE(patched_rowsets.has_value()) << patched_rowsets.error(); + ASSERT_EQ(patched_rowsets->size(), 1); + ASSERT_TRUE(patched_rowsets->front()->tablet_schema()->has_inverted_index_with_index_id(20008)); + + auto dropped_rowsets = drop_inverted_indexes({index}); + ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error(); + ASSERT_EQ(dropped_rowsets->size(), 1); + EXPECT_FALSE( + dropped_rowsets->front()->tablet_schema()->has_inverted_index_with_index_id(20008)); +} + +TEST_F(IndexStorageLifecycleTest, PatchedSchemaAddDropTextColumnReadsDefaultsAndCompacts) { + IndexTabletOptions options; + options.tablet_id = 110016; + options.text_columns = { + TextColumnSpec {.unique_id = 2, .name = "title"}, + TextColumnSpec {.unique_id = 3, .name = "obsolete", .nullable = true}, + }; + options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2)); + options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_obsolete", 3)); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + VariantJsonBatch batch0; + batch0.keys = {0, 1}; + batch0.text_values_by_column = {{"hello", "other"}, {"drop-a", "drop-b"}}; + rowset0.batches.push_back(std::move(batch0)); + auto rowset0_result = write_rowset(rowset0); + ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error(); + + IndexRowsetSpec rowset1; + rowset1.version = 1; + VariantJsonBatch batch1; + batch1.keys = {2, 3}; + batch1.text_values_by_column = {{"hello", "later"}, {"drop-c", "drop-d"}}; + rowset1.batches.push_back(std::move(batch1)); + auto rowset1_result = write_rowset(rowset1); + ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error(); + + IndexSchemaPatch patch; + patch.drop_column_uids.insert(3); + patch.add_text_columns.push_back( + TextColumnSpec {.unique_id = 4, .name = "body", .nullable = true}); + auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch); + ASSERT_NE(patched_schema, nullptr); + EXPECT_FALSE(patched_schema->has_column_unique_id(3)); + ASSERT_TRUE(patched_schema->has_column_unique_id(4)); + + auto patched_rowsets = rowsets_with_schema({rowset0_result.value(), rowset1_result.value()}, + std::move(patched_schema)); + ASSERT_TRUE(patched_rowsets.has_value()) << patched_rowsets.error(); + + IndexReadOptions read_options; + read_options.collect_string_values = true; + read_options.need_ordered_result = true; + read_options.return_columns = {1, 2}; + auto read_result = read_rowsets(patched_rowsets.value(), read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 4); + ASSERT_TRUE(read_result->string_values_by_uid.contains(2)); + ASSERT_TRUE(read_result->string_values_by_uid.contains(4)); + EXPECT_EQ(read_result->string_values_by_uid.at(2), + (std::vector> {"hello", "other", "hello", "later"})); + EXPECT_EQ(read_result->string_values_by_uid.at(4), + (std::vector> {std::nullopt, std::nullopt, std::nullopt, + std::nullopt})); + + IndexReadOptions index_read_options; + index_read_options.predicates.push_back(title_equals("hello")); + auto index_read = read_rowsets(patched_rowsets.value(), index_read_options); + ASSERT_TRUE(index_read.has_value()) << index_read.error(); + EXPECT_EQ(index_read->rows_read, 2); + expect_applied_title_index(index_read.value(), 2); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, patched_rowsets.value()); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + + IndexReadOptions compacted_read_options; + compacted_read_options.collect_string_values = true; + compacted_read_options.need_ordered_result = true; + compacted_read_options.return_columns = {1, 2}; + auto compacted_read = read_rowsets({compacted.value()}, compacted_read_options); + ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error(); + EXPECT_EQ(compacted_read->rows_read, 4); + ASSERT_TRUE(compacted_read->string_values_by_uid.contains(2)); + ASSERT_TRUE(compacted_read->string_values_by_uid.contains(4)); + EXPECT_EQ(compacted_read->string_values_by_uid.at(2), + (std::vector> {"hello", "other", "hello", "later"})); + EXPECT_EQ(compacted_read->string_values_by_uid.at(4), + (std::vector> {std::nullopt, std::nullopt, std::nullopt, + std::nullopt})); +} + +TEST_F(IndexStorageLifecycleTest, PatchedSchemaAddDropVariantColumnCompactsNewRows) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + variant.max_subcolumns_count = 4; + + VariantColumnSpec obsolete; + obsolete.unique_id = 3; + obsolete.name = "old_v"; + obsolete.max_subcolumns_count = 4; + + IndexTabletOptions options; + options.tablet_id = 110017; + options.variant_columns = {std::move(variant), std::move(obsolete)}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + VariantJsonBatch batch0; + batch0.keys = {0, 1}; + batch0.variant_jsons_by_column = { + {R"({"a": "old-one"})", R"({"a": "old-two"})"}, + {R"({"gone": "drop-one"})", R"({"gone": "drop-two"})"}, + }; + rowset0.batches.push_back(std::move(batch0)); + auto rowset0_result = write_rowset(rowset0); + ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error(); + + VariantColumnSpec added; + added.unique_id = 4; + added.name = "v2"; + added.nullable = true; + added.max_subcolumns_count = 4; + added.predefined_paths = { + VariantPathSpec {.path = "x", .type = FieldType::OLAP_FIELD_TYPE_STRING}, + }; + + IndexSchemaPatch patch; + patch.drop_column_uids.insert(3); + patch.add_variant_columns.push_back(std::move(added)); + auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch); + ASSERT_NE(patched_schema, nullptr); + ASSERT_TRUE(patched_schema->has_column_unique_id(2)); + EXPECT_FALSE(patched_schema->has_column_unique_id(3)); + ASSERT_TRUE(patched_schema->has_column_unique_id(4)); + + auto patched_old_rowsets = + rowsets_with_schema({rowset0_result.value()}, std::move(patched_schema)); + ASSERT_TRUE(patched_old_rowsets.has_value()) << patched_old_rowsets.error(); + + IndexRowsetSpec rowset1; + rowset1.version = 1; + VariantJsonBatch batch1; + batch1.keys = {2, 3}; + batch1.variant_jsons_by_column = { + {R"({"a": "new-one"})", R"({"a": "new-two"})"}, + {R"({"x": "added-one"})", R"({"x": "added-two"})"}, + }; + rowset1.batches.push_back(std::move(batch1)); + auto rowset1_result = write_rowset(rowset1); + ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error(); + + auto read_result = read_rowsets({patched_old_rowsets->front(), rowset1_result.value()}); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 4); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, + {patched_old_rowsets->front(), rowset1_result.value()}); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + + auto compacted_probe = probe_rowset(compacted.value()); + ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error(); + EXPECT_TRUE(has_variant_layout(compacted_probe.value(), 4, "x")); + EXPECT_FALSE(has_variant_parent(compacted_probe.value(), 3)); + + auto compacted_read = read_rowsets({compacted.value()}); + ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error(); + EXPECT_EQ(compacted_read->rows_read, 4); +} + +TEST_F(IndexStorageLifecycleTest, MissingRequiredVariantColumnFailsExtendedInfoAggregation) { + IndexTabletOptions options; + options.tablet_id = 110018; + options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset; + rowset.version = 0; + rowset.batches.push_back(VariantJsonBatch::single_text({"hello", "other"}, 0)); + auto rowset_result = write_rowset(rowset); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + + VariantColumnSpec missing_required_variant; + missing_required_variant.unique_id = 3; + missing_required_variant.name = "required_v"; + missing_required_variant.nullable = false; + + IndexSchemaPatch patch; + patch.add_variant_columns.push_back(std::move(missing_required_variant)); + auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch); + ASSERT_NE(patched_schema, nullptr); + ASSERT_TRUE(patched_schema->has_column_unique_id(3)); + + auto patched_rowsets = rowsets_with_schema({rowset_result.value()}, std::move(patched_schema)); + ASSERT_TRUE(patched_rowsets.has_value()) << patched_rowsets.error(); + + std::unordered_map variant_extended_info; + auto status = variant_util::VariantCompactionUtil::aggregate_variant_extended_info( + patched_rowsets->front(), &variant_extended_info); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("column not found in segment"), std::string::npos) + << status.to_string(); +} + +TEST_F(IndexStorageLifecycleTest, VariantPathIndexHitAfterCumulativeCompaction) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + variant.predefined_paths = { + VariantPathSpec {.path = "b", .type = FieldType::OLAP_FIELD_TYPE_STRING}, + }; + const auto index_case = + IndexStorageCaseBuilder("variant_path_index_hit_after_cumulative_compaction") + .tablet_id(110006) + .variant_column(std::move(variant)) + .inverted_index(IndexSpec::field_pattern_index(20002, "idx_v_b", 2, "b")) + .rowset(0, IndexDataSourceSpec::inline_variant( + {R"({"b": "one"})", R"({"b": "other"})"}, 0)) + .rowset(1, IndexDataSourceSpec::variant_jsonl( + "be/test/storage/test_data/index_storage/" + "variant_path_index.jsonl", + 100, 1)) + .build(); + ASSERT_TRUE(create_tablet(index_case.tablet_options).ok()); + auto rowsets = write_rowsets(index_case.rowsets); + ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); + + auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value()); + ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error(); + const int32_t path_column_id = column_id_by_path("v.b"); + ASSERT_GE(path_column_id, 0); + const auto& path_column = tablet_schema()->column(path_column_id); + + IndexReadOptions read_options; + read_options.return_columns = {0, static_cast(path_column_id)}; + read_options.target_cast_type_for_variants[path_column.name()] = + std::make_shared(); + read_options.predicates.push_back(string_equals(path_column_id, path_column.name(), "one")); + auto read_result = read_rowsets(readable_rowsets.value(), read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 2); + expect_applied_variant_path_index(read_result.value(), "b", 20002, 1); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value()); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + + auto reloaded = reload_rowsets({compacted.value()}); + ASSERT_TRUE(reloaded.has_value()) << reloaded.error(); + auto readable_compacted = rowsets_with_variant_extended_schema(reloaded.value()); + ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error(); + const int32_t compacted_path_column_id = column_id_by_path("v.b"); + ASSERT_EQ(compacted_path_column_id, path_column_id); + auto compacted_read = read_rowsets(readable_compacted.value(), read_options); + ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error(); + EXPECT_EQ(compacted_read->rows_read, 2); + expect_applied_variant_path_index(compacted_read.value(), "b", 20002, 2); +} + +TEST_F(IndexStorageLifecycleTest, WriteReadProbeAndCumulativeCompact) { + IndexTabletOptions options; + options.tablet_id = 110002; + options.external_segment_meta = true; + options.variant_columns = {VariantColumnSpec {}}; + options.variant_columns[0].unique_id = 2; + options.variant_columns[0].name = "v"; + options.variant_columns[0].max_subcolumns_count = 4; + options.variant_columns[0].sparse_hash_shard_count = 2; + options.variant_columns[0].predefined_paths = { + VariantPathSpec {.path = "a", .type = FieldType::OLAP_FIELD_TYPE_INT}, + VariantPathSpec {.path = "b", .type = FieldType::OLAP_FIELD_TYPE_STRING}, + }; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back(VariantJsonBatch::single_variant( + {R"({"a": 1, "b": "one"})", R"({"a": 2, "c": 20})"}, 0)); + auto rowset0_result = write_rowset(rowset0); + ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error(); + + IndexRowsetSpec rowset1; + rowset1.version = 1; + rowset1.batches.push_back(VariantJsonBatch::single_variant( + {R"({"a": 3, "b": "three"})", R"({"a": 4, "d": 40})"}, 100)); + auto rowset1_result = write_rowset(rowset1); + ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error(); + + auto read_result = read_rowsets({rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + EXPECT_EQ(read_result->rows_read, 4); + + auto probe_result = probe_rowset(rowset0_result.value()); + ASSERT_TRUE(probe_result.has_value()) << probe_result.error(); + EXPECT_EQ(probe_result->num_rows, 2); + EXPECT_EQ(probe_result->num_segments, 1); + EXPECT_TRUE(probe_result->contains_relative_path("a")); + EXPECT_TRUE(probe_result->contains_relative_path("b")); + expect_index_files(probe_result.value(), false); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, + {rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + + auto compacted_probe = probe_rowset(compacted.value()); + ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error(); + EXPECT_TRUE(compacted_probe->contains_relative_path("a")); + EXPECT_TRUE(compacted_probe->contains_relative_path("b")); +} + +TEST_F(IndexStorageLifecycleTest, DeepSparseVariantCompactsWithExternalSegmentMeta) { + run_deep_sparse_variant_lifecycle(true, 110023); +} + +TEST_F(IndexStorageLifecycleTest, DeepSparseVariantCompactsWithoutExternalSegmentMeta) { + run_deep_sparse_variant_lifecycle(false, 110024); +} + +TEST_F(IndexStorageLifecycleTest, ExactSparsePathReadsHiddenChildAfterSparseStatsLimitTruncated) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + variant.max_subcolumns_count = 1; + variant.max_sparse_column_statistics_size = 2; + + IndexTabletOptions options; + options.tablet_id = 110028; + options.variant_columns = {std::move(variant)}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back( + VariantJsonBatch::single_variant({R"({"a": "hot-0", "b": "scalar-0", "d": "dense-0"})", + R"({"a": "hot-1", "b": "scalar-1", "d": "dense-1"})", + R"({"a": "hot-2", "b": "scalar-2", "d": "dense-2"})"}, + 0)); + + IndexRowsetSpec rowset1; + rowset1.version = 1; + rowset1.batches.push_back( + VariantJsonBatch::single_variant({R"({"a": "hot-3", "b": "scalar-3", "d": "dense-3"})", + R"({"a": "hot-4", "b": {"c": "child-0"}})"}, + 100)); + + auto rowsets = write_rowsets({rowset0, rowset1}); + ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value()); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + ASSERT_EQ(compacted.value()->num_rows(), 5); + + auto reloaded = reload_rowsets({compacted.value()}); + ASSERT_TRUE(reloaded.has_value()) << reloaded.error(); + + auto read_schema = build_schema_with_variant_path_column(*tablet_schema(), 2, "b", + FieldType::OLAP_FIELD_TYPE_VARIANT); + ASSERT_NE(read_schema, nullptr); + auto readable_compacted = rowsets_with_schema(reloaded.value(), std::move(read_schema)); + ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error(); + + auto compacted_probe = probe_rowset(readable_compacted->front()); + ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error(); + ASSERT_TRUE(has_variant_layout(compacted_probe.value(), 2, "a")); + ASSERT_TRUE(has_sparse_path_stat(compacted_probe.value(), "b")); + ASSERT_FALSE(has_sparse_path_stat(compacted_probe.value(), "b.c")); + + const int32_t b_column_id = column_id_by_path("v.b"); + ASSERT_GE(b_column_id, 0) << dump_schema_paths(*tablet_schema()); + const auto& b_column = tablet_schema()->column(b_column_id); + ASSERT_TRUE(b_column.is_variant_type()); + + IndexReadOptions read_options; + read_options.return_columns = {0, static_cast(b_column_id)}; + read_options.collect_variant_values = true; + auto read_result = read_rowsets(readable_compacted.value(), read_options); + ASSERT_TRUE(read_result.has_value()) << read_result.error(); + ASSERT_EQ(read_result->rows_read, 5); + ASSERT_TRUE(read_result->variant_values_by_uid.contains(b_column.unique_id())); + + const auto& b_values = read_result->variant_values_by_uid.at(b_column.unique_id()); + ASSERT_EQ(b_values.size(), 5); + const auto has_hidden_child = std::any_of(b_values.begin(), b_values.end(), [](const auto& v) { + return v.has_value() && v->find("child-0") != std::string::npos; + }); + + std::ostringstream serialized_values; + for (const auto& value : b_values) { + serialized_values << (value.has_value() ? value.value() : "NULL") << '\n'; + } + EXPECT_TRUE(has_hidden_child) << serialized_values.str(); +} + +TEST_F(IndexStorageLifecycleTest, NestedGroupVariantCompactsWithExternalSegmentMeta) { + run_nested_group_variant_lifecycle(true, 110025); +} + +TEST_F(IndexStorageLifecycleTest, NestedGroupVariantCompactsWithoutExternalSegmentMeta) { + run_nested_group_variant_lifecycle(false, 110026); +} + +TEST_F(IndexStorageLifecycleTest, VariantDocModeWritesDocValueColumnsAfterCompaction) { + VariantColumnSpec variant; + variant.unique_id = 2; + variant.name = "v"; + variant.max_subcolumns_count = 4; + variant.enable_doc_mode = true; + variant.doc_materialization_min_rows = 100000; + variant.doc_hash_shard_count = 2; + + IndexTabletOptions options; + options.tablet_id = 110012; + options.variant_columns = {std::move(variant)}; + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back(VariantJsonBatch::single_variant( + {R"({"a": "one", "b": 1})", R"({"a": "two", "c": 2})"}, 0)); + auto rowset0_result = write_rowset(rowset0); + ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error(); + + IndexRowsetSpec rowset1; + rowset1.version = 1; + rowset1.batches.push_back(VariantJsonBatch::single_variant( + {R"({"a": "three", "d": 3})", R"({"a": "four", "e": 4})"}, 100)); + auto rowset1_result = write_rowset(rowset1); + ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error(); + + auto before_compaction_read = read_rowsets({rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(before_compaction_read.has_value()) << before_compaction_read.error(); + EXPECT_EQ(before_compaction_read->rows_read, 4); + + auto rowset0_probe = probe_rowset(rowset0_result.value()); + ASSERT_TRUE(rowset0_probe.has_value()) << rowset0_probe.error(); + EXPECT_TRUE(has_doc_value_column(rowset0_probe.value())); + + auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, + {rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + EXPECT_EQ(compacted.value()->num_rows(), 4); + + auto compacted_probe = probe_rowset(compacted.value()); + ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error(); + EXPECT_TRUE(has_doc_value_column(compacted_probe.value())); + + auto compacted_read = read_rowsets({compacted.value()}); + ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error(); + EXPECT_EQ(compacted_read->rows_read, 4); +} + +TEST(IndexStorageSchemaPatchTest, AddDropIndexAndModifyVariantProperties) { + IndexTabletOptions options; + options.tablet_id = 110003; + options.variant_columns = {VariantColumnSpec {}}; + options.variant_columns[0].unique_id = 2; + options.variant_columns[0].name = "v"; + options.inverted_indexes.push_back( + IndexSpec::field_pattern_index(20001, "idx_v_old", 2, "old")); + auto base_schema = build_tablet_schema(options); + + VariantColumnSpec modified = options.variant_columns[0]; + modified.max_subcolumns_count = 16; + modified.enable_doc_mode = true; + modified.doc_materialization_min_rows = 128; + + VariantColumnSpec added; + added.unique_id = 3; + added.name = "v2"; + added.max_subcolumns_count = 2; + + IndexSchemaPatch patch; + patch.modify_variant_columns.emplace(2, modified); + patch.add_variant_columns.push_back(added); + patch.drop_index_names.insert("idx_v_old"); + patch.add_inverted_indexes.push_back( + IndexSpec::field_pattern_index(20002, "idx_v_new", 2, "a")); + + auto patched_schema = build_patched_tablet_schema(*base_schema, patch); + ASSERT_NE(patched_schema, nullptr); + + ASSERT_TRUE(patched_schema->has_column_unique_id(2)); + ASSERT_TRUE(patched_schema->has_column_unique_id(3)); + const auto& variant = patched_schema->column_by_uid(2); + EXPECT_EQ(variant.variant_max_subcolumns_count(), 16); + EXPECT_TRUE(variant.variant_enable_doc_mode()); + EXPECT_TRUE(patched_schema->inverted_index_by_field_pattern(2, "old").empty()); + auto new_indexes = patched_schema->inverted_index_by_field_pattern(2, "a"); + ASSERT_EQ(new_indexes.size(), 1); + EXPECT_EQ(new_indexes[0]->index_name(), "idx_v_new"); +} + +TEST(IndexStorageSchemaPatchTest, VariantDocAndNestedGroupOptionsArePreserved) { + IndexTabletOptions options; + options.tablet_id = 110009; + options.variant_columns = {VariantColumnSpec {}}; + options.variant_columns[0].unique_id = 2; + options.variant_columns[0].name = "v"; + options.variant_columns[0].enable_doc_mode = true; + options.variant_columns[0].doc_materialization_min_rows = 64; + options.variant_columns[0].doc_hash_shard_count = 8; + options.variant_columns[0].enable_nested_group = true; + + auto schema = build_tablet_schema(options); + ASSERT_NE(schema, nullptr); + ASSERT_TRUE(schema->has_column_unique_id(2)); + const auto& variant = schema->column_by_uid(2); + EXPECT_TRUE(variant.variant_enable_doc_mode()); + EXPECT_EQ(variant.variant_doc_materialization_min_rows(), 64); + EXPECT_EQ(variant.variant_doc_hash_shard_count(), 8); + EXPECT_TRUE(variant.variant_enable_nested_group()); +} + +TEST(IndexStorageSchemaPatchTest, AddAndDropTextColumns) { + IndexTabletOptions options; + options.tablet_id = 110014; + options.text_columns = { + TextColumnSpec {.unique_id = 2, .name = "title"}, + TextColumnSpec {.unique_id = 3, .name = "obsolete"}, + }; + options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2)); + options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_obsolete", 3)); + auto base_schema = build_tablet_schema(options); + ASSERT_NE(base_schema, nullptr); + + IndexSchemaPatch patch; + patch.drop_column_uids.insert(3); + patch.add_text_columns.push_back(TextColumnSpec {.unique_id = 4, .name = "body"}); + + auto patched_schema = build_patched_tablet_schema(*base_schema, patch); + ASSERT_NE(patched_schema, nullptr); + + ASSERT_TRUE(patched_schema->has_column_unique_id(2)); + EXPECT_FALSE(patched_schema->has_column_unique_id(3)); + ASSERT_TRUE(patched_schema->has_column_unique_id(4)); + const auto& body = patched_schema->column_by_uid(4); + EXPECT_EQ(body.name(), "body"); + EXPECT_EQ(body.type(), FieldType::OLAP_FIELD_TYPE_STRING); + EXPECT_EQ(patched_schema->next_column_unique_id(), 5); + EXPECT_TRUE(patched_schema->inverted_index_by_field_pattern(3, "").empty()); +} + +} // namespace doris::index_storage_test diff --git a/be/test/storage/test_data/index_storage/variant_path_index.jsonl b/be/test/storage/test_data/index_storage/variant_path_index.jsonl new file mode 100644 index 00000000000000..296f047ef61343 --- /dev/null +++ b/be/test/storage/test_data/index_storage/variant_path_index.jsonl @@ -0,0 +1,2 @@ +{"b": "one"} +{"b": "other"} diff --git a/be/test/testutil/index_storage_test_util.cpp b/be/test/testutil/index_storage_test_util.cpp new file mode 100644 index 00000000000000..703b4bb86b6336 --- /dev/null +++ b/be/test/testutil/index_storage_test_util.cpp @@ -0,0 +1,1452 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "testutil/index_storage_test_util.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_variant.h" +#include "exec/common/variant_util.h" +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "storage/compaction/cumulative_compaction.h" +#include "storage/compaction/full_compaction.h" +#include "storage/data_dir.h" +#include "storage/index/index_writer.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/options.h" +#include "storage/rowset/rowset_factory.h" +#include "storage/rowset/rowset_meta.h" +#include "storage/rowset/rowset_reader.h" +#include "storage/rowset/rowset_writer.h" +#include "storage/rowset/rowset_writer_context.h" +#include "storage/segment/segment.h" +#include "storage/storage_engine.h" +#include "storage/tablet/tablet_meta.h" +#include "storage/task/index_builder.h" + +namespace doris::index_storage_test { +namespace { + +constexpr uint32_t kMaxPathLen = 1024; + +#define RETURN_RESULT_IF_ERROR(stmt) \ + do { \ + Status _status_ = (stmt); \ + if (UNLIKELY(!_status_.ok())) { \ + return ResultError(std::move(_status_)); \ + } \ + } while (false) + +std::string escape_for_variant_index_suffix(const std::string& value) { + auto hex_digit = [](int value) -> char { + return value < 10 ? static_cast('0' + value) : static_cast('A' + value - 10); + }; + + std::string escaped; + escaped.reserve(value.size()); + for (unsigned char ch : value) { + if (ch == '.' || ch == '_') { + escaped.push_back('%'); + escaped.push_back(hex_digit(ch / 16)); + escaped.push_back(hex_digit(ch % 16)); + } else { + escaped.push_back(static_cast(ch)); + } + } + return escaped; +} + +std::string sanitize_test_name(std::string name) { + for (auto& ch : name) { + if (!std::isalnum(static_cast(ch))) { + ch = '_'; + } + } + return name; +} + +std::string current_working_dir() { + char buffer[kMaxPathLen]; + CHECK(getcwd(buffer, kMaxPathLen) != nullptr); + return std::string(buffer); +} + +void init_key_column(ColumnPB* column_pb) { + column_pb->set_unique_id(1); + column_pb->set_name("k"); + column_pb->set_type("INT"); + column_pb->set_is_key(true); + column_pb->set_length(4); + column_pb->set_index_length(4); + column_pb->set_is_nullable(false); + column_pb->set_is_bf_column(false); +} + +void init_text_column(ColumnPB* column_pb, const TextColumnSpec& spec) { + column_pb->set_unique_id(spec.unique_id); + column_pb->set_name(spec.name); + column_pb->set_type(TabletColumn::get_string_by_field_type(spec.type)); + column_pb->set_is_key(false); + column_pb->set_is_nullable(spec.nullable); + column_pb->set_is_bf_column(false); +} + +void init_variant_column(ColumnPB* column_pb, const VariantColumnSpec& spec) { + column_pb->set_unique_id(spec.unique_id); + column_pb->set_name(spec.name); + column_pb->set_type("VARIANT"); + column_pb->set_is_key(false); + column_pb->set_is_nullable(spec.nullable); + column_pb->set_variant_max_subcolumns_count(spec.max_subcolumns_count); + column_pb->set_variant_max_sparse_column_statistics_size( + spec.max_sparse_column_statistics_size); + column_pb->set_variant_sparse_hash_shard_count(spec.sparse_hash_shard_count); + column_pb->set_variant_enable_doc_mode(spec.enable_doc_mode); + column_pb->set_variant_doc_materialization_min_rows(spec.doc_materialization_min_rows); + if (spec.doc_hash_shard_count > 0) { + column_pb->set_variant_doc_hash_shard_count(spec.doc_hash_shard_count); + } + column_pb->set_variant_enable_nested_group(spec.enable_nested_group); +} + +bool is_text_field_type(FieldType type) { + return type == FieldType::OLAP_FIELD_TYPE_STRING || + type == FieldType::OLAP_FIELD_TYPE_VARCHAR || type == FieldType::OLAP_FIELD_TYPE_CHAR; +} + +std::string relative_variant_path(std::string full_path) { + auto dot_pos = full_path.find('.'); + if (dot_pos == std::string::npos) { + return full_path; + } + return full_path.substr(dot_pos + 1); +} + +TabletColumn make_predefined_path_template(const std::string&, const VariantPathSpec& path_spec, + int32_t parent_unique_id) { + ColumnPB column_pb; + column_pb.set_unique_id(-1); + column_pb.set_name(path_spec.path); + column_pb.set_type(TabletColumn::get_string_by_field_type(path_spec.type)); + column_pb.set_is_key(false); + column_pb.set_is_nullable(path_spec.nullable); + column_pb.set_pattern_type(path_spec.pattern_type); + + TabletColumn subcolumn; + subcolumn.init_from_pb(column_pb); + subcolumn.set_parent_unique_id(parent_unique_id); + return subcolumn; +} + +void append_predefined_paths(TabletSchema* schema, const IndexTabletOptions& options) { + for (const auto& column_spec : options.variant_columns) { + if (column_spec.predefined_paths.empty()) { + continue; + } + TabletColumn& variant = schema->mutable_column_by_uid(column_spec.unique_id); + for (const auto& path_spec : column_spec.predefined_paths) { + TabletColumn subcolumn = make_predefined_path_template(column_spec.name, path_spec, + column_spec.unique_id); + variant.add_sub_column(subcolumn); + } + } +} + +bool index_references_any_column(const TabletIndexPB& index, const std::set& column_uids) { + return std::any_of(index.col_unique_id().begin(), index.col_unique_id().end(), + [&](int32_t uid) { return column_uids.contains(uid); }); +} + +TQueryOptions make_read_query_options(const IndexReadOptions& options) { + TQueryOptions query_options; + query_options.__set_enable_inverted_index_query(options.enable_inverted_index_query); + query_options.__set_enable_fallback_on_missing_inverted_index( + options.enable_fallback_on_missing_inverted_index); + query_options.__set_enable_no_need_read_data_opt(options.enable_no_need_read_data_opt); + query_options.__set_enable_common_expr_pushdown(options.enable_common_expr_pushdown); + query_options.__set_enable_inverted_index_query_cache( + options.enable_inverted_index_query_cache); + query_options.__set_enable_file_cache(false); + query_options.__set_disable_file_cache(true); + query_options.__set_enable_segment_cache(false); + query_options.__set_batch_size(1024); + return query_options; +} + +void append_index(TabletSchemaPB* schema_pb, const IndexSpec& spec) { + auto* index_pb = schema_pb->add_index(); + index_pb->set_index_id(spec.index_id); + index_pb->set_index_name(spec.name); + index_pb->set_index_type(IndexType::INVERTED); + index_pb->add_col_unique_id(spec.column_uid); + if (!spec.suffix_path.empty()) { + index_pb->set_index_suffix_name(escape_for_variant_index_suffix(spec.suffix_path)); + } + auto* properties = index_pb->mutable_properties(); + for (const auto& [key, value] : spec.properties) { + (*properties)[key] = value; + } +} + +TOlapTableIndex to_alter_index(const TabletSchema& schema, const IndexSpec& spec) { + TOlapTableIndex index; + index.__set_index_id(spec.index_id); + index.__set_index_name(spec.name); + index.__set_index_type(TIndexType::INVERTED); + if (!spec.properties.empty()) { + index.__set_properties(spec.properties); + } + + std::string column_name; + if (schema.has_column_unique_id(spec.column_uid)) { + column_name = schema.column_by_uid(spec.column_uid).name(); + } + index.__set_columns({column_name}); + index.__set_column_unique_ids({spec.column_uid}); + return index; +} + +Result> current_rowsets_for_versions( + Tablet& tablet, const std::vector& versions) { + std::vector rowsets; + rowsets.reserve(versions.size()); + std::shared_lock lock(tablet.get_header_lock()); + for (const auto& version : versions) { + const auto& rowset_map = tablet.rowset_map(); + auto it = rowset_map.find(version); + if (it == rowset_map.end()) { + return ResultError(Status::InternalError( + "rowset version {} not found after index builder", version.to_string())); + } + rowsets.push_back(it->second); + } + std::sort(rowsets.begin(), rowsets.end(), Rowset::comparator); + return rowsets; +} + +std::filesystem::path repo_root_from_this_source_file() { + std::filesystem::path source_path(__FILE__); + if (source_path.is_relative()) { + source_path = std::filesystem::absolute(source_path); + } + for (int i = 0; i < 4; ++i) { + source_path = source_path.parent_path(); + } + return source_path; +} + +std::ifstream open_jsonl_file(const std::string& file_path, std::string* opened_path) { + std::vector candidates; + const std::filesystem::path input_path(file_path); + candidates.push_back(input_path); + if (!input_path.is_absolute()) { + if (const char* doris_home = std::getenv("DORIS_HOME"); doris_home != nullptr) { + candidates.push_back(std::filesystem::path(doris_home) / input_path); + } + candidates.push_back(repo_root_from_this_source_file() / input_path); + } + + for (const auto& candidate : candidates) { + std::ifstream input(candidate); + if (input.is_open()) { + *opened_path = candidate.string(); + return input; + } + } + *opened_path = file_path; + return {}; +} + +Result> batches_from_data_source( + const IndexDataSourceSpec& data_source) { + switch (data_source.kind) { + case IndexDataSourceKind::INLINE_TEXT_ROWS: + return std::vector { + VariantJsonBatch::single_text(data_source.rows, data_source.first_key)}; + case IndexDataSourceKind::INLINE_VARIANT_ROWS: + return std::vector { + VariantJsonBatch::single_variant(data_source.rows, data_source.first_key)}; + case IndexDataSourceKind::VARIANT_JSONL_FILE: { + if (data_source.batch_size == 0) { + return ResultError(Status::InvalidArgument("jsonl batch size must be positive")); + } + + std::string opened_path; + std::ifstream input = open_jsonl_file(data_source.file_path, &opened_path); + if (!input.is_open()) { + return ResultError( + Status::InvalidArgument("failed to open jsonl file {}", data_source.file_path)); + } + + std::vector batches; + std::vector rows; + rows.reserve(data_source.batch_size); + std::string line; + int32_t next_key = data_source.first_key; + while (std::getline(input, line)) { + if (line.empty()) { + continue; + } + rows.push_back(std::move(line)); + if (rows.size() == data_source.batch_size) { + batches.push_back(VariantJsonBatch::single_variant(std::move(rows), next_key)); + next_key += static_cast(batches.back().num_rows()); + rows.clear(); + rows.reserve(data_source.batch_size); + } + } + if (!rows.empty()) { + batches.push_back(VariantJsonBatch::single_variant(std::move(rows), next_key)); + } + if (batches.empty()) { + return ResultError(Status::InvalidArgument("jsonl file {} has no rows", opened_path)); + } + return batches; + } + } + + return ResultError(Status::InvalidArgument("unknown index data source kind")); +} + +Result> materialize_batches(const IndexRowsetSpec& spec) { + std::vector batches = spec.batches; + for (const auto& data_source : spec.data_sources) { + auto materialized = batches_from_data_source(data_source); + if (!materialized.has_value()) { + return ResultError(materialized.error()); + } + batches.insert(batches.end(), std::make_move_iterator(materialized->begin()), + std::make_move_iterator(materialized->end())); + } + return batches; +} + +void ensure_variant_json_shape(const VariantJsonBatch& batch, size_t column_pos) { + CHECK(!batch.variant_jsons_by_column.empty()); + CHECK_LT(column_pos, batch.variant_jsons_by_column.size()); + const size_t rows = batch.num_rows(); + for (const auto& jsons : batch.variant_jsons_by_column) { + CHECK_EQ(jsons.size(), rows); + } +} + +Status fill_variant_column(const VariantColumnSpec& column_spec, const VariantJsonBatch& batch, + size_t column_pos, MutableColumnPtr* output) { + ensure_variant_json_shape(batch, column_pos); + auto variant_column = + ColumnVariant::create(column_spec.max_subcolumns_count, column_spec.enable_doc_mode); + auto json_column = ColumnString::create(); + for (const auto& json : batch.variant_jsons_by_column[column_pos]) { + json_column->insert_data(json.data(), json.size()); + } + + ParseConfig config; + config.deprecated_enable_flatten_nested = batch.deprecated_enable_flatten_nested; + config.check_duplicate_json_path = batch.check_duplicate_json_path; + config.parse_to = + column_spec.enable_doc_mode ? ParseConfig::ParseTo::OnlyDocValueColumn : batch.parse_to; + variant_util::parse_json_to_variant(*variant_column, *json_column, config); + if (column_spec.nullable) { + auto null_map = ColumnUInt8::create(); + null_map->insert_many_defaults(variant_column->size()); + *output = ColumnNullable::create(std::move(variant_column), std::move(null_map)); + } else { + *output = std::move(variant_column); + } + return Status::OK(); +} + +Status fill_text_column(const VariantJsonBatch& batch, size_t column_pos, + MutableColumnPtr* output) { + if (column_pos >= batch.text_values_by_column.size()) { + return Status::InvalidArgument("text batch column count mismatch: column_pos={}", + column_pos); + } + const auto& values = batch.text_values_by_column[column_pos]; + auto text_column = ColumnString::create(); + for (const auto& value : values) { + text_column->insert_data(value.data(), value.size()); + } + *output = std::move(text_column); + return Status::OK(); +} + +Status fill_block(const TabletSchema& schema, const IndexTabletOptions& tablet_options, + const VariantJsonBatch& batch, int32_t* next_key, Block* block) { + auto columns = std::move(*block).mutate_columns(); + size_t column_pos = 0; + const size_t rows = batch.num_rows(); + + if (tablet_options.include_key_column) { + for (size_t row = 0; row < rows; ++row) { + int32_t key = batch.keys.empty() ? (*next_key)++ : batch.keys[row]; + columns[column_pos]->insert_data(reinterpret_cast(&key), sizeof(key)); + } + ++column_pos; + } + + for (size_t text_pos = 0; text_pos < tablet_options.text_columns.size(); + ++text_pos, ++column_pos) { + MutableColumnPtr text_column; + RETURN_IF_ERROR(fill_text_column(batch, text_pos, &text_column)); + columns[column_pos] = std::move(text_column); + } + + for (size_t variant_pos = 0; variant_pos < tablet_options.variant_columns.size(); + ++variant_pos, ++column_pos) { + MutableColumnPtr variant_column; + RETURN_IF_ERROR(fill_variant_column(tablet_options.variant_columns[variant_pos], batch, + variant_pos, &variant_column)); + columns[column_pos] = std::move(variant_column); + } + + CHECK_EQ(column_pos, schema.num_columns()); + block->set_columns(std::move(columns)); + return Status::OK(); +} + +void collect_string_values_from_block(const TabletSchema& schema, + const std::vector& return_columns, + const Block& block, IndexReadResult* result) { + for (size_t pos = 0; pos < return_columns.size() && pos < block.columns(); ++pos) { + const auto cid = return_columns[pos]; + if (cid >= schema.num_columns()) { + continue; + } + const auto& tablet_column = schema.column(cid); + if (!is_text_field_type(tablet_column.type())) { + continue; + } + auto full_column = block.get_by_position(pos).column->convert_to_full_column_if_const(); + auto& values = result->string_values_by_uid[tablet_column.unique_id()]; + for (size_t row = 0; row < full_column->size(); ++row) { + if (full_column->is_null_at(row)) { + values.push_back(std::nullopt); + } else { + values.push_back(full_column->get_data_at(row).to_string()); + } + } + } +} + +void collect_variant_values_from_block(const TabletSchema& schema, + const std::vector& return_columns, + const Block& block, IndexReadResult* result) { + DataTypeSerDe::FormatOptions options; + auto tz = cctz::utc_time_zone(); + options.timezone = &tz; + + for (size_t pos = 0; pos < return_columns.size() && pos < block.columns(); ++pos) { + const auto cid = return_columns[pos]; + if (cid >= schema.num_columns()) { + continue; + } + const auto& tablet_column = schema.column(cid); + if (!tablet_column.is_variant_type()) { + continue; + } + + auto full_column = block.get_by_position(pos).column->convert_to_full_column_if_const(); + const auto* nullable_column = check_and_get_column(*full_column); + const auto* variant_column = + nullable_column != nullptr + ? assert_cast(&nullable_column->get_nested_column()) + : assert_cast(full_column.get()); + auto& values = result->variant_values_by_uid[tablet_column.unique_id()]; + for (size_t row = 0; row < full_column->size(); ++row) { + if (full_column->is_null_at(row)) { + values.push_back(std::nullopt); + continue; + } + std::string value; + variant_column->serialize_one_row_to_string(row, &value, options); + values.emplace_back(std::move(value)); + } + } +} + +void collect_variant_column_layout(const ColumnMetaPB& column_meta, IndexSegmentLayout* layout) { + if (column_meta.has_column_path_info()) { + PathInData path; + path.from_protobuf(column_meta.column_path_info()); + + IndexColumnLayout column_layout; + column_layout.parent_unique_id = column_meta.column_path_info().parrent_column_unique_id(); + column_layout.full_path = path.get_path(); + column_layout.relative_path = relative_variant_path(column_layout.full_path); + column_layout.none_null_size = column_meta.none_null_size(); + column_layout.is_sparse_column = + column_layout.full_path.find("__DORIS_VARIANT_SPARSE__") != std::string::npos; + column_layout.is_doc_value_column = + column_layout.full_path.find("__DORIS_VARIANT_DOC_VALUE__") != std::string::npos; + if (column_meta.has_variant_statistics()) { + for (const auto& [sub_path, non_null_size] : + column_meta.variant_statistics().sparse_column_non_null_size()) { + column_layout.sparse_non_null_size[sub_path] = non_null_size; + } + } + layout->variant_columns.push_back(std::move(column_layout)); + } + + for (const auto& child_meta : column_meta.children_columns()) { + collect_variant_column_layout(child_meta, layout); + } +} + +Result probe_segment(const RowsetSharedPtr& rowset, int64_t segment_id) { + auto segment_path = rowset->segment_path(segment_id); + if (!segment_path.has_value()) { + return ResultError(segment_path.error()); + } + + OlapReaderStatistics stats; + std::shared_ptr segment; + auto status = segment_v2::Segment::open( + rowset->rowset_meta()->fs(), segment_path.value(), rowset->rowset_meta()->tablet_id(), + static_cast(segment_id), rowset->rowset_id(), rowset->tablet_schema(), + io::FileReaderOptions {}, &segment, + rowset->rowset_meta()->inverted_index_file_info(static_cast(segment_id)), &stats); + if (!status.ok()) { + return ResultError(status); + } + + IndexSegmentLayout layout; + layout.segment_id = segment_id; + layout.num_rows = segment->num_rows(); + + status = segment->traverse_column_meta_pbs([&](const ColumnMetaPB& column_meta) { + collect_variant_column_layout(column_meta, &layout); + }); + if (!status.ok()) { + return ResultError(status); + } + return layout; +} + +Result probe_index_files(const RowsetSharedPtr& rowset) { + IndexFileProbe probe; + probe.rowset_meta_entries = rowset->rowset_meta()->inverted_index_file_info().size(); + + for (const auto& info : rowset->rowset_meta()->inverted_index_file_info()) { + if (info.has_index_size()) { + probe.rowset_meta_index_size += info.index_size(); + } + } + + if (!rowset->tablet_schema()->has_inverted_index()) { + return probe; + } + + const auto file_names = rowset->get_index_file_names(); + probe.expected_files = file_names.size(); + for (const auto& file_name : file_names) { + const std::string path = rowset->tablet_path() + "/" + file_name; + bool exists = false; + auto status = rowset->rowset_meta()->fs()->exists(path, &exists); + if (!status.ok()) { + return ResultError(status); + } + if (exists) { + ++probe.existing_files; + } else { + probe.missing_files.push_back(path); + } + } + return probe; +} + +} // namespace + +IndexSpec IndexSpec::column_index(int64_t index_id, std::string name, int32_t column_uid, + std::map properties) { + IndexSpec spec; + spec.index_id = index_id; + spec.name = std::move(name); + spec.column_uid = column_uid; + spec.properties = std::move(properties); + return spec; +} + +IndexSpec IndexSpec::field_pattern_index(int64_t index_id, std::string name, int32_t column_uid, + std::string field_pattern) { + IndexSpec spec; + spec.index_id = index_id; + spec.name = std::move(name); + spec.column_uid = column_uid; + spec.properties["field_pattern"] = std::move(field_pattern); + return spec; +} + +VariantJsonBatch VariantJsonBatch::single_text(std::vector values, int32_t first_key) { + VariantJsonBatch batch; + batch.keys.reserve(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + batch.keys.push_back(first_key + static_cast(i)); + } + batch.text_values_by_column.emplace_back(std::move(values)); + return batch; +} + +VariantJsonBatch VariantJsonBatch::single_variant(std::vector jsons, + int32_t first_key) { + VariantJsonBatch batch; + batch.keys.reserve(jsons.size()); + for (size_t i = 0; i < jsons.size(); ++i) { + batch.keys.push_back(first_key + static_cast(i)); + } + batch.variant_jsons_by_column.emplace_back(std::move(jsons)); + return batch; +} + +size_t VariantJsonBatch::num_rows() const { + if (!keys.empty()) { + return keys.size(); + } + if (!text_values_by_column.empty()) { + return text_values_by_column.front().size(); + } + if (!variant_jsons_by_column.empty()) { + return variant_jsons_by_column.front().size(); + } + return 0; +} + +IndexDataSourceSpec IndexDataSourceSpec::inline_text(std::vector values, + int32_t first_key) { + IndexDataSourceSpec spec; + spec.kind = IndexDataSourceKind::INLINE_TEXT_ROWS; + spec.rows = std::move(values); + spec.first_key = first_key; + return spec; +} + +IndexDataSourceSpec IndexDataSourceSpec::inline_variant(std::vector jsons, + int32_t first_key) { + IndexDataSourceSpec spec; + spec.kind = IndexDataSourceKind::INLINE_VARIANT_ROWS; + spec.rows = std::move(jsons); + spec.first_key = first_key; + return spec; +} + +IndexDataSourceSpec IndexDataSourceSpec::variant_jsonl(std::string file_path, int32_t first_key, + size_t batch_size) { + IndexDataSourceSpec spec; + spec.kind = IndexDataSourceKind::VARIANT_JSONL_FILE; + spec.file_path = std::move(file_path); + spec.first_key = first_key; + spec.batch_size = batch_size; + return spec; +} + +IndexStorageCaseBuilder::IndexStorageCaseBuilder(std::string name) { + _case.name = std::move(name); +} + +IndexStorageCaseBuilder& IndexStorageCaseBuilder::tablet_id(int64_t tablet_id) { + _case.tablet_options.tablet_id = tablet_id; + return *this; +} + +IndexStorageCaseBuilder& IndexStorageCaseBuilder::text_column(TextColumnSpec column) { + _case.tablet_options.text_columns.push_back(std::move(column)); + return *this; +} + +IndexStorageCaseBuilder& IndexStorageCaseBuilder::variant_column(VariantColumnSpec column) { + _case.tablet_options.variant_columns.push_back(std::move(column)); + return *this; +} + +IndexStorageCaseBuilder& IndexStorageCaseBuilder::inverted_index(IndexSpec index) { + _case.tablet_options.inverted_indexes.push_back(std::move(index)); + return *this; +} + +IndexStorageCaseBuilder& IndexStorageCaseBuilder::rowset(int64_t version, + IndexDataSourceSpec data_source, + int64_t max_rows_per_segment) { + IndexRowsetSpec rowset; + rowset.version = version; + rowset.max_rows_per_segment = max_rows_per_segment; + rowset.data_sources.push_back(std::move(data_source)); + _case.rowsets.push_back(std::move(rowset)); + return *this; +} + +IndexStorageCase IndexStorageCaseBuilder::build() const { + return _case; +} + +bool IndexSegmentLayout::contains_relative_path(const std::string& path) const { + return std::any_of(variant_columns.begin(), variant_columns.end(), + [&](const auto& column) { return column.relative_path == path; }); +} + +bool IndexRowsetProbe::contains_relative_path(const std::string& path) const { + return std::any_of(segments.begin(), segments.end(), + [&](const auto& segment) { return segment.contains_relative_path(path); }); +} + +bool IndexReadResult::inverted_index_used() const { + return std::any_of(stats.index_probe_events.begin(), stats.index_probe_events.end(), + [](const auto& event) { return event.state == IndexProbeState::APPLIED; }); +} + +bool IndexReadResult::inverted_index_attempted() const { + return std::any_of(stats.index_probe_events.begin(), stats.index_probe_events.end(), + [](const auto& event) { + return event.state == IndexProbeState::APPLIED || + event.state == IndexProbeState::FALLBACK; + }); +} + +bool IndexReadResult::inverted_index_downgraded() const { + return std::any_of(stats.index_probe_events.begin(), stats.index_probe_events.end(), + [](const auto& event) { return event.state == IndexProbeState::FALLBACK; }); +} + +bool IndexReadResult::inverted_index_effective_filter() const { + return std::any_of(stats.index_probe_events.begin(), stats.index_probe_events.end(), + [](const auto& event) { + return event.state == IndexProbeState::APPLIED && + event.filtered_rows > 0; + }); +} + +TabletSchemaPB build_tablet_schema_pb(const IndexTabletOptions& options) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_num_short_key_columns(options.include_key_column ? 1 : 0); + schema_pb.set_num_rows_per_row_block(1024); + schema_pb.set_compress_kind(COMPRESS_NONE); + schema_pb.set_inverted_index_storage_format(options.index_storage_format); + + int32_t max_unique_id = 0; + if (options.include_key_column) { + init_key_column(schema_pb.add_column()); + max_unique_id = 1; + } + + for (const auto& column : options.text_columns) { + init_text_column(schema_pb.add_column(), column); + max_unique_id = std::max(max_unique_id, column.unique_id); + } + + for (const auto& column : options.variant_columns) { + init_variant_column(schema_pb.add_column(), column); + max_unique_id = std::max(max_unique_id, column.unique_id); + } + + for (const auto& index : options.inverted_indexes) { + append_index(&schema_pb, index); + } + + schema_pb.set_next_column_unique_id(max_unique_id + 1); + return schema_pb; +} + +TabletSchemaSPtr build_tablet_schema(const IndexTabletOptions& options) { + auto tablet_schema = std::make_shared(); + auto schema_pb = build_tablet_schema_pb(options); + tablet_schema->init_from_pb(schema_pb); + append_predefined_paths(tablet_schema.get(), options); + tablet_schema->set_storage_format(options.external_segment_meta + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); + return tablet_schema; +} + +TabletSchemaPB apply_schema_patch(const TabletSchema& base_schema, const IndexSchemaPatch& patch) { + TabletSchemaPB schema_pb; + base_schema.to_schema_pb(&schema_pb); + + auto existing_columns = schema_pb.column(); + schema_pb.clear_column(); + int32_t max_unique_id = 0; + for (const auto& column : existing_columns) { + if (patch.drop_column_uids.contains(column.unique_id())) { + continue; + } + auto* dst = schema_pb.add_column(); + dst->CopyFrom(column); + if (auto it = patch.modify_variant_columns.find(column.unique_id()); + it != patch.modify_variant_columns.end()) { + init_variant_column(dst, it->second); + } + max_unique_id = std::max(max_unique_id, dst->unique_id()); + } + + for (const auto& column : patch.add_text_columns) { + init_text_column(schema_pb.add_column(), column); + max_unique_id = std::max(max_unique_id, column.unique_id); + } + + for (const auto& column : patch.add_variant_columns) { + init_variant_column(schema_pb.add_column(), column); + max_unique_id = std::max(max_unique_id, column.unique_id); + } + + auto existing_indexes = schema_pb.index(); + schema_pb.clear_index(); + for (const auto& index : existing_indexes) { + if (patch.drop_index_ids.contains(index.index_id()) || + patch.drop_index_names.contains(index.index_name()) || + index_references_any_column(index, patch.drop_column_uids)) { + continue; + } + schema_pb.add_index()->CopyFrom(index); + } + for (const auto& index : patch.add_inverted_indexes) { + append_index(&schema_pb, index); + } + + schema_pb.set_next_column_unique_id( + std::max(schema_pb.next_column_unique_id(), max_unique_id + 1)); + return schema_pb; +} + +TabletSchemaSPtr build_patched_tablet_schema(const TabletSchema& base_schema, + const IndexSchemaPatch& patch) { + auto schema = std::make_shared(); + auto schema_pb = apply_schema_patch(base_schema, patch); + schema->init_from_pb(schema_pb); + schema->set_storage_format(base_schema.storage_format()); + + IndexTabletOptions path_options; + path_options.variant_columns.clear(); + for (const auto& column : patch.add_variant_columns) { + if (!column.predefined_paths.empty()) { + path_options.variant_columns.push_back(column); + } + } + for (const auto& [_, column] : patch.modify_variant_columns) { + if (!column.predefined_paths.empty()) { + path_options.variant_columns.push_back(column); + } + } + append_predefined_paths(schema.get(), path_options); + return schema; +} + +TabletSchemaSPtr build_schema_with_variant_path_column(const TabletSchema& base_schema, + int32_t parent_unique_id, + std::string relative_path, FieldType type) { + auto schema = std::make_shared(); + TabletSchemaPB schema_pb; + base_schema.to_schema_pb(&schema_pb); + schema->init_from_pb(schema_pb); + schema->set_storage_format(base_schema.storage_format()); + + const auto& parent_column = schema->column_by_uid(parent_unique_id); + const std::string full_path = relative_path.find('.') == std::string::npos + ? parent_column.name_lower_case() + "." + relative_path + : std::move(relative_path); + + TabletColumn path_column; + path_column.set_unique_id(-1); + path_column.set_name(full_path); + path_column.set_type(type); + path_column.set_parent_unique_id(parent_column.unique_id()); + path_column.set_path_info(PathInData(full_path)); + path_column.set_aggregation_method(parent_column.aggregation()); + path_column.set_variant_max_subcolumns_count(parent_column.variant_max_subcolumns_count()); + path_column.set_variant_max_sparse_column_statistics_size( + parent_column.variant_max_sparse_column_statistics_size()); + path_column.set_variant_sparse_hash_shard_count( + parent_column.variant_sparse_hash_shard_count()); + path_column.set_variant_enable_doc_mode(parent_column.variant_enable_doc_mode()); + path_column.set_variant_doc_materialization_min_rows( + parent_column.variant_doc_materialization_min_rows()); + path_column.set_variant_doc_hash_shard_count(parent_column.variant_doc_hash_shard_count()); + path_column.set_variant_enable_nested_group(parent_column.variant_enable_nested_group()); + path_column.set_is_nullable(true); + schema->append_column(std::move(path_column)); + return schema; +} + +void expect_index_filter_stats(const IndexReadResult& result, int64_t expected_filtered_rows) { + EXPECT_EQ(result.stats.rows_inverted_index_filtered, expected_filtered_rows); + int64_t event_filtered_rows = 0; + for (const auto& event : result.stats.index_probe_events) { + if (event.state == IndexProbeState::APPLIED) { + event_filtered_rows += event.filtered_rows; + } + } + EXPECT_EQ(event_filtered_rows, expected_filtered_rows); +} + +void expect_inverted_index_used(const IndexReadResult& result) { + EXPECT_TRUE(result.inverted_index_used()) + << "expected inverted index probe, rows_inverted_index_filtered=" + << result.stats.rows_inverted_index_filtered + << ", inverted_index_query_timer=" << result.stats.inverted_index_query_timer + << ", inverted_index_lookup_timer=" << result.stats.inverted_index_lookup_timer + << ", inverted_index_downgrade_count=" << result.stats.inverted_index_downgrade_count; + EXPECT_EQ(result.stats.inverted_index_downgrade_count, 0); +} + +void expect_inverted_index_fallback(const IndexReadResult& result) { + EXPECT_TRUE(result.inverted_index_attempted()) + << "expected fallback after an inverted index attempt"; + EXPECT_TRUE(result.inverted_index_downgraded()) + << "expected scalar fallback downgrade, rows_inverted_index_filtered=" + << result.stats.rows_inverted_index_filtered + << ", inverted_index_query_timer=" << result.stats.inverted_index_query_timer + << ", inverted_index_lookup_timer=" << result.stats.inverted_index_lookup_timer + << ", inverted_index_downgrade_count=" << result.stats.inverted_index_downgrade_count; + EXPECT_FALSE(result.inverted_index_used()); +} + +void expect_inverted_index_not_attempted(const IndexReadResult& result) { + EXPECT_FALSE(result.inverted_index_attempted()) + << "expected no inverted index attempt, rows_inverted_index_filtered=" + << result.stats.rows_inverted_index_filtered + << ", inverted_index_query_timer=" << result.stats.inverted_index_query_timer + << ", inverted_index_lookup_timer=" << result.stats.inverted_index_lookup_timer + << ", inverted_index_downgrade_count=" << result.stats.inverted_index_downgrade_count; + EXPECT_TRUE(std::any_of( + result.stats.index_probe_events.begin(), result.stats.index_probe_events.end(), + [](const auto& event) { return event.state == IndexProbeState::NOT_ATTEMPTED; })); +} + +void expect_index_files(const IndexRowsetProbe& probe, bool expected_present) { + if (expected_present) { + EXPECT_GT(probe.index_files.expected_files, 0); + EXPECT_EQ(probe.index_files.expected_files, probe.index_files.existing_files); + EXPECT_TRUE(probe.index_files.missing_files.empty()); + } else { + EXPECT_EQ(probe.index_files.expected_files, 0); + EXPECT_TRUE(probe.index_files.missing_files.empty()); + } +} + +IndexStorageTestFixture::~IndexStorageTestFixture() = default; + +void IndexStorageTestFixture::SetUp() { + const auto* test_info = testing::UnitTest::GetInstance()->current_test_info(); + const std::string test_name = + sanitize_test_name(std::string(test_info->test_suite_name()) + "_" + test_info->name()); + const std::string cwd = current_working_dir(); + _test_dir = cwd + "/ut_dir/" + test_name; + _tmp_dir = _test_dir + "/tmp"; + + ASSERT_TRUE(io::global_local_filesystem()->delete_directory(_test_dir).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(_test_dir).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(_tmp_dir).ok()); + + std::vector tmp_paths; + tmp_paths.emplace_back(_tmp_dir, 1024000000); + auto tmp_file_dirs = std::make_unique(tmp_paths); + auto status = tmp_file_dirs->init(); + ASSERT_TRUE(status.ok()) << status.to_json(); + ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs)); + + EngineOptions options; + auto engine = std::make_unique(options); + _engine_ref = engine.get(); + _data_dir = std::make_unique(*_engine_ref, _test_dir); + status = _data_dir->init(true); + ASSERT_TRUE(status.ok()) << status.to_json(); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + + constexpr int64_t inverted_index_cache_limit = 1024 * 1024 * 1024; + _inverted_index_searcher_cache = std::unique_ptr( + segment_v2::InvertedIndexSearcherCache::create_global_instance( + inverted_index_cache_limit, 1)); + _inverted_index_query_cache = std::unique_ptr( + segment_v2::InvertedIndexQueryCache::create_global_cache(inverted_index_cache_limit, + 1)); + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_inverted_index_searcher_cache.get()); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_inverted_index_query_cache.get()); +} + +void IndexStorageTestFixture::TearDown() { + _tablet.reset(); + _tablet_schema.reset(); + _data_dir.reset(); + _engine_ref = nullptr; + ExecEnv::GetInstance()->set_storage_engine(nullptr); + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(nullptr); + ExecEnv::GetInstance()->set_inverted_index_query_cache(nullptr); + _inverted_index_searcher_cache.reset(); + _inverted_index_query_cache.reset(); + static_cast(io::global_local_filesystem()->delete_directory(_test_dir)); +} + +Status IndexStorageTestFixture::create_tablet(const IndexTabletOptions& options) { + _tablet_schema = build_tablet_schema(options); + + TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); + tablet_meta->_tablet_id = options.tablet_id; + tablet_meta->set_tablet_uid(TabletUid(options.tablet_id, options.tablet_id + 1)); + _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); + RETURN_IF_ERROR(_tablet->init()); + static_cast(io::global_local_filesystem()->delete_directory(_tablet->tablet_path())); + return io::global_local_filesystem()->create_directory(_tablet->tablet_path()); +} + +Result IndexStorageTestFixture::write_rowset(const IndexRowsetSpec& spec) { + if (_tablet == nullptr || _tablet_schema == nullptr) { + return ResultError(Status::InvalidArgument("tablet is not initialized")); + } + auto batches_result = materialize_batches(spec); + if (!batches_result.has_value()) { + return ResultError(batches_result.error()); + } + auto batches = std::move(batches_result).value(); + if (batches.empty()) { + return ResultError(Status::InvalidArgument("rowset spec has no batches")); + } + + RowsetWriterContext context; + RowsetId rowset_id; + rowset_id.init(spec.version + 1000); + context.rowset_id = rowset_id; + context.rowset_type = BETA_ROWSET; + context.data_dir = _data_dir.get(); + context.rowset_state = VISIBLE; + context.tablet_schema = _tablet_schema; + context.tablet_path = _tablet->tablet_path(); + context.tablet_id = _tablet->tablet_id(); + context.tablet_uid = _tablet->tablet_uid(); + context.tablet = _tablet; + context.version = Version(spec.version, spec.version); + context.segments_overlap = NONOVERLAPPING; + context.max_rows_per_segment = spec.max_rows_per_segment; + context.write_type = spec.write_type; + + auto writer_result = RowsetFactory::create_rowset_writer(*_engine_ref, context, false); + if (!writer_result.has_value()) { + return ResultError(writer_result.error()); + } + auto rowset_writer = std::move(writer_result).value(); + + int32_t next_key = 0; + IndexTabletOptions tablet_options; + tablet_options.include_key_column = _tablet_schema->num_key_columns() > 0; + tablet_options.text_columns.clear(); + tablet_options.variant_columns.clear(); + for (int i = 0; i < _tablet_schema->num_columns(); ++i) { + const auto& column = _tablet_schema->column(i); + if (column.is_key()) { + continue; + } + if (is_text_field_type(column.type())) { + TextColumnSpec column_spec; + column_spec.unique_id = column.unique_id(); + column_spec.name = column.name(); + column_spec.type = column.type(); + column_spec.nullable = column.is_nullable(); + tablet_options.text_columns.push_back(std::move(column_spec)); + continue; + } + if (!column.is_variant_type()) { + continue; + } + VariantColumnSpec column_spec; + column_spec.unique_id = column.unique_id(); + column_spec.name = column.name(); + column_spec.nullable = column.is_nullable(); + column_spec.max_subcolumns_count = column.variant_max_subcolumns_count(); + column_spec.max_sparse_column_statistics_size = + column.variant_max_sparse_column_statistics_size(); + column_spec.sparse_hash_shard_count = column.variant_sparse_hash_shard_count(); + column_spec.enable_doc_mode = column.variant_enable_doc_mode(); + tablet_options.variant_columns.push_back(std::move(column_spec)); + } + + for (const auto& batch : batches) { + if (batch.num_rows() == 0) { + return ResultError(Status::InvalidArgument("variant json batch is empty")); + } + if (!batch.keys.empty() && batch.keys.size() != batch.num_rows()) { + return ResultError(Status::InvalidArgument("variant json batch key count mismatch")); + } + if (batch.text_values_by_column.size() != tablet_options.text_columns.size()) { + return ResultError(Status::InvalidArgument( + "text batch column count mismatch: expected={}, actual={}", + tablet_options.text_columns.size(), batch.text_values_by_column.size())); + } + if (batch.variant_jsons_by_column.size() != tablet_options.variant_columns.size()) { + return ResultError(Status::InvalidArgument( + "variant json batch column count mismatch: expected={}, actual={}", + tablet_options.variant_columns.size(), batch.variant_jsons_by_column.size())); + } + + Block block = _tablet_schema->create_block(); + RETURN_RESULT_IF_ERROR( + fill_block(*_tablet_schema, tablet_options, batch, &next_key, &block)); + RETURN_RESULT_IF_ERROR(rowset_writer->add_block(&block)); + RETURN_RESULT_IF_ERROR(rowset_writer->flush()); + } + + RowsetSharedPtr rowset; + RETURN_RESULT_IF_ERROR(rowset_writer->build(rowset)); + if (spec.add_to_tablet) { + RETURN_RESULT_IF_ERROR(_tablet->add_rowset(rowset)); + } + return rowset; +} + +Result> IndexStorageTestFixture::write_rowsets( + const std::vector& specs) { + std::vector rowsets; + rowsets.reserve(specs.size()); + for (const auto& spec : specs) { + auto rowset = write_rowset(spec); + if (!rowset.has_value()) { + return ResultError(rowset.error()); + } + rowsets.push_back(std::move(rowset).value()); + } + return rowsets; +} + +Result IndexStorageTestFixture::read_rowsets( + const std::vector& rowsets, IndexReadOptions options) { + if (_tablet_schema == nullptr) { + return ResultError(Status::InvalidArgument("tablet schema is not initialized")); + } + + IndexReadResult result; + result.stats.collect_index_probe_events = true; + std::vector return_columns = std::move(options.return_columns); + if (return_columns.empty()) { + return_columns.resize(_tablet_schema->num_columns()); + std::iota(return_columns.begin(), return_columns.end(), 0); + } + + RuntimeState runtime_state; + runtime_state.set_exec_env(ExecEnv::GetInstance()); + runtime_state.set_query_options(make_read_query_options(options)); + + for (const auto& rowset : rowsets) { + RowsetReaderSharedPtr reader; + RETURN_RESULT_IF_ERROR(rowset->create_reader(&reader)); + + RowsetReaderContext context; + context.reader_type = options.reader_type; + context.tablet_schema = _tablet_schema; + context.need_ordered_result = options.need_ordered_result; + context.return_columns = &return_columns; + context.predicates = &options.predicates; + context.stats = &result.stats; + context.target_cast_type_for_variants = options.target_cast_type_for_variants; + context.all_access_paths = options.all_access_paths; + context.predicate_access_paths = options.predicate_access_paths; + context.push_down_agg_type_opt = options.push_down_agg_type_opt; + context.runtime_state = &runtime_state; + RETURN_RESULT_IF_ERROR(reader->init(&context)); + + while (true) { + Block block = _tablet_schema->create_block_by_cids(return_columns); + auto status = reader->next_batch(&block); + if (status.is()) { + break; + } + RETURN_RESULT_IF_ERROR(status); + if (options.collect_string_values) { + collect_string_values_from_block(*_tablet_schema, return_columns, block, &result); + } + if (options.collect_variant_values) { + collect_variant_values_from_block(*_tablet_schema, return_columns, block, &result); + } + result.rows_read += block.rows(); + } + } + + result.index_probe_events.push_back(IndexProbeEventSnapshot { + .label = std::move(options.index_probe_label), + .rows_inverted_index_filtered = result.stats.rows_inverted_index_filtered, + .inverted_index_filter_timer = result.stats.inverted_index_filter_timer, + .inverted_index_query_timer = result.stats.inverted_index_query_timer, + .inverted_index_query_null_bitmap_timer = + result.stats.inverted_index_query_null_bitmap_timer, + .inverted_index_query_bitmap_copy_timer = + result.stats.inverted_index_query_bitmap_copy_timer, + .inverted_index_searcher_open_timer = result.stats.inverted_index_searcher_open_timer, + .inverted_index_searcher_search_timer = + result.stats.inverted_index_searcher_search_timer, + .inverted_index_searcher_search_init_timer = + result.stats.inverted_index_searcher_search_init_timer, + .inverted_index_searcher_search_exec_timer = + result.stats.inverted_index_searcher_search_exec_timer, + .inverted_index_downgrade_count = result.stats.inverted_index_downgrade_count, + .inverted_index_analyzer_timer = result.stats.inverted_index_analyzer_timer, + .inverted_index_lookup_timer = result.stats.inverted_index_lookup_timer, + }); + + return result; +} + +Result IndexStorageTestFixture::compact_rowsets( + IndexCompactionKind kind, const std::vector& rowsets) { + if (_tablet == nullptr) { + return ResultError(Status::InvalidArgument("tablet is not initialized")); + } + + switch (kind) { + case IndexCompactionKind::CUMULATIVE: { + CumulativeCompaction compaction(*_engine_ref, _tablet); + compaction._input_rowsets = rowsets; + RETURN_RESULT_IF_ERROR(compaction.CompactionMixin::execute_compact()); + return compaction._output_rowset; + } + case IndexCompactionKind::FULL: { + FullCompaction compaction(*_engine_ref, _tablet); + compaction._input_rowsets = rowsets; + RETURN_RESULT_IF_ERROR(compaction.CompactionMixin::execute_compact()); + return compaction._output_rowset; + } + } + + return ResultError(Status::InvalidArgument("unknown variant compaction kind")); +} + +Result IndexStorageTestFixture::reload_rowset(const RowsetSharedPtr& rowset) { + if (rowset == nullptr) { + return ResultError(Status::InvalidArgument("rowset is null")); + } + + RowsetSharedPtr reloaded; + RETURN_RESULT_IF_ERROR(RowsetFactory::create_rowset( + rowset->tablet_schema(), rowset->tablet_path(), rowset->rowset_meta(), &reloaded)); + return reloaded; +} + +Result> IndexStorageTestFixture::reload_rowsets( + const std::vector& rowsets) { + std::vector reloaded_rowsets; + reloaded_rowsets.reserve(rowsets.size()); + for (const auto& rowset : rowsets) { + auto reloaded = reload_rowset(rowset); + if (!reloaded.has_value()) { + return ResultError(reloaded.error()); + } + reloaded_rowsets.push_back(std::move(reloaded).value()); + } + if (!reloaded_rowsets.empty()) { + _tablet_schema = reloaded_rowsets.front()->tablet_schema(); + } + return reloaded_rowsets; +} + +Result> IndexStorageTestFixture::rowsets_with_schema( + const std::vector& rowsets, TabletSchemaSPtr schema) { + if (rowsets.empty()) { + return ResultError(Status::InvalidArgument("rowsets are empty")); + } + if (schema == nullptr) { + return ResultError(Status::InvalidArgument("tablet schema is null")); + } + + if (_tablet != nullptr) { + _tablet->tablet_meta()->mutable_tablet_schema()->copy_from(*schema); + _tablet_schema = _tablet->tablet_meta()->tablet_schema(); + } else { + _tablet_schema = std::move(schema); + } + + std::vector reloaded_rowsets; + reloaded_rowsets.reserve(rowsets.size()); + for (const auto& rowset : rowsets) { + if (rowset == nullptr) { + return ResultError(Status::InvalidArgument("rowset is null")); + } + RowsetMetaSharedPtr rowset_meta = std::make_shared(); + if (!rowset_meta->init(rowset->rowset_meta().get())) { + return ResultError(Status::InternalError("failed to clone rowset meta")); + } + rowset_meta->set_tablet_schema(_tablet_schema); + + RowsetSharedPtr reloaded; + RETURN_RESULT_IF_ERROR(RowsetFactory::create_rowset(_tablet_schema, rowset->tablet_path(), + rowset_meta, &reloaded)); + reloaded_rowsets.push_back(std::move(reloaded)); + } + if (_tablet != nullptr) { + std::vector to_add = reloaded_rowsets; + std::vector to_delete = rowsets; + RETURN_RESULT_IF_ERROR(_tablet->modify_rowsets(to_add, to_delete, true)); + } + return reloaded_rowsets; +} + +Result> IndexStorageTestFixture::build_inverted_indexes( + const std::vector& indexes) { + if (_tablet == nullptr || _tablet_schema == nullptr) { + return ResultError(Status::InvalidArgument("tablet is not initialized")); + } + if (indexes.empty()) { + return ResultError(Status::InvalidArgument("index builder requires at least one index")); + } + + std::set index_ids; + std::vector alter_indexes; + alter_indexes.reserve(indexes.size()); + for (const auto& index : indexes) { + index_ids.insert(index.index_id); + alter_indexes.push_back(to_alter_index(*_tablet_schema, index)); + } + + auto input_rowsets = _tablet->pick_candidate_rowsets_to_build_inverted_index(index_ids, false); + if (input_rowsets.empty()) { + return std::vector {}; + } + + std::vector versions; + versions.reserve(input_rowsets.size()); + for (const auto& rowset : input_rowsets) { + versions.push_back(rowset->version()); + } + + IndexBuilder builder(*_engine_ref, _tablet, {}, alter_indexes, false); + RETURN_RESULT_IF_ERROR(builder.init()); + RETURN_RESULT_IF_ERROR(builder.do_build_inverted_index()); + + auto rowsets = current_rowsets_for_versions(*_tablet, versions); + if (rowsets.has_value() && !rowsets->empty()) { + _tablet_schema = rowsets->front()->tablet_schema(); + } + return rowsets; +} + +Result> IndexStorageTestFixture::drop_inverted_indexes( + const std::vector& indexes) { + if (_tablet == nullptr || _tablet_schema == nullptr) { + return ResultError(Status::InvalidArgument("tablet is not initialized")); + } + if (indexes.empty()) { + return ResultError(Status::InvalidArgument("index builder requires at least one index")); + } + + std::set index_ids; + std::vector alter_indexes; + alter_indexes.reserve(indexes.size()); + for (const auto& index : indexes) { + index_ids.insert(index.index_id); + alter_indexes.push_back(to_alter_index(*_tablet_schema, index)); + } + + auto input_rowsets = _tablet->pick_candidate_rowsets_to_build_inverted_index(index_ids, true); + if (input_rowsets.empty()) { + return std::vector {}; + } + + std::vector versions; + versions.reserve(input_rowsets.size()); + for (const auto& rowset : input_rowsets) { + versions.push_back(rowset->version()); + } + + IndexBuilder builder(*_engine_ref, _tablet, {}, alter_indexes, true); + RETURN_RESULT_IF_ERROR(builder.init()); + RETURN_RESULT_IF_ERROR(builder.do_build_inverted_index()); + + auto rowsets = current_rowsets_for_versions(*_tablet, versions); + if (rowsets.has_value() && !rowsets->empty()) { + _tablet_schema = rowsets->front()->tablet_schema(); + } + return rowsets; +} + +Result IndexStorageTestFixture::probe_rowset(const RowsetSharedPtr& rowset) { + IndexRowsetProbe probe; + probe.num_rows = rowset->num_rows(); + probe.num_segments = rowset->num_segments(); + + auto index_probe = probe_index_files(rowset); + if (!index_probe.has_value()) { + return ResultError(index_probe.error()); + } + probe.index_files = std::move(index_probe).value(); + + for (int64_t segment_id = 0; segment_id < rowset->num_segments(); ++segment_id) { + auto segment_probe = probe_segment(rowset, segment_id); + if (!segment_probe.has_value()) { + return ResultError(segment_probe.error()); + } + probe.segments.push_back(std::move(segment_probe).value()); + } + return probe; +} + +void IndexStorageTestFixture::use_rowset_schema(const RowsetSharedPtr& rowset) { + CHECK(rowset != nullptr); + _tablet_schema = rowset->tablet_schema(); +} + +Result> IndexStorageTestFixture::rowsets_with_variant_extended_schema( + const std::vector& rowsets) { + if (rowsets.empty()) { + return ResultError(Status::InvalidArgument("rowsets are empty")); + } + if (rowsets.front() == nullptr || rowsets.front()->tablet_schema() == nullptr) { + return ResultError(Status::InvalidArgument("rowset schema is not initialized")); + } + + auto schema = variant_util::VariantCompactionUtil::calculate_variant_extended_schema( + rowsets, rowsets.front()->tablet_schema()); + if (schema == nullptr) { + return ResultError(Status::InternalError("failed to calculate variant extended schema")); + } + return rowsets_with_schema(rowsets, std::move(schema)); +} + +int32_t IndexStorageTestFixture::column_id_by_path(const std::string& path) const { + if (_tablet_schema == nullptr) { + return -1; + } + const int32_t column_id = _tablet_schema->field_index(PathInData(path)); + if (column_id >= 0) { + return column_id; + } + + const std::string relative_query_path = relative_variant_path(path); + for (int32_t i = 0; i < _tablet_schema->num_columns(); ++i) { + const auto& column = _tablet_schema->column(i); + if (!column.has_path_info()) { + continue; + } + auto relative_path = column.path_info_ptr()->copy_pop_front().get_path(); + if (relative_path == path || relative_path == relative_query_path) { + return i; + } + } + return -1; +} + +#undef RETURN_RESULT_IF_ERROR + +} // namespace doris::index_storage_test diff --git a/be/test/testutil/index_storage_test_util.h b/be/test/testutil/index_storage_test_util.h new file mode 100644 index 00000000000000..aa1a647ced00da --- /dev/null +++ b/be/test/testutil/index_storage_test_util.h @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "runtime/descriptors.h" +#include "storage/olap_common.h" +#include "storage/olap_define.h" +#include "storage/predicate/column_predicate.h" +#include "storage/rowset/rowset.h" +#include "storage/tablet/tablet.h" +#include "storage/tablet/tablet_schema.h" +#include "util/json/json_parser.h" + +namespace doris { + +class DataDir; +class StorageEngine; + +namespace segment_v2 { +class InvertedIndexQueryCache; +class InvertedIndexSearcherCache; +} // namespace segment_v2 + +namespace index_storage_test { + +struct TextColumnSpec { + int32_t unique_id = 2; + std::string name = "title"; + FieldType type = FieldType::OLAP_FIELD_TYPE_STRING; + bool nullable = false; +}; + +struct VariantPathSpec { + std::string path; + FieldType type = FieldType::OLAP_FIELD_TYPE_STRING; + bool nullable = true; + PatternTypePB pattern_type = PatternTypePB::MATCH_NAME; +}; + +struct VariantColumnSpec { + int32_t unique_id = 2; + std::string name = "v"; + bool nullable = false; + int32_t max_subcolumns_count = 8; + int32_t max_sparse_column_statistics_size = 10000; + int32_t sparse_hash_shard_count = 0; + bool enable_doc_mode = false; + int64_t doc_materialization_min_rows = 0; + int32_t doc_hash_shard_count = 0; + bool enable_nested_group = false; + std::vector predefined_paths; +}; + +struct IndexSpec { + int64_t index_id = 10001; + std::string name = "idx_v"; + int32_t column_uid = 2; + std::string suffix_path; + std::map properties; + + static IndexSpec column_index(int64_t index_id, std::string name, int32_t column_uid, + std::map properties = {}); + static IndexSpec field_pattern_index(int64_t index_id, std::string name, int32_t column_uid, + std::string field_pattern); +}; + +struct IndexTabletOptions { + int64_t tablet_id = 100000; + bool include_key_column = true; + bool external_segment_meta = true; + InvertedIndexStorageFormatPB index_storage_format = InvertedIndexStorageFormatPB::V2; + std::vector text_columns; + std::vector variant_columns; + std::vector inverted_indexes; +}; + +struct VariantJsonBatch { + std::vector keys; + std::vector> text_values_by_column; + std::vector> variant_jsons_by_column; + bool deprecated_enable_flatten_nested = false; + bool check_duplicate_json_path = false; + ParseConfig::ParseTo parse_to = ParseConfig::ParseTo::OnlySubcolumns; + + static VariantJsonBatch single_text(std::vector values, int32_t first_key = 0); + static VariantJsonBatch single_variant(std::vector jsons, int32_t first_key = 0); + size_t num_rows() const; +}; + +enum class IndexDataSourceKind { + INLINE_TEXT_ROWS, + INLINE_VARIANT_ROWS, + VARIANT_JSONL_FILE, +}; + +struct IndexDataSourceSpec { + IndexDataSourceKind kind = IndexDataSourceKind::INLINE_TEXT_ROWS; + std::vector rows; + std::string file_path; + int32_t first_key = 0; + size_t batch_size = 4096; + + static IndexDataSourceSpec inline_text(std::vector values, int32_t first_key = 0); + static IndexDataSourceSpec inline_variant(std::vector jsons, + int32_t first_key = 0); + static IndexDataSourceSpec variant_jsonl(std::string file_path, int32_t first_key = 0, + size_t batch_size = 4096); +}; + +struct IndexRowsetSpec { + int64_t version = 0; + int64_t max_rows_per_segment = 200; + DataWriteType write_type = DataWriteType::TYPE_DIRECT; + bool add_to_tablet = true; + std::vector data_sources; + std::vector batches; +}; + +struct IndexReadOptions { + ReaderType reader_type = ReaderType::READER_QUERY; + bool need_ordered_result = false; + std::string index_probe_label; + std::vector return_columns; + std::vector> predicates; + std::map target_cast_type_for_variants; + std::map all_access_paths; + std::map predicate_access_paths; + bool collect_string_values = false; + bool collect_variant_values = false; + bool enable_inverted_index_query = true; + bool enable_fallback_on_missing_inverted_index = true; + bool enable_no_need_read_data_opt = true; + bool enable_common_expr_pushdown = true; + bool enable_inverted_index_query_cache = false; + TPushAggOp::type push_down_agg_type_opt = TPushAggOp::NONE; +}; + +struct IndexProbeEventSnapshot { + std::string label; + int64_t rows_inverted_index_filtered = 0; + int64_t inverted_index_filter_timer = 0; + int64_t inverted_index_query_timer = 0; + int64_t inverted_index_query_null_bitmap_timer = 0; + int64_t inverted_index_query_bitmap_copy_timer = 0; + int64_t inverted_index_searcher_open_timer = 0; + int64_t inverted_index_searcher_search_timer = 0; + int64_t inverted_index_searcher_search_init_timer = 0; + int64_t inverted_index_searcher_search_exec_timer = 0; + int64_t inverted_index_downgrade_count = 0; + int64_t inverted_index_analyzer_timer = 0; + int64_t inverted_index_lookup_timer = 0; + + bool attempted_index() const { + return rows_inverted_index_filtered > 0 || inverted_index_query_timer > 0 || + inverted_index_filter_timer > 0 || inverted_index_query_null_bitmap_timer > 0 || + inverted_index_query_bitmap_copy_timer > 0 || + inverted_index_searcher_open_timer > 0 || inverted_index_searcher_search_timer > 0 || + inverted_index_searcher_search_init_timer > 0 || + inverted_index_searcher_search_exec_timer > 0 || inverted_index_analyzer_timer > 0 || + inverted_index_lookup_timer > 0; + } + + bool downgraded_fallback() const { return inverted_index_downgrade_count > 0; } + bool used_index() const { return attempted_index() && !downgraded_fallback(); } + bool effective_filter() const { + return rows_inverted_index_filtered > 0 && !downgraded_fallback(); + } +}; + +struct IndexReadResult { + int64_t rows_read = 0; + OlapReaderStatistics stats; + std::vector index_probe_events; + std::map>> string_values_by_uid; + std::map>> variant_values_by_uid; + + bool inverted_index_attempted() const; + bool inverted_index_downgraded() const; + bool inverted_index_used() const; + bool inverted_index_effective_filter() const; +}; + +struct IndexColumnLayout { + int32_t parent_unique_id = -1; + std::string full_path; + std::string relative_path; + int64_t none_null_size = 0; + bool is_sparse_column = false; + bool is_doc_value_column = false; + std::map sparse_non_null_size; +}; + +struct IndexSegmentLayout { + int64_t segment_id = -1; + int64_t num_rows = 0; + std::vector variant_columns; + + bool contains_relative_path(const std::string& path) const; +}; + +struct IndexFileProbe { + int64_t expected_files = 0; + int64_t existing_files = 0; + int64_t rowset_meta_entries = 0; + int64_t rowset_meta_index_size = 0; + std::vector missing_files; + + bool all_expected_files_exist() const { return expected_files == existing_files; } +}; + +struct IndexRowsetProbe { + int64_t num_rows = 0; + int64_t num_segments = 0; + std::vector segments; + IndexFileProbe index_files; + + bool contains_relative_path(const std::string& path) const; +}; + +enum class IndexCompactionKind { + CUMULATIVE, + FULL, +}; + +struct IndexSchemaPatch { + std::vector add_text_columns; + std::vector add_variant_columns; + std::set drop_column_uids; + std::map modify_variant_columns; + std::vector add_inverted_indexes; + std::set drop_index_ids; + std::set drop_index_names; +}; + +struct IndexStorageCase { + std::string name; + IndexTabletOptions tablet_options; + std::vector rowsets; +}; + +class IndexStorageCaseBuilder { +public: + explicit IndexStorageCaseBuilder(std::string name); + + IndexStorageCaseBuilder& tablet_id(int64_t tablet_id); + IndexStorageCaseBuilder& text_column(TextColumnSpec column); + IndexStorageCaseBuilder& variant_column(VariantColumnSpec column); + IndexStorageCaseBuilder& inverted_index(IndexSpec index); + IndexStorageCaseBuilder& rowset(int64_t version, IndexDataSourceSpec data_source, + int64_t max_rows_per_segment = 200); + IndexStorageCase build() const; + +private: + IndexStorageCase _case; +}; + +TabletSchemaPB build_tablet_schema_pb(const IndexTabletOptions& options); +TabletSchemaSPtr build_tablet_schema(const IndexTabletOptions& options); +TabletSchemaPB apply_schema_patch(const TabletSchema& base_schema, const IndexSchemaPatch& patch); +TabletSchemaSPtr build_patched_tablet_schema(const TabletSchema& base_schema, + const IndexSchemaPatch& patch); +TabletSchemaSPtr build_schema_with_variant_path_column(const TabletSchema& base_schema, + int32_t parent_unique_id, + std::string relative_path, FieldType type); + +void expect_index_filter_stats(const IndexReadResult& result, int64_t expected_filtered_rows); +void expect_inverted_index_used(const IndexReadResult& result); +void expect_inverted_index_fallback(const IndexReadResult& result); +void expect_inverted_index_not_attempted(const IndexReadResult& result); +void expect_index_files(const IndexRowsetProbe& probe, bool expected_present); + +class IndexStorageTestFixture : public testing::Test { +public: + ~IndexStorageTestFixture() override; + +protected: + void SetUp() override; + void TearDown() override; + + Status create_tablet(const IndexTabletOptions& options); + Result write_rowset(const IndexRowsetSpec& spec); + Result> write_rowsets(const std::vector& specs); + Result read_rowsets(const std::vector& rowsets, + IndexReadOptions options = {}); + Result compact_rowsets(IndexCompactionKind kind, + const std::vector& rowsets); + Result reload_rowset(const RowsetSharedPtr& rowset); + Result> reload_rowsets( + const std::vector& rowsets); + Result> rowsets_with_schema( + const std::vector& rowsets, TabletSchemaSPtr schema); + Result> build_inverted_indexes( + const std::vector& indexes); + Result> drop_inverted_indexes( + const std::vector& indexes); + Result probe_rowset(const RowsetSharedPtr& rowset); + void use_rowset_schema(const RowsetSharedPtr& rowset); + Result> rowsets_with_variant_extended_schema( + const std::vector& rowsets); + int32_t column_id_by_path(const std::string& path) const; + + const TabletSchemaSPtr& tablet_schema() const { return _tablet_schema; } + const TabletSharedPtr& tablet() const { return _tablet; } + StorageEngine* storage_engine() const { return _engine_ref; } + DataDir* data_dir() const { return _data_dir.get(); } + +private: + std::string _test_dir; + std::string _tmp_dir; + TabletSchemaSPtr _tablet_schema; + TabletSharedPtr _tablet; + StorageEngine* _engine_ref = nullptr; + std::unique_ptr _data_dir; + std::unique_ptr _inverted_index_searcher_cache; + std::unique_ptr _inverted_index_query_cache; +}; + +} // namespace index_storage_test +} // namespace doris