diff --git a/cpp/benchmarks/io/parquet/parquet_reader_encoding.cpp b/cpp/benchmarks/io/parquet/parquet_reader_encoding.cpp index dcf8d9a0e13..58b643a7149 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_encoding.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_encoding.cpp @@ -56,11 +56,6 @@ void set_encoding_recursive(cudf::io::column_in_metadata& col_meta, } } -data_profile make_profile(cudf::size_type cardinality, cudf::size_type run_length) -{ - return data_profile_builder().cardinality(cardinality).avg_run_length(run_length); -} - data_profile make_list_profile(cudf::size_type cardinality, cudf::size_type run_length, cudf::size_type nesting, @@ -108,23 +103,37 @@ std::unique_ptr create_nested_table(std::vector cons return std::make_unique(std::move(columns)); } -void bench_read_encoding(nvbench::state& state, std::vector const& d_types) +void bench_read_encoding(nvbench::state& state, + std::vector const& d_types, + bool const use_nullable_page_size_matrix = false) { auto const encoding = retrieve_column_encoding_enum(state.get_string("encoding")); auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); auto const data_size = static_cast(state.get_int64("data_size")); auto const cardinality = static_cast(state.get_int64("cardinality")); auto const run_length = static_cast(state.get_int64("run_length")); - auto const nesting = static_cast(state.get_int64("nesting")); + auto const nesting = use_nullable_page_size_matrix + ? cudf::size_type{0} + : static_cast(state.get_int64("nesting")); + auto const validity = + use_nullable_page_size_matrix ? state.get_string("validity") : "no_validity"; + auto const page_rows = use_nullable_page_size_matrix + ? static_cast(state.get_int64("page_rows")) + : cudf::size_type{0}; cuio_source_sink_pair source_sink(source_type); auto const num_rows_written = [&]() { auto const leaf_types = cycle_dtypes(d_types, num_cols); + auto profile_builder = + data_profile_builder().cardinality(cardinality).avg_run_length(run_length); + if (validity == "no_validity") { + profile_builder.no_validity(); + } else { + profile_builder.null_probability(validity == "nullable_1" ? 0.01 : 0.50); + } auto const tbl = - nesting > 0 - ? create_nested_table(leaf_types, data_size, cardinality, run_length, nesting) - : create_random_table( - leaf_types, table_size_bytes{data_size}, make_profile(cardinality, run_length)); + nesting > 0 ? create_nested_table(leaf_types, data_size, cardinality, run_length, nesting) + : create_random_table(leaf_types, table_size_bytes{data_size}, profile_builder); auto const view = tbl->view(); cudf::io::table_input_metadata metadata(view); @@ -138,6 +147,7 @@ void bench_read_encoding(nvbench::state& state, std::vector const .compression(cudf::io::compression_type::NONE) .dictionary_policy(cudf::io::dictionary_policy::NEVER) .write_v2_headers(true); + if (use_nullable_page_size_matrix) { write_opts.set_max_page_size_rows(page_rows); } cudf::io::write_parquet(write_opts); return view.num_rows(); }(); @@ -157,6 +167,16 @@ void BM_parquet_read_delta_string(nvbench::state& state) bench_read_encoding(state, {cudf::type_id::STRING}); } +void BM_parquet_read_delta_binary_nullable_page_sizes(nvbench::state& state) +{ + bench_read_encoding(state, {cudf::type_id::INT32, cudf::type_id::INT64}, true); +} + +void BM_parquet_read_delta_string_nullable_page_sizes(nvbench::state& state) +{ + bench_read_encoding(state, {cudf::type_id::STRING}, true); +} + NVBENCH_BENCH(BM_parquet_read_delta_binary) .set_name("parquet_read_delta_binary") .add_string_axis("encoding", {"PLAIN", "DELTA_BINARY_PACKED"}) @@ -176,3 +196,25 @@ NVBENCH_BENCH(BM_parquet_read_delta_string) .add_int64_axis("run_length", {1, 32}) .add_int64_axis("nesting", {0, 1}) .add_int64_axis("data_size", {512 << 20}); + +NVBENCH_BENCH(BM_parquet_read_delta_binary_nullable_page_sizes) + .set_name("parquet_read_delta_binary_nullable_page_sizes") + .add_string_axis("encoding", {"PLAIN", "DELTA_BINARY_PACKED"}) + .add_string_axis("io_type", {"DEVICE_BUFFER"}) + .add_string_axis("validity", {"nullable_1", "nullable_50"}) + .set_min_samples(4) + .add_int64_axis("cardinality", {0}) + .add_int64_axis("run_length", {1}) + .add_int64_axis("page_rows", {31, 32, 33, 255, 256, 257}) + .add_int64_axis("data_size", {8 << 20}); + +NVBENCH_BENCH(BM_parquet_read_delta_string_nullable_page_sizes) + .set_name("parquet_read_delta_string_nullable_page_sizes") + .add_string_axis("encoding", {"PLAIN", "DELTA_LENGTH_BYTE_ARRAY", "DELTA_BYTE_ARRAY"}) + .add_string_axis("io_type", {"DEVICE_BUFFER"}) + .add_string_axis("validity", {"nullable_1", "nullable_50"}) + .set_min_samples(4) + .add_int64_axis("cardinality", {0}) + .add_int64_axis("run_length", {1}) + .add_int64_axis("page_rows", {31, 32, 33, 255, 256, 257}) + .add_int64_axis("data_size", {8 << 20}); diff --git a/cpp/benchmarks/io/parquet/parquet_reader_input.cpp b/cpp/benchmarks/io/parquet/parquet_reader_input.cpp index 748ce682c77..3afc9e14dd4 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_input.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_input.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -72,6 +72,37 @@ void BM_parquet_read_fixed_width_struct(nvbench::state& state, type_list); } +// A single PLAIN INT32 column isolates nullable flat-page decode with bounded tiny-page coverage. +void BM_parquet_read_flat_nullable_pages(nvbench::state& state) +{ + cudf::size_type constexpr num_benchmark_cols = 1; + auto const data_size = static_cast(state.get_int64("data_size")); + auto const validity = state.get_string("validity"); + auto const page_rows = static_cast(state.get_int64("page_rows")); + cuio_source_sink_pair source_sink(io_type::DEVICE_BUFFER); + + auto const num_rows_written = [&]() { + auto profile = data_profile_builder(); + if (validity == "no_validity") { + profile.no_validity(); + } else { + profile.null_probability(validity == "nullable_1" ? 0.01 : 0.50); + } + auto const tbl = + create_random_table({cudf::type_id::INT32}, table_size_bytes{data_size}, profile); + auto const view = tbl->view(); + + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(source_sink.make_sink_info(), view) + .compression(cudf::io::compression_type::NONE) + .dictionary_policy(cudf::io::dictionary_policy::NEVER) + .max_page_size_rows(page_rows)); + return view.num_rows(); + }(); + + parquet_read_common(num_rows_written, num_benchmark_cols, source_sink, state); +} + void BM_parquet_read_io_small_mixed(nvbench::state& state) { auto const d_type = @@ -131,6 +162,13 @@ NVBENCH_BENCH_TYPES(BM_parquet_read_data, NVBENCH_TYPE_AXES(d_type_list)) .add_int64_axis("row_group_size_bytes", {0}) .add_int64_axis("row_group_size_rows", {0}); +NVBENCH_BENCH(BM_parquet_read_flat_nullable_pages) + .set_name("parquet_read_flat_nullable_pages") + .set_min_samples(4) + .add_string_axis("validity", {"no_validity", "nullable_1", "nullable_50"}) + .add_int64_axis("page_rows", {31, 32, 33, 255, 256, 257}) + .add_int64_axis("data_size", {1 << 20}); + NVBENCH_BENCH(BM_parquet_read_io_small_mixed) .set_name("parquet_read_io_small_mixed") .add_string_axis("io_type", {"FILEPATH"}) diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index e1d47c1fe91..0e21968cad5 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -190,6 +191,275 @@ TEST_F(ParquetReaderTest, UserBoundsWithNulls) } } +// Required struct leaves have no definition stream. Exercise the reader's +// nested state across page and read boundaries without relying on the +// optional-level path used by the neighboring user-bounds tests. +TEST_F(ParquetReaderTest, RequiredStructUserBoundsAcrossPages) +{ + constexpr int num_rows = 257; + auto values = cuda::counting_iterator{0}; + + cudf::test::fixed_width_column_wrapper child0(values, values + num_rows); + cudf::test::fixed_width_column_wrapper child1(values, values + num_rows); + auto struct_col = cudf::test::structs_column_wrapper{{child0, child1}}; + cudf::table_view const expected{{struct_col}}; + + auto const filepath = temp_env->get_temp_filepath("RequiredStructUserBoundsAcrossPages.parquet"); + auto out_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .max_page_size_rows(32); + cudf::io::write_parquet(out_opts); + + for (auto const [skip_rows, num_rows_to_read] : + std::vector>{{0, num_rows}, {31, 3}, {32, 3}, {33, 3}, {255, 2}}) { + auto read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .skip_rows(skip_rows) + .num_rows(num_rows_to_read); + auto const result = cudf::io::read_parquet(read_opts); + auto const sliced = cudf::slice(expected, {skip_rows, skip_rows + num_rows_to_read}); + CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, sliced.front()); + } +} + +// Exercise BYTE_STREAM_SPLIT for required flat fixed-width pages, including a read that crosses +// page boundaries and confirmation that the file uses the requested encoding. +TEST_F(ParquetReaderTest, RequiredFlatByteStreamSplit) +{ + constexpr int num_rows = 257; + auto values = cudf::detail::make_counting_transform_iterator( + 0, [](int const i) { return static_cast(i) * 1.25f; }); + cudf::test::fixed_width_column_wrapper col(values, values + num_rows); + cudf::table_view const expected{{col}}; + auto const filepath = temp_env->get_temp_filepath("RequiredFlatByteStreamSplit.parquet"); + cudf::io::table_input_metadata metadata(expected); + metadata.column_metadata[0].set_encoding(cudf::io::column_encoding::BYTE_STREAM_SPLIT); + auto out_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .metadata(metadata) + .max_page_size_rows(32); + cudf::io::write_parquet(out_opts); + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::FileMetaData fmd; + read_footer(source, &fmd); + auto const& encodings = fmd.row_groups.front().columns.front().meta_data.encodings; + EXPECT_NE( + std::find(encodings.begin(), encodings.end(), cudf::io::parquet::Encoding::BYTE_STREAM_SPLIT), + encodings.end()); + + auto const result = cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .skip_rows(31) + .num_rows(33)); + auto const sliced = cudf::slice(expected, {31, 64}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(0), sliced.front().column(0)); +} + +// Exercise nullable nested state for mixed BOOL8 and STRING leaves while slicing across Parquet +// page boundaries. +TEST_F(ParquetReaderTest, NullableStructBoolAndStringUserBounds) +{ + constexpr int row_count = 257; + std::vector strings; + strings.reserve(row_count); + std::vector parent_validity(row_count); + std::vector bool_validity(row_count); + std::vector string_validity(row_count); + for (int row = 0; row < row_count; ++row) { + strings.push_back("nested-" + std::to_string(row % 17)); + parent_validity[row] = row % 11 != 0; + bool_validity[row] = row % 7 != 0; + string_validity[row] = row % 5 != 0; + } + auto bool_values = + cudf::detail::make_counting_transform_iterator(0, [](int const row) { return row % 2 == 0; }); + cudf::test::fixed_width_column_wrapper bool_child( + bool_values, bool_values + row_count, bool_validity.begin()); + cudf::test::strings_column_wrapper string_child( + strings.begin(), strings.end(), string_validity.begin()); + auto struct_col = + cudf::test::structs_column_wrapper{{bool_child, string_child}, parent_validity.begin()}; + cudf::table_view const expected{{struct_col}}; + + auto const filepath = + temp_env->get_temp_filepath("NullableStructBoolAndStringUserBounds.parquet"); + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .dictionary_policy(cudf::io::dictionary_policy::NEVER) + .max_page_size_rows(32) + .write_v2_headers(true)); + + for (auto const [skip_rows, num_rows_to_read] : + std::vector>{{0, row_count}, {31, 3}, {32, 3}, {33, 3}, {255, 2}}) { + auto const result = cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .skip_rows(skip_rows) + .num_rows(num_rows_to_read)); + auto const sliced = cudf::slice(expected, {skip_rows, skip_rows + num_rows_to_read}); + CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, sliced.front()); + } +} + +// Exercise nullable STRUCT reads for dictionary and BYTE_STREAM_SPLIT encoded leaves, including +// encoding verification and page-boundary user bounds. +TEST_F(ParquetReaderTest, NullableStructDictionaryAndByteStreamSplit) +{ + constexpr int row_count = 257; + std::vector parent_validity(row_count); + std::vector int_validity(row_count); + std::vector float_validity(row_count); + std::vector string_validity(row_count); + std::vector strings; + strings.reserve(row_count); + for (int row = 0; row < row_count; ++row) { + parent_validity[row] = row % 11 != 0; + int_validity[row] = row % 7 != 0; + float_validity[row] = row % 5 != 0; + string_validity[row] = row % 3 != 0; + strings.push_back("dictionary-" + std::to_string(row % 17)); + } + auto int_values = + cudf::detail::make_counting_transform_iterator(0, [](int const row) { return row % 13; }); + auto float_values = cudf::detail::make_counting_transform_iterator( + 0, [](int const row) { return static_cast(row % 19) * 1.25f; }); + cudf::test::fixed_width_column_wrapper int_child( + int_values, int_values + row_count, int_validity.begin()); + cudf::test::fixed_width_column_wrapper float_child( + float_values, float_values + row_count, float_validity.begin()); + cudf::test::strings_column_wrapper string_child( + strings.begin(), strings.end(), string_validity.begin()); + auto struct_col = cudf::test::structs_column_wrapper{{int_child, float_child, string_child}, + parent_validity.begin()}; + cudf::table_view const expected{{struct_col}}; + + auto const dictionary_filepath = + temp_env->get_temp_filepath("NullableStructDictionaryUserBounds.parquet"); + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{dictionary_filepath}, expected) + .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) + .max_page_size_rows(32) + .write_v2_headers(true)); + auto const dictionary_source = cudf::io::datasource::create(dictionary_filepath); + cudf::io::parquet::FileMetaData dictionary_fmd; + read_footer(dictionary_source, &dictionary_fmd); + for (auto const& column : dictionary_fmd.row_groups.front().columns) { + auto const& encodings = column.meta_data.encodings; + EXPECT_NE( + std::find(encodings.begin(), encodings.end(), cudf::io::parquet::Encoding::RLE_DICTIONARY), + encodings.end()); + } + + auto const bss_filepath = temp_env->get_temp_filepath("NullableStructBssUserBounds.parquet"); + cudf::io::table_input_metadata metadata(expected); + metadata.column_metadata[0].set_name("s"); + metadata.column_metadata[0].child(0).set_name("i"); + metadata.column_metadata[0].child(1).set_name("f"); + metadata.column_metadata[0].child(2).set_name("str"); + metadata.column_metadata[0].child(1).set_encoding(cudf::io::column_encoding::BYTE_STREAM_SPLIT); + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{bss_filepath}, expected) + .metadata(metadata) + .dictionary_policy(cudf::io::dictionary_policy::NEVER) + .max_page_size_rows(32) + .write_v2_headers(true)); + auto const bss_source = cudf::io::datasource::create(bss_filepath); + cudf::io::parquet::FileMetaData bss_fmd; + read_footer(bss_source, &bss_fmd); + auto const& bss_encodings = bss_fmd.row_groups.front().columns[1].meta_data.encodings; + EXPECT_NE( + std::find( + bss_encodings.begin(), bss_encodings.end(), cudf::io::parquet::Encoding::BYTE_STREAM_SPLIT), + bss_encodings.end()); + + for (auto const& filepath : {dictionary_filepath, bss_filepath}) { + for (auto const [skip_rows, num_rows_to_read] : + std::vector>{{0, row_count}, {31, 3}, {32, 3}, {33, 3}, {255, 2}}) { + auto const result = cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .skip_rows(skip_rows) + .num_rows(num_rows_to_read)); + auto const sliced = cudf::slice(expected, {skip_rows, skip_rows + num_rows_to_read}); + CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, sliced.front()); + } + } +} + +// Exercise BYTE_STREAM_SPLIT under both nesting and repetition. The nullable STRUCT and LIST levels +// add definition and repetition streams; 32-row pages and the selected ranges cover both page +// boundaries and the 32-value decode boundary. +TEST_F(ParquetReaderTest, ByteStreamSplitNestedListFloat) +{ + constexpr cudf::size_type num_rows = 257; + std::vector offsets{0}; + std::vector values; + std::vector list_validity; + std::vector value_validity; + std::vector struct_validity; + for (cudf::size_type row = 0; row < num_rows; ++row) { + auto const struct_is_valid = row % 11 != 0; + auto const list_is_valid = struct_is_valid && row % 7 != 0; + auto const list_size = list_is_valid ? row % 4 : 0; + list_validity.push_back(list_is_valid); + struct_validity.push_back(struct_is_valid); + for (cudf::size_type element = 0; element < list_size; ++element) { + values.push_back(static_cast(row * 4 + element) * 1.25f); + value_validity.push_back((row + element) % 5 != 0); + } + offsets.push_back(offsets.back() + list_size); + } + + auto [list_mask, list_null_count] = + cudf::test::detail::make_null_mask(list_validity.begin(), list_validity.end()); + auto list_col = cudf::make_lists_column( + num_rows, + cudf::test::fixed_width_column_wrapper(offsets.begin(), offsets.end()) + .release(), + cudf::test::fixed_width_column_wrapper( + values.begin(), values.end(), value_validity.begin()) + .release(), + list_null_count, + std::move(list_mask)); + + auto [struct_mask, struct_null_count] = + cudf::test::detail::make_null_mask(struct_validity.begin(), struct_validity.end()); + std::vector> children; + children.push_back(std::move(list_col)); + auto struct_col = cudf::make_structs_column( + num_rows, std::move(children), struct_null_count, std::move(struct_mask)); + cudf::table_view const expected{{*struct_col}}; + + auto const filepath = temp_env->get_temp_filepath("ByteStreamSplitNestedListFloat.parquet"); + cudf::io::table_input_metadata metadata(expected); + metadata.column_metadata[0].set_name("s"); + metadata.column_metadata[0].child(0).set_name("floats"); + metadata.column_metadata[0].child(0).child(1).set_encoding( + cudf::io::column_encoding::BYTE_STREAM_SPLIT); + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .metadata(std::move(metadata)) + .dictionary_policy(cudf::io::dictionary_policy::NEVER) + .max_page_size_rows(32) + .write_v2_headers(true)); + + // Prove that the requested encoding made it into the file, rather than only testing a writer + // fallback. A nullable list leaf has RLE level encodings in addition to the data encoding. + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::FileMetaData fmd; + read_footer(source, &fmd); + auto const& encodings = fmd.row_groups.front().columns.front().meta_data.encodings; + EXPECT_NE( + std::find(encodings.begin(), encodings.end(), cudf::io::parquet::Encoding::BYTE_STREAM_SPLIT), + encodings.end()); + + for (auto const [skip_rows, num_rows_to_read] : + std::vector>{ + {0, num_rows}, {31, 3}, {32, 3}, {33, 3}, {255, 2}}) { + auto const result = cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .skip_rows(skip_rows) + .num_rows(num_rows_to_read)); + auto const sliced = cudf::slice(expected, {skip_rows, skip_rows + num_rows_to_read}); + CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, sliced.front()); + } +} + TEST_F(ParquetReaderTest, UserBoundsWithNullsMixedTypes) { constexpr int num_rows = 32 * 1024; @@ -363,6 +633,39 @@ TEST_F(ParquetReaderTest, ListUserBoundsWithNullsLarge) } } +// Exercise repeated-level handling for nullable LIST pages that contain no leaf values because +// every list is null. +TEST_F(ParquetReaderTest, NullableListAllNullPages) +{ + constexpr int num_rows = 257; + std::vector offsets(num_rows + 1, 0); + auto validity = cuda::make_constant_iterator(false); + auto [null_mask, null_count] = cudf::test::detail::make_null_mask(validity, validity + num_rows); + + auto input = cudf::make_lists_column( + num_rows, + cudf::test::fixed_width_column_wrapper(offsets.begin(), offsets.end()) + .release(), + cudf::test::strings_column_wrapper{}.release(), + null_count, + std::move(null_mask)); + cudf::table_view const expected_table{{*input}}; + auto const filepath = temp_env->get_temp_filepath("NullableListAllNullPages.parquet"); + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected_table) + .max_page_size_rows(32)); + + for (auto const [skip_rows, num_rows_to_read] : + std::vector>{{0, num_rows}, {31, 3}, {32, 3}, {33, 3}, {255, 2}}) { + auto const result = cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .skip_rows(skip_rows) + .num_rows(num_rows_to_read)); + auto const sliced = cudf::slice(expected_table, {skip_rows, skip_rows + num_rows_to_read}); + CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, sliced.front()); + } +} + TEST_F(ParquetReaderTest, ReorderedColumns) { {