Skip to content

Add Parquet reader edge case coverage #23909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vyasr wants to merge 1 commit into
base: main
Choose a base branch
from
+395 −12
Open

Add Parquet reader edge case coverage #23909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 53 additions & 11 deletions cpp/benchmarks/io/parquet/parquet_reader_encoding.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,6 @@ void set_encoding_recursive(cudf::io::column_in_metadata& col_meta,
}
}

data_profile make_profile(cudf::size_type cardinality, cudf::size_type run_length)
{
return data_profile_builder().cardinality(cardinality).avg_run_length(run_length);
}

data_profile make_list_profile(cudf::size_type cardinality,
cudf::size_type run_length,
cudf::size_type nesting,
Expand Down Expand Up @@ -108,23 +103,37 @@ std::unique_ptr<cudf::table> create_nested_table(std::vector<cudf::type_id> cons
return std::make_unique<cudf::table>(std::move(columns));
}

void bench_read_encoding(nvbench::state& state, std::vector<cudf::type_id> const& d_types)
void bench_read_encoding(nvbench::state& state,
std::vector<cudf::type_id> const& d_types,
bool const use_nullable_page_size_matrix = false)
{
auto const encoding = retrieve_column_encoding_enum(state.get_string("encoding"));
auto const source_type = retrieve_io_type_enum(state.get_string("io_type"));
auto const data_size = static_cast<size_t>(state.get_int64("data_size"));
auto const cardinality = static_cast<cudf::size_type>(state.get_int64("cardinality"));
auto const run_length = static_cast<cudf::size_type>(state.get_int64("run_length"));
auto const nesting = static_cast<cudf::size_type>(state.get_int64("nesting"));
auto const nesting = use_nullable_page_size_matrix
? cudf::size_type{0}
: static_cast<cudf::size_type>(state.get_int64("nesting"));
auto const validity =
use_nullable_page_size_matrix ? state.get_string("validity") : "no_validity";
auto const page_rows = use_nullable_page_size_matrix
? static_cast<cudf::size_type>(state.get_int64("page_rows"))
: cudf::size_type{0};
cuio_source_sink_pair source_sink(source_type);

auto const num_rows_written = [&]() {
auto const leaf_types = cycle_dtypes(d_types, num_cols);
auto profile_builder =
data_profile_builder().cardinality(cardinality).avg_run_length(run_length);
if (validity == "no_validity") {
profile_builder.no_validity();
} else {
profile_builder.null_probability(validity == "nullable_1" ? 0.01 : 0.50);
}
auto const tbl =
nesting > 0
? create_nested_table(leaf_types, data_size, cardinality, run_length, nesting)
: create_random_table(
leaf_types, table_size_bytes{data_size}, make_profile(cardinality, run_length));
nesting > 0 ? create_nested_table(leaf_types, data_size, cardinality, run_length, nesting)
: create_random_table(leaf_types, table_size_bytes{data_size}, profile_builder);
auto const view = tbl->view();

cudf::io::table_input_metadata metadata(view);
Expand All @@ -138,6 +147,7 @@ void bench_read_encoding(nvbench::state& state, std::vector<cudf::type_id> const
.compression(cudf::io::compression_type::NONE)
.dictionary_policy(cudf::io::dictionary_policy::NEVER)
.write_v2_headers(true);
if (use_nullable_page_size_matrix) { write_opts.set_max_page_size_rows(page_rows); }
cudf::io::write_parquet(write_opts);
return view.num_rows();
}();
Expand All @@ -157,6 +167,16 @@ void BM_parquet_read_delta_string(nvbench::state& state)
bench_read_encoding(state, {cudf::type_id::STRING});
}

void BM_parquet_read_delta_binary_nullable_page_sizes(nvbench::state& state)
{
bench_read_encoding(state, {cudf::type_id::INT32, cudf::type_id::INT64}, true);
}

void BM_parquet_read_delta_string_nullable_page_sizes(nvbench::state& state)
{
bench_read_encoding(state, {cudf::type_id::STRING}, true);
}

NVBENCH_BENCH(BM_parquet_read_delta_binary)
.set_name("parquet_read_delta_binary")
.add_string_axis("encoding", {"PLAIN", "DELTA_BINARY_PACKED"})
Expand All @@ -176,3 +196,25 @@ NVBENCH_BENCH(BM_parquet_read_delta_string)
.add_int64_axis("run_length", {1, 32})
.add_int64_axis("nesting", {0, 1})
.add_int64_axis("data_size", {512 << 20});

NVBENCH_BENCH(BM_parquet_read_delta_binary_nullable_page_sizes)
.set_name("parquet_read_delta_binary_nullable_page_sizes")
.add_string_axis("encoding", {"PLAIN", "DELTA_BINARY_PACKED"})
.add_string_axis("io_type", {"DEVICE_BUFFER"})
.add_string_axis("validity", {"nullable_1", "nullable_50"})
.set_min_samples(4)
.add_int64_axis("cardinality", {0})
.add_int64_axis("run_length", {1})
.add_int64_axis("page_rows", {31, 32, 33, 255, 256, 257})
.add_int64_axis("data_size", {8 << 20});

NVBENCH_BENCH(BM_parquet_read_delta_string_nullable_page_sizes)
.set_name("parquet_read_delta_string_nullable_page_sizes")
.add_string_axis("encoding", {"PLAIN", "DELTA_LENGTH_BYTE_ARRAY", "DELTA_BYTE_ARRAY"})
.add_string_axis("io_type", {"DEVICE_BUFFER"})
.add_string_axis("validity", {"nullable_1", "nullable_50"})
.set_min_samples(4)
.add_int64_axis("cardinality", {0})
.add_int64_axis("run_length", {1})
.add_int64_axis("page_rows", {31, 32, 33, 255, 256, 257})
.add_int64_axis("data_size", {8 << 20});
40 changes: 39 additions & 1 deletion cpp/benchmarks/io/parquet/parquet_reader_input.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand Down Expand Up @@ -72,6 +72,37 @@ void BM_parquet_read_fixed_width_struct(nvbench::state& state,
type_list);
}

// A single PLAIN INT32 column isolates nullable flat-page decode with bounded tiny-page coverage.
void BM_parquet_read_flat_nullable_pages(nvbench::state& state)
{
cudf::size_type constexpr num_benchmark_cols = 1;
auto const data_size = static_cast<size_t>(state.get_int64("data_size"));
auto const validity = state.get_string("validity");
auto const page_rows = static_cast<cudf::size_type>(state.get_int64("page_rows"));
cuio_source_sink_pair source_sink(io_type::DEVICE_BUFFER);

auto const num_rows_written = [&]() {
auto profile = data_profile_builder();
if (validity == "no_validity") {
profile.no_validity();
} else {
profile.null_probability(validity == "nullable_1" ? 0.01 : 0.50);
}
auto const tbl =
create_random_table({cudf::type_id::INT32}, table_size_bytes{data_size}, profile);
auto const view = tbl->view();

cudf::io::write_parquet(
cudf::io::parquet_writer_options::builder(source_sink.make_sink_info(), view)
.compression(cudf::io::compression_type::NONE)
.dictionary_policy(cudf::io::dictionary_policy::NEVER)
.max_page_size_rows(page_rows));
return view.num_rows();
}();

parquet_read_common(num_rows_written, num_benchmark_cols, source_sink, state);
}

void BM_parquet_read_io_small_mixed(nvbench::state& state)
{
auto const d_type =
Expand Down Expand Up @@ -131,6 +162,13 @@ NVBENCH_BENCH_TYPES(BM_parquet_read_data, NVBENCH_TYPE_AXES(d_type_list))
.add_int64_axis("row_group_size_bytes", {0})
.add_int64_axis("row_group_size_rows", {0});

NVBENCH_BENCH(BM_parquet_read_flat_nullable_pages)
.set_name("parquet_read_flat_nullable_pages")
.set_min_samples(4)
.add_string_axis("validity", {"no_validity", "nullable_1", "nullable_50"})
.add_int64_axis("page_rows", {31, 32, 33, 255, 256, 257})
.add_int64_axis("data_size", {1 << 20});

NVBENCH_BENCH(BM_parquet_read_io_small_mixed)
.set_name("parquet_read_io_small_mixed")
.add_string_axis("io_type", {"FILEPATH"})
Expand Down
Loading
Loading