This is an automated email from the ASF dual-hosted git repository.
sdf-jkl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new aee9535fb7 Add benchmark for parquet variant get (#11107)
aee9535fb7 is described below
commit aee9535fb74e2991516716aa32e7475d1c1ffd0c
Author: Congxian Qiu <[email protected]>
AuthorDate: Mon Sep 21 23:40:44 2026 +0800
Add benchmark for parquet variant get (#11107)
# Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax.
-->
- Closes #10988 .
# Are these changes tested?
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
If this PR claims a performance improvement, please include evidence
such as benchmark results.
-->
Only benchmarks were added in this pr; no tests need to be added.
# Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
If there are any breaking changes to public APIs, please call them out.
-->
No
---
parquet-variant-compute/benches/variant_kernels.rs | 229 ++++++++++++++++++++-
1 file changed, 227 insertions(+), 2 deletions(-)
diff --git a/parquet-variant-compute/benches/variant_kernels.rs
b/parquet-variant-compute/benches/variant_kernels.rs
index e06c0e0b1e..5a8862f1a9 100644
--- a/parquet-variant-compute/benches/variant_kernels.rs
+++ b/parquet-variant-compute/benches/variant_kernels.rs
@@ -18,8 +18,12 @@
use arrow::array::{Array, ArrayRef, BinaryViewArray, BinaryViewBuilder,
StringArray, StructArray};
use arrow::buffer::Buffer;
use arrow_schema::{DataType, Field, FieldRef, Fields};
+use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime};
use criterion::{BatchSize, Criterion, criterion_group, criterion_main};
-use parquet_variant::{EMPTY_VARIANT_METADATA_BYTES, Variant, VariantBuilder,
VariantPath};
+use parquet_variant::{
+ EMPTY_VARIANT_METADATA_BYTES, Variant, VariantBuilder, VariantBuilderExt,
VariantDecimal8,
+ VariantPath,
+};
use parquet_variant_compute::{
GetOptions, VariantArray, VariantArrayBuilder, json_to_variant,
shred_variant, variant_get,
};
@@ -252,6 +256,214 @@ pub fn shred_variant_unmatched_object_bench(c: &mut
Criterion) {
});
}
+pub fn variant_get_utf8_from_int_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(c, "variant_get_utf8_from_int", |rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ vab.append_variant(Variant::Int64(rng.random()));
+ }
+ vab.build()
+ });
+}
+
+pub fn variant_get_utf8_from_decimal_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(c, "variant_get_utf8_from_decimal", |rng,
array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ vab.append_variant(Variant::Decimal8(
+ VariantDecimal8::try_new(rng.random_range(0..10000000000),
2).unwrap(),
+ ));
+ }
+ vab.build()
+ });
+}
+
+pub fn variant_get_utf8_from_float_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(c, "variant_get_utf8_from_float", |rng, array_size|
{
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ vab.append_variant(Variant::Double(rng.random()))
+ }
+ vab.build()
+ });
+}
+
+pub fn variant_get_utf8_from_timestamp_without_timezone_bench(c: &mut
Criterion) {
+ bench_variant_get_utf8(
+ c,
+ "variant_get_utf8_from_timestamp_without_timezone",
+ |rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ let timestamp = rng.random_range(
+ NaiveDateTime::MIN.and_utc().timestamp_micros()
+ ..=NaiveDateTime::MAX.and_utc().timestamp_micros(),
+ );
+ vab.append_variant(Variant::TimestampNtzMicros(
+ DateTime::from_timestamp_micros(timestamp)
+ .unwrap() // input always valid
+ .naive_utc(),
+ ));
+ }
+ vab.build()
+ },
+ );
+}
+
+pub fn variant_get_utf8_from_timestamp_with_timezone_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(
+ c,
+ "variant_get_utf8_from_timestamp_with_timezone",
+ |rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ let timestamp = rng.random_range(
+ NaiveDateTime::MIN.and_utc().timestamp_micros()
+ ..=NaiveDateTime::MAX.and_utc().timestamp_micros(),
+ );
+ vab.append_variant(Variant::TimestampMicros(
+ DateTime::from_timestamp_micros(timestamp).unwrap(), //
input always valid
+ ));
+ }
+ vab.build()
+ },
+ );
+}
+
+pub fn variant_get_utf8_from_date_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(c, "variant_get_utf8_from_date", |rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ let days_since_epoch =
+
rng.random_range(NaiveDate::MIN.to_epoch_days()..=NaiveDate::MAX.to_epoch_days());
+ vab.append_variant(Variant::Date(
+ NaiveDate::from_epoch_days(days_since_epoch).unwrap(),
+ ));
+ }
+ vab.build()
+ });
+}
+
+pub fn variant_get_utf8_from_time_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(c, "variant_get_utf8_from_time", |rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ let micros_since_midnight =
rng.random_range(0..=86_400_000_000i64);
+ let sec = (micros_since_midnight / 1_000_000) as u32;
+ let nano = ((micros_since_midnight % 1_000_000) * 1000) as u32;
+ vab.append_variant(Variant::Time(
+ NaiveTime::from_num_seconds_from_midnight_opt(sec,
nano).unwrap(),
+ ));
+ }
+ vab.build()
+ });
+}
+
+pub fn variant_get_utf8_from_list_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(c, "variant_get_utf8_from_list", |rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ let mut list_builder = vab.new_list();
+ list_builder.append_value(Variant::from(rng.random::<i64>()));
+ list_builder.append_value(Variant::from(rng.random::<i64>()));
+ list_builder.finish();
+ }
+ vab.build()
+ });
+}
+
+pub fn variant_get_utf8_from_map_in_list_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(c, "variant_get_utf8_from_map_in_list", |rng,
array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for _ in 0..array_size {
+ let mut list_builder = vab.new_list();
+ let mut inner_map_builder = list_builder.new_object();
+ inner_map_builder.insert("key1", rng.random::<i64>());
+ inner_map_builder.insert("key2", rng.random::<i64>());
+ inner_map_builder.insert("key3", rng.random::<i64>());
+ inner_map_builder.finish();
+ list_builder.finish();
+ }
+ vab.build()
+ })
+}
+
+pub fn variant_get_utf8_from_unshredded_string_bench(c: &mut Criterion) {
+ bench_variant_get_utf8(
+ c,
+ "variant_get_utf8_from_unshredded_string",
+ |_rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for i in 0..array_size {
+
vab.append_variant(Variant::String(format!("value_{i}").as_str()));
+ }
+ vab.build()
+ },
+ )
+}
+
+fn bench_variant_get_utf8(
+ c: &mut Criterion,
+ name: &str,
+ variant_gen_fun: impl Fn(&mut StdRng, usize) -> VariantArray,
+) {
+ let field: FieldRef = Arc::new(Field::new("typed_value", DataType::Utf8,
true));
+ let options = GetOptions::new().with_as_type(Some(field));
+
+ bench_variant_get(c, name, variant_gen_fun, options);
+}
+
+fn bench_variant_get(
+ c: &mut Criterion,
+ name: &str,
+ variant_gen_fun: impl Fn(&mut StdRng, usize) -> VariantArray,
+ options: GetOptions,
+) {
+ let array_size = 8192;
+ let mut rng = StdRng::seed_from_u64(42);
+
+ let array = variant_gen_fun(&mut rng, array_size);
+
+ let input = ArrayRef::from(array);
+ c.bench_function(name, |b| {
+ b.iter(|| variant_get(&input, options.clone()).unwrap())
+ });
+}
+
+pub fn variant_get_binary_from_unshredded_binary_bench(c: &mut Criterion) {
+ let field: FieldRef = Arc::new(Field::new("typed_value", DataType::Binary,
true));
+ let options = GetOptions::new().with_as_type(Some(field));
+ bench_variant_get(
+ c,
+ "variant_get_binary_from_unshredded_binary",
+ |_rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for i in 0..array_size {
+
vab.append_variant(Variant::Binary(format!("value_{i}").as_bytes()));
+ }
+ vab.build()
+ },
+ options,
+ )
+}
+
+pub fn variant_get_binary_from_string_bench(c: &mut Criterion) {
+ let field: FieldRef = Arc::new(Field::new("typed_value", DataType::Binary,
true));
+ let options = GetOptions::new().with_as_type(Some(field));
+ bench_variant_get(
+ c,
+ "variant_get_binary_from_string",
+ |_rng, array_size| {
+ let mut vab = VariantArrayBuilder::new(array_size);
+ for i in 0..array_size {
+
vab.append_variant(Variant::String(format!("value_{i}").as_str()));
+ }
+ vab.build()
+ },
+ options,
+ );
+}
+
criterion_group!(
benches,
variant_get_bench,
@@ -260,8 +472,21 @@ criterion_group!(
shred_variant_partial_object_bench,
shred_variant_unmatched_object_bench,
variant_array_builder_build_bench,
- benchmark_batch_json_string_to_variant
+ benchmark_batch_json_string_to_variant,
+ variant_get_utf8_from_unshredded_string_bench,
+ variant_get_binary_from_unshredded_binary_bench,
+ variant_get_utf8_from_int_bench,
+ variant_get_utf8_from_decimal_bench,
+ variant_get_utf8_from_float_bench,
+ variant_get_utf8_from_timestamp_without_timezone_bench,
+ variant_get_utf8_from_timestamp_with_timezone_bench,
+ variant_get_utf8_from_date_bench,
+ variant_get_utf8_from_time_bench,
+ variant_get_utf8_from_list_bench,
+ variant_get_utf8_from_map_in_list_bench,
+ variant_get_binary_from_string_bench,
);
+
criterion_main!(benches);
/// Creates a `VariantArray` with a specified number of Variant::Int64 values
each with random value.
