This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 46da3e3bcc Upgrade `rand` to `0.10` (#10566)
46da3e3bcc is described below
commit 46da3e3bcccbb4267a56e907b0bee28e8691096f
Author: Jeffrey Vo <[email protected]>
AuthorDate: Fri Aug 7 21:03:51 2026 +0900
Upgrade `rand` to `0.10` (#10566)
# Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax.
-->
- Closes https://github.com/apache/arrow-rs/pull/10562
# Rationale for this change
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
Keeping dependencies up to date.
# What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
Bump `rand` to `0.10` in the arrow crate. Since we expose functions that
rely on the rand API, this is a breaking change. Also because `half`
doesn't support `rand` `0.10` (see
https://github.com/VoidStarKat/half-rs/issues/144) we have to remove
dependency on its `rand_distr` feature and handroll it ourselves.
# Are these changes tested?
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
If this PR claims a performance improvement, please include evidence
such as benchmark results.
-->
Existing tests
# Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
If there are any breaking changes to public APIs, please call them out.
-->
Yes, API change. `create_primitive_array()` from `bench_util` no longer
supports `Float16Array`s; instead you must use the new
`create_nullable_f16_array()` function.
---
Cargo.lock | 14 +-------------
arrow/Cargo.toml | 8 ++++----
arrow/benches/array_from.rs | 2 +-
arrow/benches/bitwise_kernel.rs | 2 +-
arrow/benches/boolean_append_packed.rs | 2 +-
arrow/benches/buffer_create.rs | 2 +-
arrow/benches/builder.rs | 2 +-
arrow/benches/cast_kernels.rs | 2 +-
arrow/benches/coalesce_kernels.rs | 2 +-
arrow/benches/comparison_kernels.rs | 2 +-
arrow/benches/csv_reader.rs | 2 +-
arrow/benches/decimal_validate.rs | 2 +-
arrow/benches/interleave_kernels.rs | 2 +-
arrow/benches/json_writer.rs | 2 +-
arrow/benches/merge_kernels.rs | 2 +-
arrow/benches/mutable_array.rs | 2 +-
arrow/benches/primitive_run_take.rs | 2 +-
arrow/benches/string_dictionary_builder.rs | 2 +-
arrow/benches/string_run_iterator.rs | 2 +-
arrow/benches/take_kernels.rs | 2 +-
arrow/benches/zip_kernels.rs | 2 +-
arrow/src/util/bench_util.rs | 30 ++++++++++++++++++++++++++++--
arrow/src/util/data_gen.rs | 4 ++--
arrow/src/util/test_util.rs | 2 +-
24 files changed, 55 insertions(+), 41 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 40077980e6..483f94a627 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -191,7 +191,7 @@ dependencies = [
"criterion",
"half",
"memmap2",
- "rand 0.9.5",
+ "rand 0.10.2",
"serde",
]
@@ -1545,8 +1545,6 @@ dependencies = [
"cfg-if",
"crunchy",
"num-traits",
- "rand 0.9.5",
- "rand_distr",
"zerocopy",
]
@@ -2942,16 +2940,6 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
-[[package]]
-name = "rand_distr"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
-dependencies = [
- "num-traits",
- "rand 0.9.5",
-]
-
[[package]]
name = "rand_pcg"
version = "0.10.2"
diff --git a/arrow/Cargo.toml b/arrow/Cargo.toml
index c4bab6fc3f..ba81249a63 100644
--- a/arrow/Cargo.toml
+++ b/arrow/Cargo.toml
@@ -54,8 +54,8 @@ arrow-schema = { workspace = true }
arrow-select = { workspace = true }
arrow-string = { workspace = true }
-rand = { version = "0.9", default-features = false, features = ["std",
"std_rng", "thread_rng"], optional = true }
-half = { version = "2.1", default-features = false, features = ["rand_distr"],
optional = true }
+rand = { version = "0.10", default-features = false, features = ["std",
"std_rng", "thread_rng"], optional = true }
+half = { version = "2.1", default-features = false, optional = true }
[package.metadata.docs.rs]
all-features = true
@@ -91,8 +91,8 @@ pool = ["arrow-array/pool"]
[dev-dependencies]
chrono = { workspace = true }
criterion = { workspace = true, default-features = false }
-half = { version = "2.1", default-features = false, features = ["rand_distr"] }
-rand = { version = "0.9", default-features = false, features = ["std",
"std_rng", "thread_rng"] }
+half = { version = "2.1", default-features = false }
+rand = { version = "0.10", default-features = false, features = ["std",
"std_rng", "thread_rng"] }
serde = { version = "1.0", default-features = false, features = ["derive"] }
# used in examples
memmap2 = "0.9.3"
diff --git a/arrow/benches/array_from.rs b/arrow/benches/array_from.rs
index 9eff2b8fad..875b9f4ebd 100644
--- a/arrow/benches/array_from.rs
+++ b/arrow/benches/array_from.rs
@@ -22,7 +22,7 @@ use criterion::Criterion;
use arrow::array::*;
use arrow_buffer::i256;
-use rand::Rng;
+use rand::RngExt;
use std::iter::repeat_n;
use std::{hint, sync::Arc};
diff --git a/arrow/benches/bitwise_kernel.rs b/arrow/benches/bitwise_kernel.rs
index 729d48f627..e19618d7a8 100644
--- a/arrow/benches/bitwise_kernel.rs
+++ b/arrow/benches/bitwise_kernel.rs
@@ -24,7 +24,7 @@ use arrow::compute::kernels::bitwise::{
};
use arrow::datatypes::Int64Type;
use criterion::Criterion;
-use rand::RngCore;
+use rand::Rng;
use std::hint;
use arrow::util::bench_util::create_primitive_array;
diff --git a/arrow/benches/boolean_append_packed.rs
b/arrow/benches/boolean_append_packed.rs
index 5bf98741bc..c4d887f953 100644
--- a/arrow/benches/boolean_append_packed.rs
+++ b/arrow/benches/boolean_append_packed.rs
@@ -17,7 +17,7 @@
use arrow::array::BooleanBufferBuilder;
use criterion::{Criterion, criterion_group, criterion_main};
-use rand::{Rng, rng};
+use rand::{RngExt, rng};
fn rand_bytes(len: usize) -> Vec<u8> {
let mut rng = rng();
diff --git a/arrow/benches/buffer_create.rs b/arrow/benches/buffer_create.rs
index b045bea564..3ec897baf9 100644
--- a/arrow/benches/buffer_create.rs
+++ b/arrow/benches/buffer_create.rs
@@ -19,7 +19,7 @@
extern crate criterion;
use arrow::util::test_util::seedable_rng;
use criterion::Criterion;
-use rand::Rng;
+use rand::RngExt;
use rand::distr::Uniform;
use arrow::{
diff --git a/arrow/benches/builder.rs b/arrow/benches/builder.rs
index 16d082bf37..e0b2faa9b1 100644
--- a/arrow/benches/builder.rs
+++ b/arrow/benches/builder.rs
@@ -23,7 +23,7 @@ use rand::distr::StandardUniform;
use arrow::array::*;
use arrow::util::test_util::seedable_rng;
use arrow_buffer::i256;
-use rand::Rng;
+use rand::RngExt;
use std::hint;
// Build arrays with 512k elements.
diff --git a/arrow/benches/cast_kernels.rs b/arrow/benches/cast_kernels.rs
index 9ffcde2643..8afe0238ef 100644
--- a/arrow/benches/cast_kernels.rs
+++ b/arrow/benches/cast_kernels.rs
@@ -18,7 +18,7 @@
#[macro_use]
extern crate criterion;
use criterion::Criterion;
-use rand::Rng;
+use rand::RngExt;
use rand::distr::{Distribution, StandardUniform, Uniform};
use std::hint;
diff --git a/arrow/benches/coalesce_kernels.rs
b/arrow/benches/coalesce_kernels.rs
index 4affcc346e..f5036a9bcd 100644
--- a/arrow/benches/coalesce_kernels.rs
+++ b/arrow/benches/coalesce_kernels.rs
@@ -25,7 +25,7 @@ use arrow_array::types::{Float64Type, Int32Type,
TimestampNanosecondType};
use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};
use arrow_select::coalesce::BatchCoalescer;
use criterion::{Criterion, criterion_group, criterion_main};
-use rand::{Rng, SeedableRng, rngs::StdRng};
+use rand::{RngExt, SeedableRng, rngs::StdRng};
/// Benchmarks for generating evently sized output RecordBatches
/// from a sequence of filtered source batches
diff --git a/arrow/benches/comparison_kernels.rs
b/arrow/benches/comparison_kernels.rs
index 2c55676873..ce890b7f49 100644
--- a/arrow/benches/comparison_kernels.rs
+++ b/arrow/benches/comparison_kernels.rs
@@ -26,7 +26,7 @@ use arrow_buffer::IntervalMonthDayNano;
use arrow_string::like::*;
use arrow_string::regexp::regexp_is_match_scalar;
use criterion::Criterion;
-use rand::Rng;
+use rand::RngExt;
use rand::rngs::StdRng;
use std::hint;
diff --git a/arrow/benches/csv_reader.rs b/arrow/benches/csv_reader.rs
index 092a5eef2b..f4085ac928 100644
--- a/arrow/benches/csv_reader.rs
+++ b/arrow/benches/csv_reader.rs
@@ -20,7 +20,7 @@ use std::sync::Arc;
use arrow::util::bench_util::create_string_view_array_with_len;
use criterion::*;
-use rand::Rng;
+use rand::RngExt;
use arrow::array::*;
use arrow::csv;
diff --git a/arrow/benches/decimal_validate.rs
b/arrow/benches/decimal_validate.rs
index db6eb4a14e..1aced34716 100644
--- a/arrow/benches/decimal_validate.rs
+++ b/arrow/benches/decimal_validate.rs
@@ -23,7 +23,7 @@ use arrow::array::{
Decimal128Builder, Decimal256Array, Decimal256Builder,
};
use criterion::Criterion;
-use rand::Rng;
+use rand::RngExt;
use arrow_buffer::i256;
diff --git a/arrow/benches/interleave_kernels.rs
b/arrow/benches/interleave_kernels.rs
index 6eb07af9d3..310a4557c5 100644
--- a/arrow/benches/interleave_kernels.rs
+++ b/arrow/benches/interleave_kernels.rs
@@ -21,7 +21,7 @@ extern crate criterion;
use criterion::Criterion;
use std::ops::Range;
-use rand::Rng;
+use rand::RngExt;
use arrow::datatypes::*;
use arrow::util::test_util::seedable_rng;
diff --git a/arrow/benches/json_writer.rs b/arrow/benches/json_writer.rs
index adcee16488..16d35edf6d 100644
--- a/arrow/benches/json_writer.rs
+++ b/arrow/benches/json_writer.rs
@@ -26,7 +26,7 @@ use arrow::util::test_util::seedable_rng;
use arrow_array::{Array, ListArray, RecordBatch, StructArray};
use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
use arrow_json::{LineDelimitedWriter, ReaderBuilder};
-use rand::Rng;
+use rand::RngExt;
use serde::Serialize;
use std::sync::Arc;
diff --git a/arrow/benches/merge_kernels.rs b/arrow/benches/merge_kernels.rs
index 20a00981ac..2744e6618f 100644
--- a/arrow/benches/merge_kernels.rs
+++ b/arrow/benches/merge_kernels.rs
@@ -19,7 +19,7 @@ use criterion::measurement::WallTime;
use criterion::{BenchmarkGroup, BenchmarkId, Criterion, criterion_group,
criterion_main};
use rand::distr::{Distribution, StandardUniform};
use rand::prelude::StdRng;
-use rand::{Rng, SeedableRng};
+use rand::{RngExt, SeedableRng};
use std::hint;
use std::sync::Arc;
diff --git a/arrow/benches/mutable_array.rs b/arrow/benches/mutable_array.rs
index 4e5d443d25..e06cc446dd 100644
--- a/arrow/benches/mutable_array.rs
+++ b/arrow/benches/mutable_array.rs
@@ -19,7 +19,7 @@
extern crate criterion;
use criterion::Criterion;
-use rand::Rng;
+use rand::RngExt;
use arrow::util::test_util::seedable_rng;
use arrow::{array::*, util::bench_util::create_string_array};
diff --git a/arrow/benches/primitive_run_take.rs
b/arrow/benches/primitive_run_take.rs
index c394c37c6c..ab6514fd9b 100644
--- a/arrow/benches/primitive_run_take.rs
+++ b/arrow/benches/primitive_run_take.rs
@@ -22,7 +22,7 @@ use arrow::util::bench_util::*;
use arrow::util::test_util::seedable_rng;
use arrow_array::UInt32Array;
use criterion::{Criterion, criterion_group, criterion_main};
-use rand::Rng;
+use rand::RngExt;
use std::hint;
fn create_random_index(size: usize, null_density: f32, max_value: usize) ->
UInt32Array {
diff --git a/arrow/benches/string_dictionary_builder.rs
b/arrow/benches/string_dictionary_builder.rs
index a47995efb0..471ffaa506 100644
--- a/arrow/benches/string_dictionary_builder.rs
+++ b/arrow/benches/string_dictionary_builder.rs
@@ -18,7 +18,7 @@
use arrow::array::StringDictionaryBuilder;
use arrow::datatypes::Int32Type;
use criterion::{Criterion, criterion_group, criterion_main};
-use rand::{Rng, rng};
+use rand::{RngExt, rng};
/// Note: this is best effort, not all keys are necessarily present or unique
fn build_strings(dict_size: usize, total_size: usize, key_len: usize) ->
Vec<String> {
diff --git a/arrow/benches/string_run_iterator.rs
b/arrow/benches/string_run_iterator.rs
index 3008c09f09..d467715b3c 100644
--- a/arrow/benches/string_run_iterator.rs
+++ b/arrow/benches/string_run_iterator.rs
@@ -18,7 +18,7 @@
use arrow::array::{Int32RunArray, StringArray, StringRunBuilder};
use arrow::datatypes::Int32Type;
use criterion::{Criterion, criterion_group, criterion_main};
-use rand::{Rng, rng};
+use rand::{RngExt, rng};
fn build_strings_runs(
physical_array_len: usize,
diff --git a/arrow/benches/take_kernels.rs b/arrow/benches/take_kernels.rs
index 743df22e17..4e0be5b9a5 100644
--- a/arrow/benches/take_kernels.rs
+++ b/arrow/benches/take_kernels.rs
@@ -19,7 +19,7 @@
extern crate criterion;
use criterion::Criterion;
-use rand::Rng;
+use rand::RngExt;
use arrow::compute::{TakeOptions, take, take_record_batch};
use arrow::datatypes::*;
diff --git a/arrow/benches/zip_kernels.rs b/arrow/benches/zip_kernels.rs
index 65f6bb280f..c798bee3b6 100644
--- a/arrow/benches/zip_kernels.rs
+++ b/arrow/benches/zip_kernels.rs
@@ -19,7 +19,7 @@ use criterion::measurement::WallTime;
use criterion::{BenchmarkGroup, BenchmarkId, Criterion, criterion_group,
criterion_main};
use rand::distr::{Distribution, StandardUniform};
use rand::prelude::StdRng;
-use rand::{Rng, SeedableRng};
+use rand::{RngExt, SeedableRng};
use std::hint;
use std::ops::Range;
use std::sync::Arc;
diff --git a/arrow/src/util/bench_util.rs b/arrow/src/util/bench_util.rs
index 4edd0e50bc..2ec9d0de76 100644
--- a/arrow/src/util/bench_util.rs
+++ b/arrow/src/util/bench_util.rs
@@ -23,7 +23,7 @@ use crate::util::test_util::seedable_rng;
use arrow_buffer::{Buffer, IntervalMonthDayNano, NullBuffer};
use arrow_schema::Field;
use half::f16;
-use rand::Rng;
+use rand::RngExt;
use rand::SeedableRng;
use rand::distr::uniform::SampleUniform;
use rand::rng;
@@ -34,6 +34,13 @@ use rand::{
use std::ops::Range;
use std::sync::Arc;
+fn f16_random(rng: &mut StdRng) -> f16 {
+ // Otherwise it can round up to 1.0 even though we should generate in [0,
1)
+ // See: https://github.com/VoidStarKat/half-rs/issues/152
+ let one_next_down = f16::from_bits(0x3BFFu16); // 0.9995117
+ f16::from_f32(rng.random::<f32>()).clamp(f16::ZERO, one_next_down)
+}
+
/// Creates an random (but fixed-seeded) array of a given size and null density
pub fn create_primitive_array<T>(size: usize, null_density: f32) ->
PrimitiveArray<T>
where
@@ -53,6 +60,25 @@ where
.collect()
}
+/// Same as [`create_primitive_array`] but specialized for f16 since it doesn't
+/// implement the required rand traits.
+///
+/// This may be deprecated/removed in the future if half updates to the
required
+/// rand version.
+pub fn create_nullable_f16_array(size: usize, null_density: f32) ->
Float16Array {
+ let mut rng = seedable_rng();
+
+ (0..size)
+ .map(|_| {
+ if rng.random::<f32>() < null_density {
+ None
+ } else {
+ Some(f16_random(&mut rng))
+ }
+ })
+ .collect()
+}
+
/// Creates an random (but fixed-seeded) array of a given size and null
density,
/// all the values located in the given range
pub fn create_primitive_array_range<T>(
@@ -823,7 +849,7 @@ pub fn create_f16_array(size: usize, nan_density: f32) ->
Float16Array {
if rng.random::<f32>() < nan_density {
Some(f16::NAN)
} else {
- Some(rng.random())
+ Some(f16_random(&mut rng))
}
})
.collect()
diff --git a/arrow/src/util/data_gen.rs b/arrow/src/util/data_gen.rs
index 1953519df4..7f92fc7cae 100644
--- a/arrow/src/util/data_gen.rs
+++ b/arrow/src/util/data_gen.rs
@@ -20,7 +20,7 @@
use std::sync::Arc;
use rand::{
- Rng,
+ Rng, RngExt,
distr::uniform::{SampleRange, SampleUniform},
};
@@ -92,7 +92,7 @@ pub fn create_random_array(
UInt16 => Arc::new(create_primitive_array::<UInt16Type>(size,
null_density)),
UInt32 => Arc::new(create_primitive_array::<UInt32Type>(size,
null_density)),
UInt64 => Arc::new(create_primitive_array::<UInt64Type>(size,
null_density)),
- Float16 => Arc::new(create_primitive_array::<Float16Type>(size,
null_density)),
+ Float16 => Arc::new(create_nullable_f16_array(size, null_density)),
Float32 => Arc::new(create_primitive_array::<Float32Type>(size,
null_density)),
Float64 => Arc::new(create_primitive_array::<Float64Type>(size,
null_density)),
Timestamp(unit, tz) => match unit {
diff --git a/arrow/src/util/test_util.rs b/arrow/src/util/test_util.rs
index aec867bb27..7748e95ce4 100644
--- a/arrow/src/util/test_util.rs
+++ b/arrow/src/util/test_util.rs
@@ -17,7 +17,7 @@
//! Utils to make testing easier
-use rand::{Rng, SeedableRng, rngs::StdRng};
+use rand::{RngExt, SeedableRng, rngs::StdRng};
use std::{env, error::Error, fs, io::Write, path::PathBuf};
/// Returns a vector of size `n`, filled with randomly generated bytes.
