This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 4adb023 docs: clarify cross-language interoperability (#296)
4adb023 is described below
commit 4adb0239fd976b4fe829c788d00f4a643037802a
Author: tison <[email protected]>
AuthorDate: Thu Oct 22 07:41:11 2026 +0800
docs: clarify cross-language interoperability (#296)
---
CONTRIBUTING.md | 1 +
README.md | 2 +-
datasketches/src/bloom/sketch.rs | 6 ++++--
datasketches/src/common/inv_pow2.rs | 2 +-
datasketches/src/countmin/sketch.rs | 13 ++++++++++++-
datasketches/src/cpc/sketch.rs | 10 ++++++++--
datasketches/src/cpc/wrapper.rs | 2 ++
datasketches/src/frequencies/sketch.rs | 15 ++++++++++-----
datasketches/src/hash/value/canonical_float.rs | 4 ++--
datasketches/src/hash/value/mod.rs | 16 ++++++++--------
datasketches/src/hash/value/natural_extend.rs | 2 +-
datasketches/src/hash/value/raw_bytes.rs | 6 +++---
datasketches/src/hash/value/sign_extend.rs | 3 +--
datasketches/src/hll/aux_map.rs | 2 +-
datasketches/src/hll/composite_interpolation.rs | 4 ++--
datasketches/src/hll/coupon_mapping.rs | 2 +-
datasketches/src/hll/estimator.rs | 6 +++---
datasketches/src/hll/harmonic_numbers.rs | 2 +-
datasketches/src/hll/mod.rs | 6 +++---
datasketches/src/hll/sketch.rs | 8 ++++++--
datasketches/src/kll/mod.rs | 3 +--
datasketches/src/kll/serialization.rs | 6 +++---
datasketches/src/kll/sketch.rs | 10 ++++++++--
datasketches/src/lib.rs | 8 +++-----
datasketches/src/req/compactor.rs | 8 ++++----
datasketches/src/req/mod.rs | 3 +--
datasketches/src/req/serialization.rs | 4 ++--
datasketches/src/req/sketch.rs | 16 ++++++++++------
datasketches/src/tdigest/sketch.rs | 19 +++++++++++++++----
datasketches/src/thetafamily/common/constants.rs | 2 +-
datasketches/src/thetafamily/common/hash_table.rs | 2 +-
datasketches/src/thetafamily/theta/sketch.rs | 17 +++++++++++++----
datasketches/src/thetafamily/tuple/serialization.rs | 2 +-
datasketches/src/thetafamily/tuple/sketch.rs | 12 +++++++++---
tests-integration/tests/serde_tests/bloom.rs | 2 +-
tests-integration/tests/serde_tests/countmin.rs | 4 ++--
tests-integration/tests/serde_tests/frequencies.rs | 2 +-
tests-integration/tests/serde_tests/kll.rs | 10 ----------
tests-integration/tests/serde_tests/req.rs | 2 --
tests-integration/tests/serde_tests/tdigest.rs | 8 +++++---
tests-integration/tests/serde_tests/theta.rs | 6 +++---
41 files changed, 155 insertions(+), 103 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8d9f5ba..c02d8d1 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -81,6 +81,7 @@ cargo bench --package benchmarks --bench benchmarks --
cpc::serde
- End summary sentences with punctuation, and format Rust identifiers,
literals, and numeric ranges as inline code.
- Put contract sections and compatibility notes before examples. When
applicable, order sections as `# Errors`, `# Panics`, and `# Examples`. Include
only sections that describe an actual contract.
- Document return values and error conditions on the relevant methods rather
than repeating them in crate or module overviews.
+- Describe shared capabilities in language-neutral terms. Name another
implementation when it clarifies interoperability requirements, a specific
source, a format variant, or a behavior difference. Retain useful
implementation references in internal comments.
- Keep imports visible in doctests. A trailing hidden `Ok(...)` may be used to
support `?`.
## Query semantics
diff --git a/README.md b/README.md
index a4a0373..7ec87ca 100644
--- a/README.md
+++ b/README.md
@@ -84,7 +84,7 @@ The minimum supported Rust version is 1.86.0. The crate
currently supports littl
Supported serialization formats are tested with fixtures produced by Apache
DataSketches Java, C++, and Go through the [DataSketches
TCK](https://github.com/apache/datasketches-tck).
-Serialization compatibility does not imply that an ordinary Rust `Hash`
implementation produces the same update bytes as another language. When
sketches must represent the same inputs across implementations, use
`hash::value::{raw_bytes, canonical_float, sign_extend, natural_extend}` (and
the constructors within those modules) to match the other language
implementations’ hashing rules. Other DataSketches implementations skip empty
strings, so skip them before updating when that behavior [...]
+Serialization compatibility does not imply that an ordinary Rust `Hash`
implementation produces the same update bytes across languages. When sketches
must represent the same inputs across implementations, use
`hash::value::{raw_bytes, canonical_float, sign_extend, natural_extend}` (and
the constructors within those modules) to select matching input encodings. When
interoperating with an implementation that skips empty strings, skip them
before updating here as well.
See the [changelog](CHANGELOG.md) for release notes and migration guidance.
diff --git a/datasketches/src/bloom/sketch.rs b/datasketches/src/bloom/sketch.rs
index fb311a8..0d399d5 100644
--- a/datasketches/src/bloom/sketch.rs
+++ b/datasketches/src/bloom/sketch.rs
@@ -261,7 +261,7 @@ impl BloomFilter {
/// * Items never inserted into this filter may return `true` (false
positives), at a rate no
/// higher than this filter's false positive rate before the operation.
///
- /// This is the transformation other DataSketches libraries expose as A
NOT B.
+ /// This operation is also known as A NOT B.
///
/// # Errors
///
@@ -379,7 +379,7 @@ impl BloomFilter {
/// Serializes the filter to a byte vector.
///
- /// The format is compatible with other Apache DataSketches
implementations.
+ /// The serialized data can be read by other DataSketches implementations.
///
/// # Examples
///
@@ -441,6 +441,8 @@ impl BloomFilter {
/// Deserializes a filter from bytes.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// # Errors
///
/// Returns an error if:
diff --git a/datasketches/src/common/inv_pow2.rs
b/datasketches/src/common/inv_pow2.rs
index a61f1ca..64fa808 100644
--- a/datasketches/src/common/inv_pow2.rs
+++ b/datasketches/src/common/inv_pow2.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-/// Compute 1 / 2^exp using the same bit construction as DataSketches Java.
+/// Returns `2^-exp` by constructing the floating-point exponent bits, as in
datasketches-java.
#[inline]
pub fn inv_pow2(exp: u8) -> f64 {
f64::from_bits((1023 - exp as u64) << 52)
diff --git a/datasketches/src/countmin/sketch.rs
b/datasketches/src/countmin/sketch.rs
index 6c02dc0..24a60e4 100644
--- a/datasketches/src/countmin/sketch.rs
+++ b/datasketches/src/countmin/sketch.rs
@@ -306,7 +306,10 @@ impl<T: CountMinValue> CountMinSketch<T> {
Ok(())
}
- /// Serializes this sketch into the DataSketches CountMin format.
+ /// Serializes this sketch into the Count-Min binary format.
+ ///
+ /// DataSketches C++ can read the serialized data, but item queries and
updates there are
+ /// unreliable because the two implementations map items to counters
differently.
///
/// # Examples
///
@@ -357,6 +360,10 @@ impl<T: CountMinValue> CountMinSketch<T> {
/// Deserializes a sketch from bytes using the default seed.
///
+ /// Can read sketches serialized by DataSketches C++.
+ /// Item queries and updates on those sketches are unreliable because the
two
+ /// implementations map items to counters differently.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the image is malformed or its seed hash does
not match the default
@@ -379,6 +386,10 @@ impl<T: CountMinValue> CountMinSketch<T> {
/// Deserializes a sketch from bytes using the provided seed.
///
+ /// Can read sketches serialized by DataSketches C++.
+ /// Item queries and updates on those sketches are unreliable because the
two
+ /// implementations map items to counters differently.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the image is malformed, its seed hash does
not match `seed`, or
diff --git a/datasketches/src/cpc/sketch.rs b/datasketches/src/cpc/sketch.rs
index d946b1b..c42b367 100644
--- a/datasketches/src/cpc/sketch.rs
+++ b/datasketches/src/cpc/sketch.rs
@@ -179,8 +179,8 @@ impl CpcSketch {
/// Updates the sketch with a hashable value.
///
- /// You may use [`hash::value`](crate::hash::value) wrappers when another
DataSketches
- /// implementation requires a specific value hashing strategy.
+ /// Use [`hash::value`](crate::hash::value) wrappers to select an input
encoding for
+ /// cross-language hashing.
///
/// # Examples
///
@@ -490,6 +490,8 @@ impl fmt::Debug for CpcSketch {
impl CpcSketch {
/// Serializes this `CpcSketch` to bytes.
+ ///
+ /// The serialized data can be read by other DataSketches implementations.
pub fn serialize(&self) -> Vec<u8> {
let flavor = self.flavor();
let has_hip = !self.merge_flag;
@@ -623,6 +625,8 @@ impl CpcSketch {
/// Deserializes a `CpcSketch` from bytes.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the image is malformed or its seed hash does
not match the default
@@ -633,6 +637,8 @@ impl CpcSketch {
/// Deserializes a `CpcSketch` from bytes with the provided seed.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the image is malformed, its seed hash does
not match `seed`, or
diff --git a/datasketches/src/cpc/wrapper.rs b/datasketches/src/cpc/wrapper.rs
index b32df27..8263e0b 100644
--- a/datasketches/src/cpc/wrapper.rs
+++ b/datasketches/src/cpc/wrapper.rs
@@ -47,6 +47,8 @@ impl CpcWrapper {
/// Reads the cardinality metadata without copying or fully deserializing
the compressed
/// payload.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the preamble is malformed or does not
describe a compressed CPC
diff --git a/datasketches/src/frequencies/sketch.rs
b/datasketches/src/frequencies/sketch.rs
index 1eed1a7..4808ef5 100644
--- a/datasketches/src/frequencies/sketch.rs
+++ b/datasketches/src/frequencies/sketch.rs
@@ -40,9 +40,8 @@ type DeserializeItems<T> = fn(SketchSlice<'_>, usize) ->
Result<Vec<T>, Error>;
const LG_MIN_MAP_SIZE: u8 = 3;
const MIN_MAP_SIZE: usize = 1 << LG_MIN_MAP_SIZE;
-// Java represents map sizes as positive `int` powers of two, while the C++
-// implementation uses 32-bit table indices. Keep Rust configurations within
-// the same cross-language range.
+// datasketches-java represents map sizes as positive `int` powers of two,
while datasketches-cpp
+// uses 32-bit table indices. Keep Rust configurations within the same
cross-language range.
const LG_MAX_MAP_SIZE: u8 = 30;
const MAX_MAP_SIZE: usize = 1 << LG_MAX_MAP_SIZE;
const SAMPLE_SIZE: usize = 1024;
@@ -680,8 +679,8 @@ impl<T: Eq + Hash> FrequentItemsSketch<T> {
return Err(Error::deserial("empty flag does not match preamble
longs"));
}
if is_empty {
- // Java also restores empty images at the minimum size. `lg_cur`
does
- // not carry item state here and must not control an eager
allocation.
+ // datasketches-java also restores empty images at the minimum
size. `lg_cur` carries
+ // no item state here and must not control an eager allocation.
return Ok(Self::with_lg_map_sizes(lg_max, LG_MIN_MAP_SIZE));
}
@@ -755,6 +754,9 @@ impl<T: Eq + Hash> FrequentItemsSketch<T> {
impl<T: FrequentItemValue> FrequentItemsSketch<T> {
/// Serializes this sketch into a byte vector.
///
+ /// The serialized data can be read by other DataSketches implementations
when both
+ /// sides use the same [`FrequentItemValue`] item encoding.
+ ///
/// # Examples
///
/// Built-in support for `i64`:
@@ -787,6 +789,9 @@ impl<T: FrequentItemValue> FrequentItemsSketch<T> {
/// Deserializes a sketch from bytes.
///
+ /// Also accepts data serialized by other DataSketches implementations
when both
+ /// sides use the same [`FrequentItemValue`] item encoding.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the image is truncated, its metadata is
inconsistent, or an item
diff --git a/datasketches/src/hash/value/canonical_float.rs
b/datasketches/src/hash/value/canonical_float.rs
index 06edd98..8b31232 100644
--- a/datasketches/src/hash/value/canonical_float.rs
+++ b/datasketches/src/hash/value/canonical_float.rs
@@ -21,7 +21,7 @@
//! hashing. Signed zero values hash the same, all NaN values use one
canonical NaN bit pattern,
//! and equal `f32`/`f64` values hash the same.
//!
-//! This strategy is compatible with how other datasketches hashes
floating-point numbers.
+//! Use this encoding to match floating-point hashing in DataSketches Java and
DataSketches C++.
use std::hash::Hash;
use std::hash::Hasher;
@@ -109,7 +109,7 @@ impl HashStrategy<f64> for CanonicalFloatStrategy {
#[inline(always)]
fn hash<H: Hasher>(value: &f64, state: &mut H) {
let canonical = if value.is_nan() {
- // Java's Double.doubleToLongBits() NaN value.
+ // Canonical NaN used by datasketches-java via
Double.doubleToLongBits().
0x7ff8000000000000u64
} else {
// -0.0 + 0.0 == +0.0 under IEEE754 roundTiesToEven rounding mode,
diff --git a/datasketches/src/hash/value/mod.rs
b/datasketches/src/hash/value/mod.rs
index 88d87e7..a4867a3 100644
--- a/datasketches/src/hash/value/mod.rs
+++ b/datasketches/src/hash/value/mod.rs
@@ -27,7 +27,7 @@
//! pattern before hashing. Signed zero values hash the same, all NaN values
use one canonical NaN
//! bit pattern, and equal `f32`/`f64` values hash the same.
//!
-//! This strategy is the same as how other datasketches implementations hash
floating-point numbers.
+//! Use this encoding to match floating-point hashing in DataSketches Java and
DataSketches C++.
//!
//! The concrete wrapper documentation provides more details and examples.
//!
@@ -37,8 +37,8 @@
//! ## Integers
//!
//! [`sign_extend::SignExtend`] first sign-extends values to 64-bits, and then
hashes the resulting
-//! integers. This strategy is the same as how datasketches-cpp hashes short
integers for
-//! `HllSketch` and `CpcSketch`.
+//! integers. Use this encoding to match short-integer hashing in DataSketches
C++'s HLL and CPC
+//! sketches.
//!
//! The concrete wrapper documentation provides more details and examples.
//!
@@ -47,8 +47,8 @@
//! * [`sign_extend::from_i32`], [`sign_extend::from_u32`]
//!
//! [`natural_extend::NaturalExtend`] first widens signed values to `i64` and
unsigned values to
-//! `u64`, and then hashes the resulting integers. This strategy is the same
as how datasketches-cpp
-//! hashes short integers for `BloomFilter`.
+//! `u64`, and then hashes the resulting integers. Use this encoding to match
short-integer hashing
+//! in DataSketches C++'s Bloom filters.
//!
//! The concrete wrapper documentation provides more details and examples.
//!
@@ -61,9 +61,9 @@
//! [`raw_bytes::RawBytes`] hashes byte and string inputs as raw bytes without
Rust's slice or
//! string length prefix.
//!
-//! Empty byte and string inputs have zero bytes to hash. Other datasketches
implementations skip
-//! empty strings before hashing, so check `is_empty` before updating a sketch
when that behavior
-//! matters.
+//! Use this encoding to match byte and UTF-8 string hashing in DataSketches
Java and
+//! DataSketches C++. Their string update APIs skip empty strings; check
`is_empty` before updating
+//! here as well.
//!
//! The concrete wrapper documentation provides more details and examples.
//!
diff --git a/datasketches/src/hash/value/natural_extend.rs
b/datasketches/src/hash/value/natural_extend.rs
index 981c8b1..29d7fcf 100644
--- a/datasketches/src/hash/value/natural_extend.rs
+++ b/datasketches/src/hash/value/natural_extend.rs
@@ -20,7 +20,7 @@
//! [`NaturalExtend`] first widens signed values to `i64` and unsigned values
to `u64`, and then
//! hashes the resulting integers.
//!
-//! This strategy is the same as how datasketches-cpp hashes short integers
for `BloomFilter`.
+//! Use this encoding to match short-integer hashing in DataSketches C++'s
Bloom filters.
use std::hash::Hash;
use std::hash::Hasher;
diff --git a/datasketches/src/hash/value/raw_bytes.rs
b/datasketches/src/hash/value/raw_bytes.rs
index 0c48139..a31c397 100644
--- a/datasketches/src/hash/value/raw_bytes.rs
+++ b/datasketches/src/hash/value/raw_bytes.rs
@@ -20,9 +20,9 @@
//! [`RawBytes`] hashes byte and string inputs as raw bytes without Rust's
slice or string length
//! prefix.
//!
-//! Empty byte and string inputs have zero bytes to hash. Other datasketches
implementations skip
-//! empty strings before hashing, so check `is_empty` before updating a sketch
when that behavior
-//! matters.
+//! Use this encoding to match byte and UTF-8 string hashing in DataSketches
Java and
+//! DataSketches C++. Their string update APIs skip empty strings; check
`is_empty` before updating
+//! here as well.
use std::hash::Hasher;
diff --git a/datasketches/src/hash/value/sign_extend.rs
b/datasketches/src/hash/value/sign_extend.rs
index cefcafe..eaaec9d 100644
--- a/datasketches/src/hash/value/sign_extend.rs
+++ b/datasketches/src/hash/value/sign_extend.rs
@@ -19,8 +19,7 @@
//!
//! [`SignExtend`] first sign-extends values to 64-bits and then hashes the
resulting integers.
//!
-//! This strategy is the same as how datasketches-cpp hashes short integers
for `HllSketch` and
-//! `CpcSketch`.
+//! Use this encoding to match short-integer hashing in DataSketches C++'s HLL
and CPC sketches.
use std::hash::Hash;
use std::hash::Hasher;
diff --git a/datasketches/src/hll/aux_map.rs b/datasketches/src/hll/aux_map.rs
index 5286fe9..e14ef31 100644
--- a/datasketches/src/hll/aux_map.rs
+++ b/datasketches/src/hll/aux_map.rs
@@ -55,7 +55,7 @@ impl PartialEq for AuxMap {
}
}
-/// Initial table sizes from DataSketches C++:
+/// Initial table sizes from datasketches-cpp:
///
<https://github.com/apache/datasketches-cpp/blob/5a055521/hll/include/HllUtil.hpp>
fn lg_aux_arr_ints(lg_config_k: u8) -> u8 {
static LG_AUX_ARR_INTS: &[u8] = &[
diff --git a/datasketches/src/hll/composite_interpolation.rs
b/datasketches/src/hll/composite_interpolation.rs
index cf419bf..a028a80 100644
--- a/datasketches/src/hll/composite_interpolation.rs
+++ b/datasketches/src/hll/composite_interpolation.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-//! Bias-correction tables for `lg_k` 4 through 21 from DataSketches C++:
+//! Bias-correction tables for `lg_k` 4 through 21 from datasketches-cpp:
//!
<https://github.com/apache/datasketches-cpp/blob/5a055521/hll/include/CompositeInterpolationXTable-internal.hpp>
const NUM_X_VALUES: usize = 257;
@@ -4769,7 +4769,7 @@ mod tests {
let x_arr = get_x_arr(4);
assert_eq!(x_arr.len(), 257);
- // Check first few values match the C++ data
+ // Check first few values match the datasketches-cpp data.
assert_that!(x_arr[0], near(10.767999803534, 1e-6));
assert_that!(x_arr[1], near(11.237701481774, 1e-6));
}
diff --git a/datasketches/src/hll/coupon_mapping.rs
b/datasketches/src/hll/coupon_mapping.rs
index 0ee8d43..f702d8e 100644
--- a/datasketches/src/hll/coupon_mapping.rs
+++ b/datasketches/src/hll/coupon_mapping.rs
@@ -16,7 +16,7 @@
// under the License.
//! Coupon-count correction for the 26-bit slot encoding. These X/Y pairs are
used with
-//! cubic interpolation and come from DataSketches C++:
+//! cubic interpolation and come from datasketches-cpp:
//!
<https://github.com/apache/datasketches-cpp/blob/5a055521/hll/include/CubicInterpolation-internal.hpp>
/// X values (coupon counts) for interpolation table
diff --git a/datasketches/src/hll/estimator.rs
b/datasketches/src/hll/estimator.rs
index 457dff8..a72fcca 100644
--- a/datasketches/src/hll/estimator.rs
+++ b/datasketches/src/hll/estimator.rs
@@ -20,7 +20,7 @@
//! Sequential register updates use HIP (Historical Inverse Probability). Bulk
merges use the
//! composite estimator because register values do not retain their update
order.
//!
-//! Estimator formulas and crossover constants follow DataSketches C++:
+//! Estimator formulas and crossover constants follow datasketches-cpp:
//!
<https://github.com/apache/datasketches-cpp/blob/5a055521/hll/include/HllArray-internal.hpp>
use crate::common::NumStdDev;
@@ -274,7 +274,7 @@ impl Estimator {
}
/// Relative error used as `estimate / (1 + error)`: negative for upper bounds.
-/// Follows DataSketches C++ `HllUtil::getRelErr`:
+/// Follows datasketches-cpp's `HllUtil::getRelErr`:
///
<https://github.com/apache/datasketches-cpp/blob/5a055521/hll/include/HllUtil.hpp>
fn relative_error(
lg_config_k: u8,
@@ -283,7 +283,7 @@ fn relative_error(
num_std_dev: NumStdDev,
) -> f64 {
if lg_config_k > 12 {
- // RSE factors from Apache DataSketches C++ implementation
+ // RSE factors from datasketches-cpp.
// HLL_HIP_RSE_FACTOR = sqrt(ln(2)) ≈ 0.8325546
// HLL_NON_HIP_RSE_FACTOR = sqrt((3 * ln(2)) - 1) ≈ 1.03896
let rse_factor = if composite {
diff --git a/datasketches/src/hll/harmonic_numbers.rs
b/datasketches/src/hll/harmonic_numbers.rs
index cef22c7..8e6c983 100644
--- a/datasketches/src/hll/harmonic_numbers.rs
+++ b/datasketches/src/hll/harmonic_numbers.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-//! Harmonic-number approximation used by the bitmap estimator, following
DataSketches C++:
+//! Harmonic-number approximation used by the bitmap estimator, following
datasketches-cpp:
//!
<https://github.com/apache/datasketches-cpp/blob/5a055521/hll/include/HarmonicNumbers-internal.hpp>
const NUM_EXACT: usize = 25;
diff --git a/datasketches/src/hll/mod.rs b/datasketches/src/hll/mod.rs
index fdec977..06db8d3 100644
--- a/datasketches/src/hll/mod.rs
+++ b/datasketches/src/hll/mod.rs
@@ -20,7 +20,7 @@
//! Use [`HllSketch`] to collect values and [`HllUnion`] to combine sketches.
//! [`HllType`] controls register storage; `lg_k` controls the register count
and accuracy.
//!
-//! Serialized sketches are compatible with Apache DataSketches in Java and
C++.
+//! The binary serialization format supports cross-language interchange.
//! Cross-language updates must also use compatible hashing; see
//! [`hash::value`](crate::hash::value).
//!
@@ -149,8 +149,8 @@ impl Coupon {
/// Computes the HLL coupon for a hashable value.
///
- /// Use [`hash::value`](crate::hash::value) wrappers when another
DataSketches
- /// implementation requires a specific value hashing strategy.
+ /// Use [`hash::value`](crate::hash::value) wrappers to select an input
encoding for
+ /// cross-language hashing.
///
/// # Examples
///
diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs
index 06dd823..92d4d5b 100644
--- a/datasketches/src/hll/sketch.rs
+++ b/datasketches/src/hll/sketch.rs
@@ -137,8 +137,8 @@ impl HllSketch {
/// Updates the sketch with a value.
///
- /// You may use [`hash::value`](crate::hash::value) wrappers when another
DataSketches
- /// implementation requires a specific value hashing strategy.
+ /// Use [`hash::value`](crate::hash::value) wrappers to select an input
encoding for
+ /// cross-language hashing.
///
/// To reuse the hash across sketches, see [`Coupon`] and
/// [`update_with_coupon`](Self::update_with_coupon).
@@ -314,6 +314,8 @@ impl HllSketch {
/// Deserializes an HLL sketch from bytes.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the image is truncated or contains an invalid
preamble,
@@ -443,6 +445,8 @@ impl HllSketch {
/// Serializes the HLL sketch to bytes.
///
+ /// The serialized data can be read by other DataSketches implementations.
+ ///
/// # Examples
///
/// ```
diff --git a/datasketches/src/kll/mod.rs b/datasketches/src/kll/mod.rs
index 59e0fd6..e6cf6cf 100644
--- a/datasketches/src/kll/mod.rs
+++ b/datasketches/src/kll/mod.rs
@@ -21,8 +21,7 @@
//! near-optimal accuracy per retained item. It supports one-pass updates,
//! approximate quantiles, ranks, PMF, and CDF queries.
//!
-//! This implementation follows Apache DataSketches semantics and uses the
compact binary
-//! serialization format shared by the Java, C++, and Go implementations.
+//! The compact binary serialization format supports cross-language
interchange.
//!
//! Items must implement [`Ord`]. Wrap `f32` or `f64` values in [`KllFloat`],
which rejects NaN and
//! provides their ordinary numerical order. Custom ordering should be
expressed with a newtype
diff --git a/datasketches/src/kll/serialization.rs
b/datasketches/src/kll/serialization.rs
index 41e585c..e9450df 100644
--- a/datasketches/src/kll/serialization.rs
+++ b/datasketches/src/kll/serialization.rs
@@ -17,9 +17,9 @@
//! Binary serialization format constants for KLL sketches.
//!
-//! Naming and layout follow the Apache DataSketches Java implementation
-//! (`KllPreambleUtil`) and the C++ `kll_sketch` compact serialization format.
-//! Java's updatable-memory layout is not a cross-language wire format and is
+//! Naming and layout follow datasketches-java's `KllPreambleUtil` and
datasketches-cpp's
+//! `kll_sketch` compact serialization format.
+//! datasketches-java's updatable-memory layout is not a cross-language wire
format and is
//! intentionally outside this module's scope.
/// Serialization version for empty or full sketches
(KllPreambleUtil.SERIAL_VERSION_EMPTY_FULL).
diff --git a/datasketches/src/kll/sketch.rs b/datasketches/src/kll/sketch.rs
index 891c4bb..f90a07d 100644
--- a/datasketches/src/kll/sketch.rs
+++ b/datasketches/src/kll/sketch.rs
@@ -636,12 +636,18 @@ fn deserialize_with_serde<T: KllValue + Ord>(bytes:
&[u8]) -> Result<KllSketch<T
}
impl<T: KllValue + Ord> KllSketch<T> {
- /// Serializes the sketch to bytes.
+ /// Serializes the sketch to bytes in compact form.
+ ///
+ /// The serialized data can be read by other DataSketches implementations
when both
+ /// sides use the same [`KllValue`] item encoding.
pub fn serialize(&self) -> Vec<u8> {
serialize_with_serde(self)
}
- /// Deserializes a sketch from bytes.
+ /// Deserializes a sketch from bytes in compact form.
+ ///
+ /// Also accepts data serialized by other DataSketches implementations
when both
+ /// sides use the same [`KllValue`] item encoding.
///
/// # Errors
///
diff --git a/datasketches/src/lib.rs b/datasketches/src/lib.rs
index b4ff016..813a841 100644
--- a/datasketches/src/lib.rs
+++ b/datasketches/src/lib.rs
@@ -18,8 +18,6 @@
//! # Apache® DataSketches™ Core Rust Library Component
//!
//! This crate provides compact, mergeable summaries for answering queries
over large data streams.
-//! It implements a subset of the algorithms available in the other Apache
DataSketches language
-//! components.
//!
//! ## Enabling sketches
//!
@@ -49,7 +47,7 @@
//! ## Cross-language hashing
//!
//! Compatible serialization does not by itself make ordinary Rust
[`Hash`](std::hash::Hash)
-//! input compatible with Java, C++, or Go. Rust strings and slices include
type-specific framing,
+//! input compatible across languages. Rust strings and slices include
type-specific framing,
//! and short integers require different widening rules for different sketch
families. When
//! sketches must represent the same updates across languages, use the
wrappers in [`hash::value`]:
//!
@@ -58,8 +56,8 @@
//! * `sign_extend` for short integers used with HLL and CPC;
//! * `natural_extend` for short integers used with Bloom filters.
//!
-//! Other DataSketches implementations skip empty strings rather than hashing
them. Check for empty
-//! input before updating when that cross-language behavior is required.
+//! When interoperating with an implementation that skips empty strings, check
for empty input
+//! before updating here as well.
#![cfg_attr(docsrs, feature(doc_cfg))]
#![deny(missing_docs)]
diff --git a/datasketches/src/req/compactor.rs
b/datasketches/src/req/compactor.rs
index 1a4aa64..c6f6461 100644
--- a/datasketches/src/req/compactor.rs
+++ b/datasketches/src/req/compactor.rs
@@ -114,7 +114,7 @@ where
}
}
// OR-ing the schedule counters can advance state past several doubling
- // thresholds at once. Loop until no more doublings are needed (C++:
+ // thresholds at once. Loop until no more doublings are needed
(datasketches-cpp:
// req_compactor_impl.hpp:250 — `while (ensure_enough_sections()) {}`).
while self.ensure_enough_sections() {}
}
@@ -364,8 +364,8 @@ where
///
/// The wire format omits the compactor preamble for tiny sketches (n ≤
4); this
/// helper synthesises a fresh compactor and seeds it with the
deserialized items.
- /// A false wire flag stays false for byte-stable C++/Java round trips; a
true flag
- /// is cleared if the items are not actually sorted.
+ /// A false wire flag stays false for byte-stable datasketches-cpp and
datasketches-java round
+ /// trips; a true flag is cleared if the items are not actually sorted.
pub fn raw_items_compactor(
k: u16,
rank_accuracy: RankAccuracy,
@@ -513,7 +513,7 @@ mod tests {
// ensure_enough_sections() once instead of looping. Without the loop,
// num_sections doubles at most once per merge — but OR-ing a high
state
// can advance past several doubling thresholds at once and require
- // multiple doublings (matching the C++ reference at
+ // multiple doublings (matching datasketches-cpp at
// req_compactor_impl.hpp:250 — `while (ensure_enough_sections()) {}`).
//
// Setup: a fresh compactor (state=0, num_sections=3) merged with
another
diff --git a/datasketches/src/req/mod.rs b/datasketches/src/req/mod.rs
index 40b787c..91bc2ba 100644
--- a/datasketches/src/req/mod.rs
+++ b/datasketches/src/req/mod.rs
@@ -19,8 +19,7 @@
//!
//! [`ReqSketch`] provides bounded-memory rank, quantile, PMF, and CDF
estimates with
//! configurable relative accuracy at either end of the rank domain. It is
based on
-//! [Relative Error Streaming Quantiles](https://arxiv.org/abs/2004.01668) and
the Apache
-//! DataSketches C++ implementation.
+//! [Relative Error Streaming Quantiles](https://arxiv.org/abs/2004.01668).
//!
//! # Item ordering
//!
diff --git a/datasketches/src/req/serialization.rs
b/datasketches/src/req/serialization.rs
index 90c949a..c4d1f23 100644
--- a/datasketches/src/req/serialization.rs
+++ b/datasketches/src/req/serialization.rs
@@ -29,8 +29,8 @@ pub const PREAMBLE_INTS_EXACT: u8 = 2;
pub const PREAMBLE_INTS_ESTIMATION: u8 = 4;
pub const RAW_ITEMS_THRESHOLD: u64 = 4;
-/// Flag bits — match the C++ enum order: RESERVED1, RESERVED2, IS_EMPTY,
IS_HIGH_RANK, RAW_ITEMS,
-/// IS_LEVEL_ZERO_SORTED.
+// Match the datasketches-cpp enum order: RESERVED1, RESERVED2, IS_EMPTY,
IS_HIGH_RANK, RAW_ITEMS,
+// IS_LEVEL_ZERO_SORTED.
pub const FLAG_IS_EMPTY: u8 = 1 << 2;
pub const FLAG_IS_HIGH_RANK: u8 = 1 << 3;
pub const FLAG_RAW_ITEMS: u8 = 1 << 4;
diff --git a/datasketches/src/req/sketch.rs b/datasketches/src/req/sketch.rs
index 446906f..fb0245e 100644
--- a/datasketches/src/req/sketch.rs
+++ b/datasketches/src/req/sketch.rs
@@ -535,7 +535,10 @@ where
size
}
- /// Serializes the sketch using the C++/Java-compatible REQ wire format.
+ /// Serializes the sketch using the REQ wire format.
+ ///
+ /// The serialized data can be read by other DataSketches implementations
when both
+ /// sides use the same [`ReqValue`] item encoding.
pub fn serialize(&self) -> Vec<u8>
where
T: ReqValue,
@@ -583,8 +586,10 @@ where
out.into_bytes()
}
- /// Deserialize a sketch from bytes produced by [`Self::serialize`] or by
the
- /// C++/Java reference implementations.
+ /// Deserializes a sketch from bytes in the REQ wire format.
+ ///
+ /// Also accepts data serialized by other DataSketches implementations
when both
+ /// sides use the same [`ReqValue`] item encoding.
///
/// # Errors
///
@@ -803,9 +808,8 @@ where
min_item: None,
max_item: None,
};
- // C++ parity: an empty sketch has a level-0 compactor present from
the start.
- // This makes is_raw_items() and flags_byte() byte-compatible with the
C++/Java
- // wire format for the empty case.
+ // Keep a level-0 compactor, as in datasketches-cpp, so is_raw_items()
and flags_byte()
+ // produce the same empty encoding as datasketches-cpp and
datasketches-java.
sketch.grow();
sketch
}
diff --git a/datasketches/src/tdigest/sketch.rs
b/datasketches/src/tdigest/sketch.rs
index d7599af..1fb4b16 100644
--- a/datasketches/src/tdigest/sketch.rs
+++ b/datasketches/src/tdigest/sketch.rs
@@ -554,7 +554,9 @@ impl TDigestMut {
Ok(self.view().quantiles(ranks))
}
- /// Serializes this mutable t-digest to bytes.
+ /// Serializes this mutable t-digest to bytes in double precision.
+ ///
+ /// The serialized data can be read by other DataSketches implementations.
///
/// # Examples
///
@@ -581,6 +583,8 @@ impl TDigestMut {
/// Deserializes a mutable t-digest from the standard double-precision
format.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// The format of the [reference
implementation](https://github.com/tdunning/t-digest) is
/// auto-detected. Use [`deserialize_f32()`](Self::deserialize_f32) for
the compact
/// DataSketches C++ `tdigest<float>` format.
@@ -608,8 +612,8 @@ impl TDigestMut {
/// Deserializes a mutable t-digest from the compact single-precision
DataSketches format.
///
- /// This format stores centroid means and weights as `(f32, u32)` and is
emitted by the C++
- /// `tdigest<float>` implementation. Its header does not identify the
scalar width, so callers
+ /// Accepts data serialized by `tdigest<float>` in DataSketches C++. This
format stores centroid
+ /// means and weights as `(f32, u32)`. Its header does not identify the
scalar width, so callers
/// must select this entry point explicitly.
///
/// # Errors
@@ -1368,7 +1372,9 @@ impl TDigest {
self.centroids_weight
}
- /// Serializes this immutable t-digest to bytes.
+ /// Serializes this immutable t-digest to bytes in double precision.
+ ///
+ /// The serialized data can be read by other DataSketches implementations.
///
/// # Examples
///
@@ -1396,6 +1402,8 @@ impl TDigest {
/// Deserializes an immutable t-digest from the standard double-precision
format.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// The format of the [reference
implementation](https://github.com/tdunning/t-digest) is
/// auto-detected. Use [`deserialize_f32()`](Self::deserialize_f32) for
the compact
/// DataSketches C++ `tdigest<float>` format.
@@ -1409,6 +1417,9 @@ impl TDigest {
/// Deserializes an immutable t-digest from the compact single-precision
DataSketches format.
///
+ /// Accepts data serialized by `tdigest<float>` in DataSketches C++; see
+ /// [`TDigestMut::deserialize_f32`] for the encoding details.
+ ///
/// # Errors
///
/// Returns `InvalidData` under the same conditions as
[`TDigestMut::deserialize_f32`].
diff --git a/datasketches/src/thetafamily/common/constants.rs
b/datasketches/src/thetafamily/common/constants.rs
index 8f0ee6e..8b86f78 100644
--- a/datasketches/src/thetafamily/common/constants.rs
+++ b/datasketches/src/thetafamily/common/constants.rs
@@ -17,7 +17,7 @@
/// Maximum theta value.
///
-/// The value is `i64::MAX` to be compatible with datasketches-java.
+/// The shared format uses a 63-bit hash domain, bounded by `i64::MAX`, as in
datasketches-java.
pub const MAX_THETA: u64 = i64::MAX as u64;
/// Minimum log2 of K.
diff --git a/datasketches/src/thetafamily/common/hash_table.rs
b/datasketches/src/thetafamily/common/hash_table.rs
index 5525f74..d2138c9 100644
--- a/datasketches/src/thetafamily/common/hash_table.rs
+++ b/datasketches/src/thetafamily/common/hash_table.rs
@@ -173,7 +173,7 @@ where
let mut hasher = MurmurHash3X64128::with_seed(self.seed);
value.hash(&mut hasher);
let (h1, _) = hasher.finish128();
- h1 >> 1 // To make it compatible with Java version
+ h1 >> 1 // Match the 63-bit hash domain used by datasketches-java's
Theta and Tuple sketches.
}
/// Inserts or updates the entry slot for a pre-hashed key.
diff --git a/datasketches/src/thetafamily/theta/sketch.rs
b/datasketches/src/thetafamily/theta/sketch.rs
index 2c87383..b75d1af 100644
--- a/datasketches/src/thetafamily/theta/sketch.rs
+++ b/datasketches/src/thetafamily/theta/sketch.rs
@@ -218,8 +218,8 @@ impl ThetaSketch {
/// Updates the sketch with a hashable value.
///
- /// You may use [`hash::value`](crate::hash::value) wrappers when another
DataSketches
- /// implementation requires a specific value hashing strategy.
+ /// Use [`hash::value`](crate::hash::value) wrappers to select an input
encoding for
+ /// cross-language hashing.
///
/// # Examples
///
@@ -268,7 +268,7 @@ impl ThetaSketch {
/// Returns theta as a `u64`.
///
/// An empty sketch reports `MAX_THETA` even when it was built with a
sampling probability
- /// below `1.0`, matching the other DataSketches implementations.
+ /// below `1.0`.
pub fn theta64(&self) -> u64 {
if self.is_empty {
MAX_THETA
@@ -546,6 +546,8 @@ impl CompactThetaSketch {
/// Serializes this sketch in compressed form if applicable.
///
+ /// The serialized data can be read by other DataSketches implementations.
+ ///
/// Falls back to [`serialize`](Self::serialize) when compression is not
applicable.
pub fn serialize_compressed(&self) -> Vec<u8> {
if self.is_suitable_for_compression() {
@@ -563,6 +565,8 @@ impl CompactThetaSketch {
/// Serializes this sketch into the uncompressed compact Theta format.
///
+ /// The serialized data can be read by other DataSketches implementations.
+ ///
/// Empty sketches serialize with a zero seed hash.
pub fn serialize(&self) -> Vec<u8> {
if self.is_empty() {
@@ -590,7 +594,8 @@ impl CompactThetaSketch {
if pre_longs > 1 {
bytes.write_u32_le(retained_hashes.len() as u32);
- bytes.write_u32_be(0); // not used by compact sketches; match
Java/C++
+ // Unused by compact sketches; matches datasketches-java and
datasketches-cpp.
+ bytes.write_u32_be(0);
}
if self.is_estimation_mode() {
bytes.write_u64_le(self.theta64());
@@ -689,6 +694,8 @@ impl CompactThetaSketch {
/// Deserializes a compact Theta sketch using the default seed.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the image is malformed or a non-empty image's
seed hash does not
@@ -699,6 +706,8 @@ impl CompactThetaSketch {
/// Deserializes a compact Theta sketch using `seed`.
///
+ /// Also accepts data serialized by other DataSketches implementations.
+ ///
/// Empty sketches use the hash of `seed` regardless of the stored seed
hash.
///
/// # Errors
diff --git a/datasketches/src/thetafamily/tuple/serialization.rs
b/datasketches/src/thetafamily/tuple/serialization.rs
index d44a104..cc208e0 100644
--- a/datasketches/src/thetafamily/tuple/serialization.rs
+++ b/datasketches/src/thetafamily/tuple/serialization.rs
@@ -19,7 +19,7 @@
//!
//! The Tuple compact format reuses the uncompressed Theta layout (preamble,
flags, theta) but uses
//! the Tuple family id, carries a sketch-type byte, and stores the user
summary bytes after each
-//! retained hash. See the C++/Java reference implementations for the on-disk
format.
+//! retained hash. See datasketches-cpp and datasketches-java for the on-disk
format.
//!
//! A Tuple sketch keeps a user-defined summary next to every retained key.
Because the summary
//! type is opaque to the sketch, (de)serialization of summaries is delegated
to the summary type
diff --git a/datasketches/src/thetafamily/tuple/sketch.rs
b/datasketches/src/thetafamily/tuple/sketch.rs
index 509ef16..e3a690f 100644
--- a/datasketches/src/thetafamily/tuple/sketch.rs
+++ b/datasketches/src/thetafamily/tuple/sketch.rs
@@ -331,7 +331,7 @@ where
/// Returns theta as a `u64`.
///
/// An empty sketch reports `MAX_THETA` even when it was built with a
sampling probability
- /// below `1.0`, matching the other DataSketches implementations.
+ /// below `1.0`.
pub fn theta64(&self) -> u64 {
if self.is_empty {
MAX_THETA
@@ -559,8 +559,8 @@ impl<S> CompactTupleSketch<S> {
/// Serializes this sketch into the compact Tuple binary format.
///
- /// Uses [`TupleSummaryValue`] to encode summaries. Reading the output in
Java or C++ requires
- /// a compatible summary encoding.
+ /// The serialized data can be read by other DataSketches implementations
when both
+ /// sides use the same [`TupleSummaryValue`] summary encoding.
///
/// Empty sketches serialize with a zero seed hash.
///
@@ -622,6 +622,9 @@ impl<S> CompactTupleSketch<S> {
/// Deserializes a compact Tuple sketch using the default seed.
///
+ /// Also accepts data serialized by other DataSketches implementations
when both
+ /// sides use the same [`TupleSummaryValue`] summary encoding.
+ ///
/// # Errors
///
/// Returns `InvalidData` if the image is malformed or a non-empty image's
seed hash does not
@@ -636,6 +639,9 @@ impl<S> CompactTupleSketch<S> {
/// Deserializes a compact Tuple sketch using `seed`.
///
+ /// Also accepts data serialized by other DataSketches implementations
when both
+ /// sides use the same [`TupleSummaryValue`] summary encoding.
+ ///
/// Empty sketches use the hash of `seed` regardless of the stored seed
hash.
///
/// # Errors
diff --git a/tests-integration/tests/serde_tests/bloom.rs
b/tests-integration/tests/serde_tests/bloom.rs
index 4c3e712..1416645 100644
--- a/tests-integration/tests/serde_tests/bloom.rs
+++ b/tests-integration/tests/serde_tests/bloom.rs
@@ -55,7 +55,7 @@ fn test_bloom_filter_file(path: PathBuf, expected_num_items:
u64, expected_num_h
}
// Verify the items that were inserted (integers 0 to n/10-1)
- // C++ code: for (uint64_t i = 0; i < n / 10; ++i) bf.update(i);
+ // datasketches-cpp code: for (uint64_t i = 0; i < n / 10; ++i)
bf.update(i);
let num_inserted = expected_num_items / 10;
if num_inserted > 0 {
diff --git a/tests-integration/tests/serde_tests/countmin.rs
b/tests-integration/tests/serde_tests/countmin.rs
index 07786b4..4a62faa 100644
--- a/tests-integration/tests/serde_tests/countmin.rs
+++ b/tests-integration/tests/serde_tests/countmin.rs
@@ -44,8 +44,8 @@ fn rejects_weights_that_cannot_bound_counter_arithmetic() {
}
// This test validates binary format compatibility (deserialize + byte
round-trip) for
-// C++ CountMin snapshots. It intentionally does not assert estimate
equivalence against
-// original input keys because per-row hash seed derivation differs across
implementations.
+// datasketches-cpp CountMin snapshots. It intentionally does not assert
estimate equivalence
+// against original input keys because per-row hash seed derivation differs
across implementations.
fn assert_cpp_snapshot(
filename: &str,
seed: u64,
diff --git a/tests-integration/tests/serde_tests/frequencies.rs
b/tests-integration/tests/serde_tests/frequencies.rs
index 31c25cb..9c1df36 100644
--- a/tests-integration/tests/serde_tests/frequencies.rs
+++ b/tests-integration/tests/serde_tests/frequencies.rs
@@ -109,7 +109,7 @@ fn test_non_clone_item_round_trip() {
fn test_empty_round_trip() {
let sketch = FrequentItemsSketch::<i64>::new(32).unwrap();
let bytes = sketch.serialize();
- // One preamble long, matching the Java and C++ empty encoding.
+ // One preamble long, matching the datasketches-java and datasketches-cpp
empty encoding.
assert_eq!(bytes.len(), 8);
let restored = FrequentItemsSketch::<i64>::deserialize(&bytes).unwrap();
assert!(restored.is_empty());
diff --git a/tests-integration/tests/serde_tests/kll.rs
b/tests-integration/tests/serde_tests/kll.rs
index 44ba22b..c6ab416 100644
--- a/tests-integration/tests/serde_tests/kll.rs
+++ b/tests-integration/tests/serde_tests/kll.rs
@@ -15,16 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-//! KLL Sketch Serialization Compatibility Tests
-//!
-//! These tests verify binary compatibility with Apache DataSketches
implementations:
-//! - Java (datasketches-java)
-//! - C++ (datasketches-cpp)
-//! - Go (datasketches-go)
-//!
-//! Test data is generated by the reference implementations and stored in:
-//! `tests/serde_tests/{java_generated_files,cpp_generated_files}/`.
-
use std::cmp::Ordering;
use std::fs;
use std::path::PathBuf;
diff --git a/tests-integration/tests/serde_tests/req.rs
b/tests-integration/tests/serde_tests/req.rs
index a3a920f..7c14763 100644
--- a/tests-integration/tests/serde_tests/req.rs
+++ b/tests-integration/tests/serde_tests/req.rs
@@ -15,8 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-//! Serialization round-trip and cross-language compatibility tests for
ReqSketch.
-
use std::fs;
use std::path::PathBuf;
diff --git a/tests-integration/tests/serde_tests/tdigest.rs
b/tests-integration/tests/serde_tests/tdigest.rs
index 8bcd989..a426e78 100644
--- a/tests-integration/tests/serde_tests/tdigest.rs
+++ b/tests-integration/tests/serde_tests/tdigest.rs
@@ -311,8 +311,9 @@ fn
test_updates_normalize_overfull_deserialized_buffer_without_centroids() {
assert_eq!(&bytes[8..12], &0_u32.to_le_bytes()); // num centroids
assert_eq!(&bytes[12..16], &10_u32.to_le_bytes()); // num buffered
- // k=100 normally compresses at 840 buffered values. Extend a real C++
image without centroids
- // just past that producer threshold while keeping every value within the
recorded min/max.
+ // k=100 normally compresses at 840 buffered values. Extend a
datasketches-cpp image without
+ // centroids just past that producer threshold while keeping every value
within the recorded
+ // min/max.
bytes[12..16].copy_from_slice(&841_u32.to_le_bytes());
for _ in 0..831 {
bytes.extend_from_slice(&10_f64.to_le_bytes());
@@ -345,7 +346,8 @@ fn
test_updates_normalize_overfull_deserialized_mixed_buffer() {
assert_eq!(&bytes[8..12], &89_u32.to_le_bytes()); // num centroids
assert_eq!(&bytes[12..16], &160_u32.to_le_bytes()); // num buffered
- // Extend the buffered tail of a real mixed C++ image just past the k=100
producer threshold.
+ // Extend the buffered tail of a mixed datasketches-cpp image just past
the k=100 producer
+ // threshold.
bytes[12..16].copy_from_slice(&841_u32.to_le_bytes());
for _ in 0..681 {
bytes.extend_from_slice(&1_000_f64.to_le_bytes());
diff --git a/tests-integration/tests/serde_tests/theta.rs
b/tests-integration/tests/serde_tests/theta.rs
index b51fc2d..4f6dd0f 100644
--- a/tests-integration/tests/serde_tests/theta.rs
+++ b/tests-integration/tests/serde_tests/theta.rs
@@ -299,12 +299,12 @@ fn test_v2_java_empty_layouts() {
.seed_hash();
let other_seed_hash =
ThetaSketchBuilder::default().build().unwrap().seed_hash();
- // Port of Java 8.0.0 ForwardCompatibilityTest's
checkSerVer2_{1,2,3}PreLong[s]_Empty:
- //
https://github.com/apache/datasketches-java/blob/8.0.0/src/test/java/org/apache/datasketches/theta/ForwardCompatibilityTest.java#L90-L137
+ // Port of datasketches-java 8.0.0 ForwardCompatibilityTest's
+ // checkSerVer2_{1,2,3}PreLong[s]_Empty:
https://github.com/apache/datasketches-java/blob/8.0.0/src/test/java/org/apache/datasketches/theta/ForwardCompatibilityTest.java#L90-L137
for pre_longs in [1, 2, 3] {
let mut bytes = serialize_v2_exact(&[]);
bytes[0] = pre_longs;
- bytes[5] = 0x0e; // Java's v2 empty flags.
+ bytes[5] = 0x0e; // datasketches-java's v2 empty flags.
bytes.resize(pre_longs as usize * 8, 0);
if pre_longs == 3 {
bytes[16..24].copy_from_slice(&(i64::MAX as u64).to_le_bytes());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
