This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 50a95ef refactor!: tighten multi-sketch compatibility checks and
diagnostics (#373)
50a95ef is described below
commit 50a95efaf1c54211b7c84bb9fb768e8ec95e50b9
Author: tison <[email protected]>
AuthorDate: Sun Nov 15 17:02:37 2026 +0800
refactor!: tighten multi-sketch compatibility checks and diagnostics (#373)
---
CHANGELOG.md | 2 ++
datasketches/src/bloom/filter.rs | 42 +++++++++++++---------
datasketches/src/countmin/sketch.rs | 31 +++++++++++-----
datasketches/src/req/sketch.rs | 19 ++++++----
datasketches/src/thetafamily/common/a_not_b.rs | 38 +++++++-------------
.../src/thetafamily/common/intersection.rs | 2 +-
.../src/thetafamily/common/jaccard_similarity.rs | 2 +-
.../src/thetafamily/common/sketch_state.rs | 8 ++---
datasketches/src/thetafamily/theta/sketch.rs | 4 +--
datasketches/src/thetafamily/tuple/sketch.rs | 4 +--
datasketches/src/thetafamily/tuple/union.rs | 4 +--
tests-integration/tests/bloom_test/filter.rs | 5 +--
tests-integration/tests/countmin_test/sketch.rs | 16 +++++++--
tests-integration/tests/req_test/merge.rs | 9 +++--
tests-integration/tests/theta_test/a_not_b.rs | 1 +
tests-integration/tests/tuple_test/a_not_b.rs | 1 +
16 files changed, 105 insertions(+), 83 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2c8126c..a6414f7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ All significant changes to this project will be documented in
this file.
* `BloomFilter::capacity` now returns `u64` instead of `usize`; update callers
expecting `usize` to use `u64` or convert explicitly.
* `BloomFilter::contains` and `BloomFilter::contains_and_insert` now take
`item: T` (`T: Hash`) by value instead of `&T`, matching `BloomFilter::insert`.
Pass `&item` to retain owned values such as `String` or `Vec<u8>` without
cloning, and remove explicit turbofish type arguments (or change `T` to `&T`)
on borrowed calls such as `contains::<u64>(&value)`.
* Fix 32-bit (`wasm32`) double hashing in `BloomFilter` to use the full 64-bit
hash consistently with 64-bit targets, Java, C++, and Go. 64-bit targets and
serialized formats are unchanged; rebuild or re-ingest filters created on
32-bit targets instead of mixing them across versions.
+* Remove `BloomFilter::is_compatible`; handle `InvalidArgument` from `union`,
`intersect`, or `difference`, or compare `capacity()`, `num_hashes()`, and
`seed()` directly.
* `CountMinSketch::new` and `CountMinSketch::with_seed` now create serial
version 2 sketches, require `num_hashes` in `1..=127`, and emit images that
require a version 2 reader. Version 2 enables cross-language interchange when
using `i64` counters and the documented input encodings, while existing version
1 images remain readable, updatable, and serializable as version 1.
`CountMinSketch::merge` rejects mixing version 1 and version 2 sketches;
upgrade readers before writers and rebuild [...]
* `CpcSketch::max_serialized_bytes` and `ReqSketch::serialized_size_bytes` are
removed; serialize the sketch and inspect `len()` on the returned byte buffer
when the encoded size is needed.
* `ThetaJaccardSimilarity::compute` and `TupleJaccardSimilarity::compute` now
return `datasketches::common::JaccardSimilarity` instead of
`datasketches::thetacommon::JaccardSimilarity`, and `datasketches::thetacommon`
is no longer part of the public API.
@@ -42,6 +43,7 @@ All significant changes to this project will be documented in
this file.
* `ReqSketch::merge` now cascades compaction across newly grown levels when a
merged sketch overflows the top compactor's nominal capacity.
* `ReqSketch::deserialize` now returns `InvalidData` for out-of-range or
non-finite compactor `section_size_raw` values instead of overflowing
section-size rounding.
* `ThetaJaccardSimilarity` and `TupleJaccardSimilarity` now return
`InvalidArgument` instead of `InvalidData` when a non-empty input sketch's seed
hash does not match the configured seed.
+* `ThetaANotB::compute` and `TupleANotB::compute` now stamp empty results with
the operator's seed hash instead of copying an unchecked seed hash from an
empty left input.
## v0.6.0 (2026-10-29)
diff --git a/datasketches/src/bloom/filter.rs b/datasketches/src/bloom/filter.rs
index f5b70c3..7c66c99 100644
--- a/datasketches/src/bloom/filter.rs
+++ b/datasketches/src/bloom/filter.rs
@@ -234,8 +234,8 @@ impl BloomFilter {
///
/// # Errors
///
- /// Returns [`InvalidArgument`](ErrorKind::InvalidArgument) if
- /// [`is_compatible`](Self::is_compatible) is `false`.
+ /// Returns [`InvalidArgument`](ErrorKind::InvalidArgument) without
modifying `self` if
+ /// `self` and `other` have different capacities, hash counts, or seeds.
pub fn union(&mut self, other: &Self) -> Result<(), Error> {
self.ensure_compatible(other)?;
self.bits
@@ -247,8 +247,8 @@ impl BloomFilter {
///
/// # Errors
///
- /// Returns [`InvalidArgument`](ErrorKind::InvalidArgument) if
- /// [`is_compatible`](Self::is_compatible) is `false`.
+ /// Returns [`InvalidArgument`](ErrorKind::InvalidArgument) without
modifying `self` if
+ /// `self` and `other` have different capacities, hash counts, or seeds.
pub fn intersect(&mut self, other: &Self) -> Result<(), Error> {
self.ensure_compatible(other)?;
self.bits
@@ -263,8 +263,8 @@ impl BloomFilter {
///
/// # Errors
///
- /// Returns [`InvalidArgument`](ErrorKind::InvalidArgument) if
- /// [`is_compatible`](Self::is_compatible) is `false`.
+ /// Returns [`InvalidArgument`](ErrorKind::InvalidArgument) without
modifying `self` if
+ /// `self` and `other` have different capacities, hash counts, or seeds.
pub fn difference(&mut self, other: &Self) -> Result<(), Error> {
self.ensure_compatible(other)?;
self.bits
@@ -308,13 +308,6 @@ impl BloomFilter {
self.load_factor().powi(i32::from(self.num_hashes))
}
- /// Returns `true` if `self` and `other` have identical capacity,
`num_hashes`, and `seed`.
- pub fn is_compatible(&self, other: &Self) -> bool {
- self.bits.words.len() == other.bits.words.len()
- && self.num_hashes == other.num_hashes
- && self.seed == other.seed
- }
-
/// Returns the estimated heap allocation of the filter's backing bit
array in bytes.
pub fn estimated_size(&self) -> usize {
size_of_val::<[u64]>(&self.bits.words)
@@ -395,11 +388,26 @@ impl BloomFilter {
})
}
+ #[inline]
fn ensure_compatible(&self, other: &Self) -> Result<(), Error> {
- if !self.is_compatible(other) {
- return Err(Error::invalid_argument(
- "Bloom filters must have matching capacity, number of hashes,
and seed",
- ));
+ if self.bits.words.len() != other.bits.words.len() {
+ return Err(Error::invalid_argument(format!(
+ "cannot combine Bloom filters with different capacities:
expected {}, got {}",
+ self.capacity(),
+ other.capacity()
+ )));
+ }
+ if self.num_hashes != other.num_hashes {
+ return Err(Error::invalid_argument(format!(
+ "cannot combine Bloom filters with different num_hashes:
expected {}, got {}",
+ self.num_hashes, other.num_hashes
+ )));
+ }
+ if self.seed != other.seed {
+ return Err(Error::invalid_argument(format!(
+ "cannot combine Bloom filters with different seeds: expected
{}, got {}",
+ self.seed, other.seed
+ )));
}
Ok(())
}
diff --git a/datasketches/src/countmin/sketch.rs
b/datasketches/src/countmin/sketch.rs
index 1784cdf..79359f3 100644
--- a/datasketches/src/countmin/sketch.rs
+++ b/datasketches/src/countmin/sketch.rs
@@ -431,14 +431,29 @@ impl<T: CountMinValue> CountMinSketch<T> {
/// assert!(left.estimate(2i64) >= 2);
/// ```
pub fn merge(&mut self, other: &CountMinSketch<T>) -> Result<(), Error> {
- if self.num_hashes != other.num_hashes
- || self.buckets.count != other.buckets.count
- || self.seed != other.seed
- || self.serial_version != other.serial_version
- {
- return Err(Error::invalid_argument(
- "Count-Min sketches must have matching numbers of hashes,
bucket counts, seeds, and serial versions",
- ));
+ if self.num_hashes != other.num_hashes {
+ return Err(Error::invalid_argument(format!(
+ "cannot merge Count-Min sketches with different num_hashes:
expected {}, got {}",
+ self.num_hashes, other.num_hashes
+ )));
+ }
+ if self.buckets.count != other.buckets.count {
+ return Err(Error::invalid_argument(format!(
+ "cannot merge Count-Min sketches with different num_buckets:
expected {}, got {}",
+ self.buckets.count, other.buckets.count
+ )));
+ }
+ if self.seed != other.seed {
+ return Err(Error::invalid_argument(format!(
+ "cannot merge Count-Min sketches with different seeds:
expected {}, got {}",
+ self.seed, other.seed
+ )));
+ }
+ if self.serial_version != other.serial_version {
+ return Err(Error::invalid_argument(format!(
+ "cannot merge Count-Min sketches with different serial
versions: expected {}, got {}",
+ self.serial_version as u8, other.serial_version as u8
+ )));
}
self.total_weight = self
.total_weight
diff --git a/datasketches/src/req/sketch.rs b/datasketches/src/req/sketch.rs
index b9b4c09..261b9e3 100644
--- a/datasketches/src/req/sketch.rs
+++ b/datasketches/src/req/sketch.rs
@@ -362,19 +362,24 @@ where
/// ```
pub fn merge(&mut self, other: &Self) -> Result<(), Error> {
if self.rank_accuracy != other.rank_accuracy {
- return Err(Error::invalid_argument(
- "sketches must have the same rank_accuracy",
- ));
+ return Err(Error::invalid_argument(format!(
+ "cannot merge REQ sketches with different rank_accuracy:
expected {:?}, got {:?}",
+ self.rank_accuracy, other.rank_accuracy
+ )));
}
if other.is_empty() {
return Ok(());
}
- self.n = self
- .n
- .checked_add(other.n)
- .ok_or_else(|| Error::invalid_argument("total stream weight
overflow"))?;
+ self.n = self.n.checked_add(other.n).ok_or_else(|| {
+ Error::invalid_argument(format!(
+ "combined stream weight exceeds {}: left {}, right {}",
+ u64::MAX,
+ self.n,
+ other.n
+ ))
+ })?;
if let Some(m) = &other.min_item {
match &self.min_item {
diff --git a/datasketches/src/thetafamily/common/a_not_b.rs
b/datasketches/src/thetafamily/common/a_not_b.rs
index 34c9c62..8c9d0a4 100644
--- a/datasketches/src/thetafamily/common/a_not_b.rs
+++ b/datasketches/src/thetafamily/common/a_not_b.rs
@@ -46,11 +46,11 @@ where
A: EntrySketch,
B: KeySketch,
{
- // If A is empty the result is an (empty) copy of A. As with the union and
intersection, an
- // empty input carries no keys, so its seed is not validated.
+ // If A is empty the result is empty. As with the union and intersection,
an empty input
+ // carries no keys, so its seed is not validated and the result uses this
operator's seed hash.
let (a_seed_hash, a_theta, a_ordered) = match a.metadata() {
- ThetaFamilySketchMetadata::Empty { .. } => {
- return Ok(copy_to_compact_state(a, ordered));
+ ThetaFamilySketchMetadata::Empty => {
+ return Ok(CompactSketchState::empty(seed_hash));
}
ThetaFamilySketchMetadata::NonEmpty {
seed_hash,
@@ -67,8 +67,14 @@ where
// "A is non-empty but has no retained keys" state: B's seed and theta
must not influence
// the result.
let (b_seed_hash, b_theta, b_ordered, b_num_retained) = match b.metadata()
{
- ThetaFamilySketchMetadata::Empty { .. } => {
- return Ok(copy_to_compact_state(a, ordered));
+ ThetaFamilySketchMetadata::Empty => {
+ return Ok(CompactSketchState::compact(
+ a.entries().collect(),
+ a_theta,
+ a_seed_hash,
+ a_ordered,
+ ordered,
+ ));
}
ThetaFamilySketchMetadata::NonEmpty {
seed_hash,
@@ -158,26 +164,6 @@ where
entries
}
-fn copy_to_compact_state<S>(sketch: S, ordered: bool) ->
CompactSketchState<S::Entry>
-where
- S: EntrySketch,
-{
- let (seed_hash, theta, input_ordered) = match sketch.metadata() {
- ThetaFamilySketchMetadata::Empty { seed_hash } => {
- return CompactSketchState::empty(seed_hash);
- }
- ThetaFamilySketchMetadata::NonEmpty {
- seed_hash,
- theta,
- ordered,
- ..
- } => (seed_hash, theta, ordered),
- };
-
- let entries = sketch.entries().collect();
- CompactSketchState::compact(entries, theta, seed_hash, input_ordered,
ordered)
-}
-
const FOLD_MULTIPLIER: u64 = 0x9E37_79B9_7F4A_7C15;
#[inline]
diff --git a/datasketches/src/thetafamily/common/intersection.rs
b/datasketches/src/thetafamily/common/intersection.rs
index b58992a..f42b646 100644
--- a/datasketches/src/thetafamily/common/intersection.rs
+++ b/datasketches/src/thetafamily/common/intersection.rs
@@ -95,7 +95,7 @@ where
}
let (seed_hash, input_theta, input_ordered, input_num_retained) =
match sketch.metadata() {
- ThetaFamilySketchMetadata::Empty { .. } => {
+ ThetaFamilySketchMetadata::Empty => {
self.result_state = IntersectionResultState::Empty;
self.table = table_without_entries(&self.table, MAX_THETA);
return Ok(());
diff --git a/datasketches/src/thetafamily/common/jaccard_similarity.rs
b/datasketches/src/thetafamily/common/jaccard_similarity.rs
index 6a06e5c..1a33d56 100644
--- a/datasketches/src/thetafamily/common/jaccard_similarity.rs
+++ b/datasketches/src/thetafamily/common/jaccard_similarity.rs
@@ -225,7 +225,7 @@ fn identical_sets(
fn non_empty_count_and_theta(metadata: ThetaFamilySketchMetadata) ->
Option<(usize, u64)> {
match metadata {
- ThetaFamilySketchMetadata::Empty { .. } => None,
+ ThetaFamilySketchMetadata::Empty => None,
ThetaFamilySketchMetadata::NonEmpty {
theta,
num_retained,
diff --git a/datasketches/src/thetafamily/common/sketch_state.rs
b/datasketches/src/thetafamily/common/sketch_state.rs
index a99b76d..90c21a0 100644
--- a/datasketches/src/thetafamily/common/sketch_state.rs
+++ b/datasketches/src/thetafamily/common/sketch_state.rs
@@ -20,13 +20,11 @@ use crate::thetafamily::common::constants::MAX_THETA;
/// Observable metadata consumed by shared Theta-family set operations.
///
-/// Empty sketches cannot carry a retained count, theta, or ordering claim in
this representation.
-/// A non-empty sketch may retain zero entries when theta screened every input.
+/// Empty sketches carry neither a seed hash, retained count, theta, nor
ordering claim in this
+/// representation. A non-empty sketch may retain zero entries when theta
screened every input.
#[derive(Clone, Copy, Debug)]
pub enum ThetaFamilySketchMetadata {
- Empty {
- seed_hash: u16,
- },
+ Empty,
NonEmpty {
seed_hash: u16,
theta: u64,
diff --git a/datasketches/src/thetafamily/theta/sketch.rs
b/datasketches/src/thetafamily/theta/sketch.rs
index 5cbd381..3088406 100644
--- a/datasketches/src/thetafamily/theta/sketch.rs
+++ b/datasketches/src/thetafamily/theta/sketch.rs
@@ -186,9 +186,7 @@ impl<'a> ThetaSketchView<'a> {
impl KeySketch for ThetaSketchView<'_> {
fn metadata(self) -> ThetaFamilySketchMetadata {
if self.is_empty() {
- ThetaFamilySketchMetadata::Empty {
- seed_hash: self.seed_hash(),
- }
+ ThetaFamilySketchMetadata::Empty
} else {
ThetaFamilySketchMetadata::NonEmpty {
seed_hash: self.seed_hash(),
diff --git a/datasketches/src/thetafamily/tuple/sketch.rs
b/datasketches/src/thetafamily/tuple/sketch.rs
index 06ac774..68816d9 100644
--- a/datasketches/src/thetafamily/tuple/sketch.rs
+++ b/datasketches/src/thetafamily/tuple/sketch.rs
@@ -214,9 +214,7 @@ impl<'a, S> TupleSketchView<'a, S> {
impl<S> KeySketch for TupleSketchView<'_, S> {
fn metadata(self) -> ThetaFamilySketchMetadata {
if self.is_empty() {
- ThetaFamilySketchMetadata::Empty {
- seed_hash: self.seed_hash(),
- }
+ ThetaFamilySketchMetadata::Empty
} else {
ThetaFamilySketchMetadata::NonEmpty {
seed_hash: self.seed_hash(),
diff --git a/datasketches/src/thetafamily/tuple/union.rs
b/datasketches/src/thetafamily/tuple/union.rs
index f785bdd..60c0254 100644
--- a/datasketches/src/thetafamily/tuple/union.rs
+++ b/datasketches/src/thetafamily/tuple/union.rs
@@ -93,8 +93,8 @@ where
///
/// # Errors
///
- /// Returns an error if `sketch` was built with a different seed than this
union (its seed hash
- /// does not match).
+ /// Returns `InvalidArgument` if a non-empty `sketch` has a different seed
hash from this
+ /// union.
pub fn update<'a>(
&mut self,
sketch: impl Into<TupleSketchView<'a, P::Summary>>,
diff --git a/tests-integration/tests/bloom_test/filter.rs
b/tests-integration/tests/bloom_test/filter.rs
index 7a6d614..2e3b29c 100644
--- a/tests-integration/tests/bloom_test/filter.rs
+++ b/tests-integration/tests/bloom_test/filter.rs
@@ -240,9 +240,8 @@ fn union_intersect_and_difference_preserve_set_semantics() {
}
#[test]
-fn compatibility_checks_and_incompatible_set_operations_are_rejected() {
+fn incompatible_set_operations_are_rejected() {
let baseline = filter();
- assert!(baseline.is_compatible(&filter()));
let incompatible = [
BloomFilterBuilder::with_size(NUM_BITS, NUM_HASHES)
@@ -260,8 +259,6 @@ fn
compatibility_checks_and_incompatible_set_operations_are_rejected() {
];
for other in &incompatible {
- assert!(!baseline.is_compatible(other));
-
let mut union = baseline.clone();
assert_eq!(
union.union(other).unwrap_err().kind(),
diff --git a/tests-integration/tests/countmin_test/sketch.rs
b/tests-integration/tests/countmin_test/sketch.rs
index 3437293..5f76b8c 100644
--- a/tests-integration/tests/countmin_test/sketch.rs
+++ b/tests-integration/tests/countmin_test/sketch.rs
@@ -434,9 +434,19 @@ fn test_invalid_buckets_return_error() {
#[test]
fn test_merge_incompatible() {
let mut left = CountMinSketch::<i64>::new(3, 64).unwrap();
- let right = CountMinSketch::<i64>::new(2, 64).unwrap();
- let error = left.merge(&right).unwrap_err();
- assert_eq!(error.kind(), ErrorKind::InvalidArgument);
+ left.update_with_weight("apple", 5);
+ let before = left.clone();
+
+ let incompatible = [
+ CountMinSketch::<i64>::new(2, 64).unwrap(),
+ CountMinSketch::<i64>::new(3, 128).unwrap(),
+ CountMinSketch::<i64>::with_seed(3, 64, 42).unwrap(),
+ ];
+ for other in &incompatible {
+ let error = left.merge(other).unwrap_err();
+ assert_eq!(error.kind(), ErrorKind::InvalidArgument);
+ assert_eq!(left, before);
+ }
}
#[test]
diff --git a/tests-integration/tests/req_test/merge.rs
b/tests-integration/tests/req_test/merge.rs
index 8de74c8..8a5ba52 100644
--- a/tests-integration/tests/req_test/merge.rs
+++ b/tests-integration/tests/req_test/merge.rs
@@ -18,11 +18,10 @@
//! Merge behavior for ReqSketch.
use datasketches::common::SearchCriteria;
+use datasketches::error::ErrorKind;
use datasketches::req::RankAccuracy;
use datasketches::req::ReqSketch;
use googletest::assert_that;
-use googletest::prelude::anything;
-use googletest::prelude::err;
use googletest::prelude::lt;
use googletest::prelude::near;
@@ -108,7 +107,11 @@ fn merge_rejects_incompatible_accuracy_modes() {
let low_rank: ReqSketch<ReqF64> = ReqSketch::new(12,
RankAccuracy::LowRank).unwrap();
high_rank.update(req_f64(1.0));
- assert_that!(high_rank.merge(&low_rank), err(anything()));
+ let before = high_rank.serialize();
+
+ let err = high_rank.merge(&low_rank).unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::InvalidArgument);
+ assert_eq!(high_rank.serialize(), before);
}
#[test]
diff --git a/tests-integration/tests/theta_test/a_not_b.rs
b/tests-integration/tests/theta_test/a_not_b.rs
index f7b2ab2..2bc7f78 100644
--- a/tests-integration/tests/theta_test/a_not_b.rs
+++ b/tests-integration/tests/theta_test/a_not_b.rs
@@ -97,6 +97,7 @@ fn test_seed_mismatch_ignored_for_empty_inputs() {
let r = a_not_b.compute(&empty_other_seed, &good, true).unwrap();
assert!(r.is_empty());
+ assert_eq!(r.seed_hash(), good.seed_hash());
let r = a_not_b.compute(&good, &empty_other_seed, true).unwrap();
assert_eq!(r.num_retained(), 10);
diff --git a/tests-integration/tests/tuple_test/a_not_b.rs
b/tests-integration/tests/tuple_test/a_not_b.rs
index 941c52f..223b641 100644
--- a/tests-integration/tests/tuple_test/a_not_b.rs
+++ b/tests-integration/tests/tuple_test/a_not_b.rs
@@ -122,6 +122,7 @@ fn empty_inputs_do_not_impose_a_seed() {
let result = op.compute(&empty_other_seed, &non_empty, true).unwrap();
assert!(result.is_empty());
+ assert_eq!(result.seed_hash(), non_empty.seed_hash());
assert_eq!(result.theta64(), MAX_THETA);
assert!(!result.is_estimation_mode());
assert_eq!(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]