This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
The following commit(s) were added to refs/heads/main by this push:
new fe962f9 refactor!: align stream-weight and retained-entry getters
across sketches (#375)
fe962f9 is described below
commit fe962f9a48548668791b1bbeec97b61127ee6d31
Author: tison <[email protected]>
AuthorDate: Sun Nov 15 22:49:41 2026 +0800
refactor!: align stream-weight and retained-entry getters across sketches
(#375)
---
CHANGELOG.md | 3 ++
datasketches/src/frequencies/sketch.rs | 12 ++++----
datasketches/src/kll/sketch.rs | 14 ++++++----
datasketches/src/req/sketch.rs | 8 +++---
datasketches/src/tdigest/sketch.rs | 13 ++++++++-
tests-integration/tests/frequencies_test/update.rs | 28 +++++++++----------
tests-integration/tests/kll_test/core.rs | 10 +++----
tests-integration/tests/kll_test/generic.rs | 2 +-
tests-integration/tests/kll_test/merge.rs | 6 ++--
tests-integration/tests/kll_test/query.rs | 4 +--
tests-integration/tests/req_test/accuracy.rs | 2 +-
tests-integration/tests/req_test/bounds.rs | 2 +-
tests-integration/tests/req_test/core.rs | 12 ++++----
tests-integration/tests/req_test/merge.rs | 6 ++--
tests-integration/tests/req_test/query.rs | 4 +--
tests-integration/tests/req_test/structure.rs | 2 +-
tests-integration/tests/serde_tests/frequencies.rs | 6 ++--
tests-integration/tests/serde_tests/kll.rs | 10 +++----
tests-integration/tests/serde_tests/req.rs | 9 ++++--
tests-integration/tests/tdigest_test/sketch.rs | 32 ++++++++++++++++++++++
20 files changed, 120 insertions(+), 65 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index dac3ca8..79ecd9b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,12 +19,15 @@ All significant changes to this project will be documented
in this file.
* `ReqSketch::rank_lower_bound` and `ReqSketch::rank_upper_bound` now return
`Result<f64, Error>`, returning `InvalidArgument` when `rank` is NaN or outside
`[0.0, 1.0]` even on empty or exact-mode sketches.
* `SketchSlice::advance` now takes `n: usize` instead of `u64` and panics if
`n > self.remaining().len()`.
* Remove `PartialEq` from `BloomFilter`, `CountMinSketch`, `HllSketch`,
`JaccardSimilarity`, and `KllSketch`.
+* Rename `KllSketch::n` and `ReqSketch::n` to `total_weight` to match
stream-weight naming across the crate; update callers of `n()` to call
`total_weight()`.
+* Rename `FrequentItemsSketch::num_active_items` to `num_retained` to match
retained-entry naming across the crate; update callers of `num_active_items()`
to call `num_retained()`.
### New features
* Add `KllSketch::estimated_size` and `ReqSketch::estimated_size` to estimate
sketch memory usage in bytes.
* Add `CpcUnion::serialize`, `HllUnion::serialize`, `ThetaUnion::serialize`,
`ThetaUnion::serialize_compressed`, and `TupleUnion::serialize` to encode union
results without calling `to_sketch`. Direct serialization avoids intermediate
sketch allocations for `HllUnion` and `CpcUnion`, lets `HllUnion::serialize`
target any `HllType` (`Hll4`, `Hll6`, or `Hll8`) at the union's effective
`lg_k`, supports v4 compressed output via `ThetaUnion::serialize_compressed`,
and encodes `TupleUnion` sum [...]
* Add `XorFilter`, `XorFilterBuilder`, and `XorFilterType` behind the `xor`
feature for immutable membership queries on values or precomputed hashes.
Filters support 8-, 16-, and 32-bit fingerprints and can be serialized for
later queries.
+* Add `TDigest::num_retained` and `TDigestMut::num_retained` to report the
number of retained entries or centroids in a t-digest.
### Improvements
diff --git a/datasketches/src/frequencies/sketch.rs
b/datasketches/src/frequencies/sketch.rs
index a4cec1d..a5b63f2 100644
--- a/datasketches/src/frequencies/sketch.rs
+++ b/datasketches/src/frequencies/sketch.rs
@@ -158,7 +158,7 @@ impl<T> fmt::Debug for FrequentItemsSketch<T> {
.field("lg_max_map_size", &self.lg_max_map_size)
.field("cur_map_cap", &self.cur_map_cap)
.field("is_empty", &(self.stream_weight == 0))
- .field("num_active_items", &self.hash_map.num_active())
+ .field("num_retained", &self.hash_map.num_active())
.field("total_weight", &self.stream_weight)
.field("maximum_error", &self.offset)
.finish()
@@ -184,7 +184,7 @@ impl<T: Eq + Hash> FrequentItemsSketch<T> {
/// let mut sketch = FrequentItemsSketch::<i64>::new(64).unwrap();
/// sketch.update(1);
/// sketch.update(2);
- /// assert_eq!(sketch.num_active_items(), 2);
+ /// assert_eq!(sketch.num_retained(), 2);
/// ```
pub fn new(max_map_size: usize) -> Result<Self, Error> {
let lg_max_map_size = lg_for_max_map_size(max_map_size)?;
@@ -194,14 +194,14 @@ impl<T: Eq + Hash> FrequentItemsSketch<T> {
/// Returns `true` if the total stream weight is zero.
///
/// A purge can remove all active items while retaining a non-zero total
weight and
- /// maximum error. Such a sketch is not empty. Use
[`Self::num_active_items`] to check
+ /// maximum error. Such a sketch is not empty. Use [`Self::num_retained`]
to check
/// whether any items are retained.
pub fn is_empty(&self) -> bool {
self.stream_weight == 0
}
- /// Returns the number of active items being tracked.
- pub fn num_active_items(&self) -> usize {
+ /// Returns the number of retained items currently tracked in the map.
+ pub fn num_retained(&self) -> usize {
self.hash_map.num_active()
}
@@ -627,7 +627,7 @@ impl<T: Eq + Hash> FrequentItemsSketch<T> {
return bytes.into_bytes();
}
- let active_items = self.num_active_items();
+ let active_items = self.num_retained();
let active_entries = self.hash_map.active_entries();
let mut bytes = SketchBytes::with_capacity({
diff --git a/datasketches/src/kll/sketch.rs b/datasketches/src/kll/sketch.rs
index 848aaf2..ea836a4 100644
--- a/datasketches/src/kll/sketch.rs
+++ b/datasketches/src/kll/sketch.rs
@@ -69,7 +69,7 @@ impl<T: fmt::Debug> fmt::Debug for KllSketch<T> {
f.debug_struct("KllSketch")
.field("k", &self.k)
.field("min_k", &self.min_k)
- .field("n", &self.n)
+ .field("total_weight", &self.n)
.field("is_empty", &(self.n == 0))
.field("is_estimation_mode", &(self.levels.len() > 1))
.field("num_levels", &self.levels.len())
@@ -120,8 +120,8 @@ impl<T: Clone + Ord> KllSketch<T> {
self.min_k
}
- /// Returns total weight of the stream.
- pub fn n(&self) -> u64 {
+ /// Returns the total stream weight.
+ pub fn total_weight(&self) -> u64 {
self.n
}
@@ -219,7 +219,11 @@ impl<T: Clone + Ord> KllSketch<T> {
self.min_k = self.min_k.min(other.min_k);
}
- debug_assert_eq!(self.total_weight(), self.n, "total weight does not
match n");
+ debug_assert_eq!(
+ self.levels_weight(),
+ self.n,
+ "total weight does not match n"
+ );
Ok(())
}
@@ -867,7 +871,7 @@ impl<T: Clone + Ord> KllSketch<T> {
self.capacity = total_capacity(self.k, self.m, self.levels.len()) as
usize;
}
- fn total_weight(&self) -> u64 {
+ fn levels_weight(&self) -> u64 {
self.levels
.iter()
.enumerate()
diff --git a/datasketches/src/req/sketch.rs b/datasketches/src/req/sketch.rs
index fa452d2..161ebf1 100644
--- a/datasketches/src/req/sketch.rs
+++ b/datasketches/src/req/sketch.rs
@@ -67,7 +67,7 @@ impl<T: fmt::Debug> fmt::Debug for ReqSketch<T> {
f.debug_struct("ReqSketch")
.field("k", &self.k)
.field("rank_accuracy", &self.rank_accuracy)
- .field("n", &self.n)
+ .field("total_weight", &self.n)
.field("is_empty", &(self.n == 0))
.field("is_estimation_mode", &(self.compactors.len() > 1))
.field("num_levels", &self.compactors.len())
@@ -120,8 +120,8 @@ where
self.rank_accuracy
}
- /// Returns the total number of items observed.
- pub fn n(&self) -> u64 {
+ /// Returns the total stream weight.
+ pub fn total_weight(&self) -> u64 {
self.n
}
@@ -360,7 +360,7 @@ where
/// combined.merge(&first).unwrap();
/// combined.merge(&second).unwrap();
///
- /// assert_eq!(combined.n(), 2);
+ /// assert_eq!(combined.total_weight(), 2);
/// # Ok::<(), datasketches::error::Error>(())
/// ```
pub fn merge(&mut self, other: &Self) -> Result<(), Error> {
diff --git a/datasketches/src/tdigest/sketch.rs
b/datasketches/src/tdigest/sketch.rs
index e5055ce..a68474c 100644
--- a/datasketches/src/tdigest/sketch.rs
+++ b/datasketches/src/tdigest/sketch.rs
@@ -177,6 +177,12 @@ impl TDigestMut {
self.compressed_weight + self.buffer.unmerged_len() as u64
}
+ /// Returns the number of retained entries in this t-digest, including
compressed centroids and
+ /// buffered unmerged values.
+ pub fn num_retained(&self) -> usize {
+ self.buffer.len()
+ }
+
/// Merges the given t-digest into this one.
///
/// Retains this digest's `k`, even if the other digest uses a different
value.
@@ -625,7 +631,7 @@ impl fmt::Debug for TDigest {
.field("k", &self.k)
.field("is_empty", &self.is_empty())
.field("total_weight", &self.total_weight())
- .field("num_centroids", &self.centroids.len())
+ .field("num_retained", &self.centroids.len())
.field("min_value", &self.min_value())
.field("max_value", &self.max_value())
.finish()
@@ -666,6 +672,11 @@ impl TDigest {
self.centroids_weight
}
+ /// Returns the number of retained centroids in this t-digest.
+ pub fn num_retained(&self) -> usize {
+ self.centroids.len()
+ }
+
/// Serializes this immutable t-digest to bytes in double precision.
///
/// The serialized data can be read by other DataSketches implementations.
diff --git a/tests-integration/tests/frequencies_test/update.rs
b/tests-integration/tests/frequencies_test/update.rs
index 82d6b40..d4b9fc7 100644
--- a/tests-integration/tests/frequencies_test/update.rs
+++ b/tests-integration/tests/frequencies_test/update.rs
@@ -36,7 +36,7 @@ fn test_longs_update_with_zero_count_is_noop() {
assert!(sketch.is_empty());
assert_eq!(sketch.total_weight(), 0);
- assert_eq!(sketch.num_active_items(), 0);
+ assert_eq!(sketch.num_retained(), 0);
}
#[test]
@@ -73,7 +73,7 @@ fn test_longs_empty() {
let sketch: FrequentItemsSketch<i64> =
FrequentItemsSketch::new(8).unwrap();
assert!(sketch.is_empty());
- assert_eq!(sketch.num_active_items(), 0);
+ assert_eq!(sketch.num_retained(), 0);
assert_eq!(sketch.total_weight(), 0);
assert_eq!(sketch.estimate(&42), 0);
assert_eq!(sketch.lower_bound(&42), 0);
@@ -97,7 +97,7 @@ fn test_items_borrowed_key_updates_and_queries() {
assert!(!sketch.is_empty());
assert_eq!(sketch.total_weight(), 15);
- assert_eq!(sketch.num_active_items(), 12);
+ assert_eq!(sketch.num_retained(), 12);
assert_eq!(sketch.estimate("alpha"), 2);
assert_eq!(sketch.lower_bound("alpha"), 2);
assert_eq!(sketch.upper_bound("alpha"), 2);
@@ -122,7 +122,7 @@ fn test_longs_several_items_no_resize_no_purge() {
assert!(!sketch.is_empty());
assert_eq!(sketch.total_weight(), 7);
- assert_eq!(sketch.num_active_items(), 4);
+ assert_eq!(sketch.num_retained(), 4);
assert_eq!(sketch.estimate(&1), 1);
assert_eq!(sketch.estimate(&2), 3);
assert_eq!(sketch.estimate(&3), 2);
@@ -147,7 +147,7 @@ fn test_items_several_items_no_resize_no_purge() {
assert!(!sketch.is_empty());
assert_eq!(sketch.total_weight(), 7);
- assert_eq!(sketch.num_active_items(), 4);
+ assert_eq!(sketch.num_retained(), 4);
assert_eq!(sketch.estimate(&a), 1);
assert_eq!(sketch.estimate(&b), 3);
assert_eq!(sketch.estimate(&c), 2);
@@ -163,7 +163,7 @@ fn test_items_several_items_no_resize_no_purge() {
sketch.reset();
assert!(sketch.is_empty());
- assert_eq!(sketch.num_active_items(), 0);
+ assert_eq!(sketch.num_retained(), 0);
assert_eq!(sketch.total_weight(), 0);
}
@@ -187,7 +187,7 @@ fn test_items_several_items_with_resize_no_purge() {
assert!(!sketch.is_empty());
assert_eq!(sketch.total_weight(), 15);
- assert_eq!(sketch.num_active_items(), 12);
+ assert_eq!(sketch.num_retained(), 12);
assert_eq!(sketch.estimate(&a), 1);
assert_eq!(sketch.estimate(&b), 3);
assert_eq!(sketch.estimate(&c), 2);
@@ -261,7 +261,7 @@ fn test_longs_merge_estimation_mode() {
sketch1.merge(&sketch2);
assert!(!sketch1.is_empty());
assert_eq!(sketch1.total_weight(), 46);
- assert_that!(sketch1.num_active_items(), ge(2));
+ assert_that!(sketch1.num_retained(), ge(2));
let items =
sketch1.frequent_items_with_threshold(ErrorType::NoFalsePositives, 2);
assert_eq!(items.len(), 2);
@@ -289,7 +289,7 @@ fn test_items_merge_exact_mode() {
assert!(!sketch1.is_empty());
assert_eq!(sketch1.total_weight(), 5);
- assert_eq!(sketch1.num_active_items(), 3);
+ assert_eq!(sketch1.num_retained(), 3);
assert_eq!(sketch1.estimate(&a), 1);
assert_eq!(sketch1.estimate(&b), 3);
assert_eq!(sketch1.estimate(&c), 1);
@@ -305,7 +305,7 @@ fn test_longs_merge_empty_is_noop() {
sketch.merge(&empty);
assert_eq!(sketch.total_weight(), 1);
- assert_eq!(sketch.num_active_items(), 1);
+ assert_eq!(sketch.num_retained(), 1);
assert_eq!(sketch.estimate(&1), 1);
}
@@ -316,7 +316,7 @@ fn test_purged_state_survives_merge_and_reset() {
purged.update(item);
}
assert!(!purged.is_empty());
- assert_eq!(purged.num_active_items(), 0);
+ assert_eq!(purged.num_retained(), 0);
assert_eq!(purged.total_weight(), 193);
assert_eq!(purged.maximum_error(), 1);
@@ -324,7 +324,7 @@ fn test_purged_state_survives_merge_and_reset() {
merged.merge(&purged);
assert!(!merged.is_empty());
- assert_eq!(merged.num_active_items(), 0);
+ assert_eq!(merged.num_retained(), 0);
assert_eq!(merged.total_weight(), purged.total_weight());
assert_eq!(merged.maximum_error(), purged.maximum_error());
assert_eq!(merged.upper_bound(&0), 1);
@@ -365,7 +365,7 @@ fn test_merge_weight_overflow_preserves_state() {
for item in 0..7 {
purged.update(item);
}
- assert_eq!(purged.num_active_items(), 0);
+ assert_eq!(purged.num_retained(), 0);
sketch.merge(&purged);
assert_eq!(sketch.total_weight(), u64::MAX);
@@ -385,7 +385,7 @@ fn test_longs_reset() {
assert!(sketch.is_empty());
assert_eq!(sketch.total_weight(), 0);
- assert_eq!(sketch.num_active_items(), 0);
+ assert_eq!(sketch.num_retained(), 0);
assert_eq!(sketch.lg_max_map_size(), 3);
}
diff --git a/tests-integration/tests/kll_test/core.rs
b/tests-integration/tests/kll_test/core.rs
index 0505649..381301b 100644
--- a/tests-integration/tests/kll_test/core.rs
+++ b/tests-integration/tests/kll_test/core.rs
@@ -40,7 +40,7 @@ fn empty_and_reset_state() {
let mut sketch = KllSketch::<i64>::new(64).unwrap();
assert!(sketch.is_empty());
assert!(!sketch.is_estimation_mode());
- assert_eq!(sketch.n(), 0);
+ assert_eq!(sketch.total_weight(), 0);
assert_eq!(sketch.num_retained(), 0);
assert_eq!(sketch.min_item(), None);
assert_eq!(sketch.max_item(), None);
@@ -56,7 +56,7 @@ fn empty_and_reset_state() {
assert_eq!(sketch.min_k(), 64);
assert!(sketch.is_empty());
assert!(!sketch.is_estimation_mode());
- assert_eq!(sketch.n(), 0);
+ assert_eq!(sketch.total_weight(), 0);
assert_eq!(sketch.num_retained(), 0);
assert_eq!(sketch.min_item(), None);
assert_eq!(sketch.max_item(), None);
@@ -79,11 +79,11 @@ fn
retained_count_stays_consistent_through_compaction_and_roundtrip() {
let mut sketch = KllSketch::<i64>::new(32).unwrap();
for item in 0..100_000 {
sketch.update(item);
- assert!(sketch.num_retained() <= sketch.n() as usize);
+ assert!(sketch.num_retained() <= sketch.total_weight() as usize);
}
let decoded = KllSketch::<i64>::deserialize(&sketch.serialize()).unwrap();
- assert_eq!(decoded.n(), sketch.n());
+ assert_eq!(decoded.total_weight(), sketch.total_weight());
assert_eq!(decoded.num_retained(), sketch.num_retained());
assert_eq!(decoded.min_item(), Some(&0));
assert_eq!(decoded.max_item(), Some(&99_999));
@@ -99,7 +99,7 @@ fn weight_overflow_preserves_state() {
sketch.merge(&sketch.clone()).unwrap();
sketch.update(0);
}
- assert_eq!(sketch.n(), u64::MAX);
+ assert_eq!(sketch.total_weight(), u64::MAX);
let before = sketch.serialize();
assert!(catch_unwind(AssertUnwindSafe(|| sketch.update(1))).is_err());
diff --git a/tests-integration/tests/kll_test/generic.rs
b/tests-integration/tests/kll_test/generic.rs
index 544f679..1e3ed50 100644
--- a/tests-integration/tests/kll_test/generic.rs
+++ b/tests-integration/tests/kll_test/generic.rs
@@ -77,7 +77,7 @@ fn
custom_item_order_controls_queries_and_survives_roundtrip() {
);
let decoded =
KllSketch::<NumericString>::deserialize(&sketch.serialize()).unwrap();
- assert_eq!(decoded.n(), sketch.n());
+ assert_eq!(decoded.total_weight(), sketch.total_weight());
assert_eq!(decoded.num_retained(), sketch.num_retained());
assert_eq!(decoded.min_item().map(|item| item.0.as_str()), Some("1"));
assert_eq!(decoded.max_item().map(|item| item.0.as_str()), Some("10"));
diff --git a/tests-integration/tests/kll_test/merge.rs
b/tests-integration/tests/kll_test/merge.rs
index 513f0db..24797a5 100644
--- a/tests-integration/tests/kll_test/merge.rs
+++ b/tests-integration/tests/kll_test/merge.rs
@@ -29,10 +29,10 @@ fn merge_preserves_weight_extrema_and_query_invariants() {
left.merge(&right).unwrap();
- assert_eq!(left.n(), 20_000);
+ assert_eq!(left.total_weight(), 20_000);
assert_eq!(left.min_item(), Some(&0));
assert_eq!(left.max_item(), Some(&19_999));
- assert_eq!(left.sorted_view().total_weight(), left.n());
+ assert_eq!(left.sorted_view().total_weight(), left.total_weight());
let quantiles = left
.quantiles(&[0.0, 0.25, 0.5, 0.75, 1.0], SearchCriteria::Inclusive)
.unwrap()
@@ -67,7 +67,7 @@ fn merging_an_empty_lower_k_sketch_does_not_change_accuracy()
{
sketch.merge(&empty).unwrap();
- assert_eq!(sketch.n(), 10_000);
+ assert_eq!(sketch.total_weight(), 10_000);
assert_eq!(sketch.normalized_rank_error(), rank_error);
}
diff --git a/tests-integration/tests/kll_test/query.rs
b/tests-integration/tests/kll_test/query.rs
index ade962e..ac100e1 100644
--- a/tests-integration/tests/kll_test/query.rs
+++ b/tests-integration/tests/kll_test/query.rs
@@ -64,7 +64,7 @@ fn quantile_endpoints_with_large_stream_weight() {
sketch.merge(&sketch.clone()).unwrap();
}
sketch.update(1);
- assert_eq!(sketch.n(), (1 << 53) + 1);
+ assert_eq!(sketch.total_weight(), (1 << 53) + 1);
let view = sketch.sorted_view();
for criteria in [SearchCriteria::Inclusive, SearchCriteria::Exclusive] {
@@ -260,7 +260,7 @@ fn sorted_view_supports_repeated_and_batch_queries() {
.unwrap();
assert_eq!(view.len(), sketch.num_retained());
- assert_eq!(view.total_weight(), sketch.n());
+ assert_eq!(view.total_weight(), sketch.total_weight());
assert_eq!(
view.quantiles(&ranks, SearchCriteria::Inclusive)
.unwrap()
diff --git a/tests-integration/tests/req_test/accuracy.rs
b/tests-integration/tests/req_test/accuracy.rs
index 0fc361c..4cd9ad9 100644
--- a/tests-integration/tests/req_test/accuracy.rs
+++ b/tests-integration/tests/req_test/accuracy.rs
@@ -35,7 +35,7 @@ fn rank_space_error_is_bounded() -> Result<(), Error> {
sketch.update(req_f64(i as f64));
}
- assert_eq!(sketch.n(), n as u64);
+ assert_eq!(sketch.total_weight(), n as u64);
for rank in [0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99] {
let quantile = sketch
diff --git a/tests-integration/tests/req_test/bounds.rs
b/tests-integration/tests/req_test/bounds.rs
index 74dd47f..8b45e45 100644
--- a/tests-integration/tests/req_test/bounds.rs
+++ b/tests-integration/tests/req_test/bounds.rs
@@ -145,7 +145,7 @@ fn high_rank_accuracy_matches_tight_thresholds() ->
Result<(), Error> {
sketch.update(req_f64(i as f64));
}
- assert_eq!(sketch.n(), n as u64);
+ assert_eq!(sketch.total_weight(), n as u64);
for rank in [0.5, 0.9, 0.95, 0.99, 0.999] {
let true_quantile = req_f64(rank * (n - 1) as f64);
diff --git a/tests-integration/tests/req_test/core.rs
b/tests-integration/tests/req_test/core.rs
index 876c8c8..2cf7e3c 100644
--- a/tests-integration/tests/req_test/core.rs
+++ b/tests-integration/tests/req_test/core.rs
@@ -48,7 +48,7 @@ fn empty_sketch_has_default_state_and_absent_queries() {
assert_eq!(sketch.k(), 12);
assert!(sketch.is_empty());
assert!(!sketch.is_estimation_mode());
- assert_eq!(sketch.n(), 0);
+ assert_eq!(sketch.total_weight(), 0);
assert_eq!(sketch.num_retained(), 0);
assert_that!(sketch.min_item(), none());
assert_that!(sketch.max_item(), none());
@@ -79,7 +79,7 @@ fn single_value_hra_answers_exactly() {
assert!(!sketch.is_empty());
assert!(!sketch.is_estimation_mode());
- assert_eq!(sketch.n(), 1);
+ assert_eq!(sketch.total_weight(), 1);
assert_eq!(sketch.num_retained(), 1);
assert_eq!(sketch.min_item().copied(), Some(req_f32(1.0)));
assert_eq!(sketch.max_item().copied(), Some(req_f32(1.0)));
@@ -128,7 +128,7 @@ fn single_value_lra_preserves_configuration() {
assert_eq!(sketch.rank_accuracy(), RankAccuracy::LowRank);
assert!(!sketch.is_empty());
assert!(!sketch.is_estimation_mode());
- assert_eq!(sketch.n(), 1);
+ assert_eq!(sketch.total_weight(), 1);
assert_eq!(sketch.num_retained(), 1);
}
@@ -143,7 +143,7 @@ fn repeated_values_respect_search_criteria() {
}
assert!(!sketch.is_estimation_mode());
- assert_eq!(sketch.n(), 6);
+ assert_eq!(sketch.total_weight(), 6);
assert_eq!(sketch.num_retained(), 6);
assert_that!(
@@ -183,7 +183,7 @@ fn estimation_mode_compresses_and_keeps_min_max() {
assert!(!sketch.is_empty());
assert!(sketch.is_estimation_mode());
- assert_eq!(sketch.n(), n);
+ assert_eq!(sketch.total_weight(), n);
assert_that!(sketch.num_retained(), lt(n as usize));
assert_eq!(sketch.min_item().copied(), Some(req_f64(0.0)));
assert_eq!(sketch.max_item().copied(), Some(req_f64((n - 1) as f64)));
@@ -276,7 +276,7 @@ fn weight_overflow_preserves_state() {
sketch.merge(&sketch.clone()).unwrap();
sketch.update(0);
}
- assert_eq!(sketch.n(), u64::MAX);
+ assert_eq!(sketch.total_weight(), u64::MAX);
let before = sketch.serialize();
assert!(catch_unwind(AssertUnwindSafe(|| sketch.update(1))).is_err());
diff --git a/tests-integration/tests/req_test/merge.rs
b/tests-integration/tests/req_test/merge.rs
index 8a5ba52..b6ed041 100644
--- a/tests-integration/tests/req_test/merge.rs
+++ b/tests-integration/tests/req_test/merge.rs
@@ -126,7 +126,7 @@ fn many_small_merges_preserve_count_bounds_and_median() {
sketch.merge(&batch_sketch).expect("merge should succeed");
}
- assert_eq!(sketch.n(), 10_000);
+ assert_eq!(sketch.total_weight(), 10_000);
assert_eq!(sketch.min_item().copied(), Some(req_f64(0.0)));
assert_eq!(sketch.max_item().copied(), Some(req_f64(9999.0)));
@@ -152,7 +152,7 @@ fn
merge_larger_k_into_smaller_k_cascades_compaction_across_new_levels() {
target.merge(&source).expect("merge should succeed");
- assert_eq!(target.n(), 500);
+ assert_eq!(target.total_weight(), 500);
assert!(target.is_estimation_mode());
assert_that!(target.num_retained(), lt(120));
let weighted_sum: u64 = target.iter().map(|(_, weight)| weight).sum();
@@ -161,6 +161,6 @@ fn
merge_larger_k_into_smaller_k_cascades_compaction_across_new_levels() {
assert_eq!(target.max_item().copied(), Some(req_f64(499.0)));
let round_trip =
ReqSketch::<ReqF64>::deserialize(&target.serialize()).unwrap();
- assert_eq!(round_trip.n(), target.n());
+ assert_eq!(round_trip.total_weight(), target.total_weight());
assert_eq!(round_trip.num_retained(), target.num_retained());
}
diff --git a/tests-integration/tests/req_test/query.rs
b/tests-integration/tests/req_test/query.rs
index 4eae968..d104c6c 100644
--- a/tests-integration/tests/req_test/query.rs
+++ b/tests-integration/tests/req_test/query.rs
@@ -39,7 +39,7 @@ fn exact_mode_rank_quantile_pmf_and_cdf_match_reference() {
}
assert!(!sketch.is_estimation_mode());
- assert_eq!(sketch.n(), 10);
+ assert_eq!(sketch.total_weight(), 10);
assert_eq!(sketch.num_retained(), 10);
let view = sketch.sorted_view();
@@ -197,7 +197,7 @@ fn quantile_endpoints_with_large_stream_weight() {
sketch.merge(&sketch.clone()).unwrap();
}
sketch.update(1);
- assert_eq!(sketch.n(), (1 << 53) + 1);
+ assert_eq!(sketch.total_weight(), (1 << 53) + 1);
let view = sketch.sorted_view();
for criteria in [SearchCriteria::Inclusive, SearchCriteria::Exclusive] {
diff --git a/tests-integration/tests/req_test/structure.rs
b/tests-integration/tests/req_test/structure.rs
index ddf20eb..3f6d694 100644
--- a/tests-integration/tests/req_test/structure.rs
+++ b/tests-integration/tests/req_test/structure.rs
@@ -35,7 +35,7 @@ fn iterator_weights_sum_to_n_and_items_are_in_range() {
}
let total_weight: u64 = sketch.iter().map(|(_, weight)| weight).sum();
- assert_eq!(total_weight, sketch.n());
+ assert_eq!(total_weight, sketch.total_weight());
for (item, weight) in sketch.iter() {
assert_that!(weight, ge(1));
diff --git a/tests-integration/tests/serde_tests/frequencies.rs
b/tests-integration/tests/serde_tests/frequencies.rs
index d78bdca..e8508bf 100644
--- a/tests-integration/tests/serde_tests/frequencies.rs
+++ b/tests-integration/tests/serde_tests/frequencies.rs
@@ -125,7 +125,7 @@ fn check_purged_round_trip<T: FrequentItemValue>(make_item:
impl Fn(i64) -> T) {
sketch.update(make_item(i));
}
assert!(!sketch.is_empty());
- assert_eq!(sketch.num_active_items(), 0);
+ assert_eq!(sketch.num_retained(), 0);
assert_eq!(sketch.total_weight(), 193);
assert_eq!(sketch.maximum_error(), 1);
assert_eq!(sketch.upper_bound(&make_item(0)), 1);
@@ -136,7 +136,7 @@ fn check_purged_round_trip<T: FrequentItemValue>(make_item:
impl Fn(i64) -> T) {
assert_eq!(bytes[5], 0);
let mut restored = FrequentItemsSketch::<T>::deserialize(&bytes).unwrap();
assert!(!restored.is_empty());
- assert_eq!(restored.num_active_items(), 0);
+ assert_eq!(restored.num_retained(), 0);
assert_eq!(restored.total_weight(), sketch.total_weight());
assert_eq!(restored.maximum_error(), sketch.maximum_error());
assert_eq!(restored.upper_bound(&make_item(0)), 1);
@@ -163,7 +163,7 @@ fn check_purged_compatibility<T: FrequentItemValue>(dir:
&str, filename: &str) {
let bytes = fs::read(serialization_test_data(dir, filename)).unwrap();
let sketch = FrequentItemsSketch::<T>::deserialize(&bytes).unwrap();
assert!(!sketch.is_empty());
- assert_eq!(sketch.num_active_items(), 0);
+ assert_eq!(sketch.num_retained(), 0);
assert_eq!(sketch.total_weight(), 193);
assert_eq!(sketch.maximum_error(), 1);
assert_eq!(sketch.serialize(), bytes);
diff --git a/tests-integration/tests/serde_tests/kll.rs
b/tests-integration/tests/serde_tests/kll.rs
index dd023da..d9db311 100644
--- a/tests-integration/tests/serde_tests/kll.rs
+++ b/tests-integration/tests/serde_tests/kll.rs
@@ -98,7 +98,7 @@ fn test_f32_file(path: PathBuf, expected_n: usize) {
assert_eq!(sketch.k(), DEFAULT_K, "wrong k in {}", path.display());
assert_eq!(
- sketch.n() as usize,
+ sketch.total_weight() as usize,
expected_n,
"wrong n in {}",
path.display()
@@ -144,7 +144,7 @@ fn test_f64_file(path: PathBuf, expected_n: usize) {
assert_eq!(sketch.k(), DEFAULT_K, "wrong k in {}", path.display());
assert_eq!(
- sketch.n() as usize,
+ sketch.total_weight() as usize,
expected_n,
"wrong n in {}",
path.display()
@@ -190,7 +190,7 @@ fn test_i64_file(path: PathBuf, expected_n: usize) {
assert_eq!(sketch.k(), DEFAULT_K, "wrong k in {}", path.display());
assert_eq!(
- sketch.n() as usize,
+ sketch.total_weight() as usize,
expected_n,
"wrong n in {}",
path.display()
@@ -243,7 +243,7 @@ fn test_string_file(path: PathBuf, expected_n: usize) {
assert_eq!(sketch.k(), DEFAULT_K, "wrong k in {}", path.display());
assert_eq!(
- sketch.n() as usize,
+ sketch.total_weight() as usize,
expected_n,
"wrong n in {}",
path.display()
@@ -410,7 +410,7 @@ fn test_custom_kll_value_roundtrip() {
assert_eq!(decoded.serialize(), sketch.serialize());
assert_eq!(decoded.k(), sketch.k());
- assert_eq!(decoded.n(), 1_000);
+ assert_eq!(decoded.total_weight(), 1_000);
assert_eq!(decoded.num_retained(), sketch.num_retained());
assert_eq!(decoded.min_item(), sketch.min_item());
assert_eq!(decoded.max_item(), sketch.max_item());
diff --git a/tests-integration/tests/serde_tests/req.rs
b/tests-integration/tests/serde_tests/req.rs
index fa1fc1d..452477d 100644
--- a/tests-integration/tests/serde_tests/req.rs
+++ b/tests-integration/tests/serde_tests/req.rs
@@ -53,7 +53,7 @@ where
}
let bytes = a.serialize();
let b: ReqSketch<T> = ReqSketch::deserialize(&bytes).unwrap();
- assert_eq!(a.n(), b.n());
+ assert_eq!(a.total_weight(), b.total_weight());
assert_eq!(a.k(), b.k());
assert_eq!(a.rank_accuracy(), b.rank_accuracy());
assert_eq!(a.min_item(), b.min_item());
@@ -511,7 +511,12 @@ fn validate_cross_language_fixture(path: PathBuf,
expected_n: u64) {
let sketch = ReqSketch::<ReqF32>::deserialize(&bytes)
.unwrap_or_else(|e| panic!("deserialize failed for {}: {e}",
path.display()));
- assert_eq!(sketch.n(), expected_n, "n mismatch on {}", path.display());
+ assert_eq!(
+ sketch.total_weight(),
+ expected_n,
+ "n mismatch on {}",
+ path.display()
+ );
assert_eq!(sketch.k(), 12, "k mismatch on {}", path.display());
assert_eq!(sketch.rank_accuracy(), RankAccuracy::HighRank);
diff --git a/tests-integration/tests/tdigest_test/sketch.rs
b/tests-integration/tests/tdigest_test/sketch.rs
index 6ad04ab..61cf937 100644
--- a/tests-integration/tests/tdigest_test/sketch.rs
+++ b/tests-integration/tests/tdigest_test/sketch.rs
@@ -31,6 +31,7 @@ fn test_empty() {
assert!(tdigest.is_empty());
assert_eq!(tdigest.k(), 10);
assert_eq!(tdigest.total_weight(), 0);
+ assert_eq!(tdigest.num_retained(), 0);
assert_eq!(tdigest.min_value(), None);
assert_eq!(tdigest.max_value(), None);
assert_eq!(tdigest.rank(0.0).unwrap(), None);
@@ -48,6 +49,7 @@ fn test_empty() {
assert!(tdigest.is_empty());
assert_eq!(tdigest.k(), 10);
assert_eq!(tdigest.total_weight(), 0);
+ assert_eq!(tdigest.num_retained(), 0);
assert_eq!(tdigest.min_value(), None);
assert_eq!(tdigest.max_value(), None);
assert_eq!(tdigest.rank(0.0).unwrap(), None);
@@ -1035,3 +1037,33 @@ fn weight_overflow_preserves_buffer_and_extrema() {
assert!(catch_unwind(AssertUnwindSafe(|| target.merge(&sketch))).is_err());
assert_eq!(target.serialize(), before);
}
+
+#[test]
+fn test_num_retained_tracks_buffering_and_compression() {
+ let mut tdigest = TDigestMut::new(10).unwrap();
+ for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
+ tdigest.update(value);
+ }
+ assert_eq!(tdigest.num_retained(), 0);
+ assert_eq!(tdigest.clone().freeze().num_retained(), 0);
+
+ for value in 0..50 {
+ tdigest.update(f64::from(value));
+ }
+ assert_eq!(tdigest.total_weight(), 50);
+ assert_eq!(tdigest.num_retained(), 50);
+
+ let frozen = tdigest.clone().freeze();
+ assert_eq!(frozen.total_weight(), 50);
+ assert!(frozen.num_retained() > 0 && frozen.num_retained() < 50);
+
+ for value in 50..1_000 {
+ tdigest.update(f64::from(value));
+ }
+ assert_eq!(tdigest.total_weight(), 1_000);
+ assert!(tdigest.num_retained() > 0 && tdigest.num_retained() < 1_000);
+
+ let frozen = tdigest.freeze();
+ assert_eq!(frozen.total_weight(), 1_000);
+ assert!(frozen.num_retained() > 0 && frozen.num_retained() < 1_000);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]