This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch 9.1.X in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 4e024585d37c0eebb3653c55529f64ac42688364 Author: Lee Rhodes <[email protected]> AuthorDate: Thu Oct 29 14:56:25 2026 -0700 Quantiles items: check n against the image size No serialized item takes less than one bit (booleans are packed 8 per byte), so QuantilesItemsSketch.heapify() now requires that n is non-negative and that the number of retained items is no more than 8 times the remaining image bytes. Co-Authored-By: Claude Opus 5.5 <[email protected]> --- .../quantiles/QuantilesItemsSketch.java | 9 ++- .../quantiles/QuantilesItemsCorruptImageTest.java | 72 ++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java b/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java index 17c94940d..f007ac392 100644 --- a/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java @@ -214,9 +214,16 @@ public final class QuantilesItemsSketch<T> implements QuantilesGenericAPI<T> { //Not empty, must have valid preamble + min, max final long n = extractN(srcSeg); - //can't check MemorySegment capacity here, not enough information + //can't check the exact MemorySegment capacity here, not enough information final int extra = 2; //for min, max final int numSegItems = computeRetainedItems(k, n) + extra; + //No serialized item takes less than one bit (booleans are packed 8 per byte), + //so the image bounds the number of items before the buffer is allocated. + final long itemsBytes = segCapBytes - ((long) preambleLongs * Long.BYTES); + if ((n < 0) || ((long) numSegItems > (8L * itemsBytes))) { + throw new SketchesArgumentException("Possible corruption: n = " + n + " requires " + numSegItems + + " items, but only " + itemsBytes + " bytes remain"); + } //set class members sk.n_ = n; diff --git a/src/test/java/org/apache/datasketches/quantiles/QuantilesItemsCorruptImageTest.java b/src/test/java/org/apache/datasketches/quantiles/QuantilesItemsCorruptImageTest.java new file mode 100644 index 000000000..9335a5115 --- /dev/null +++ b/src/test/java/org/apache/datasketches/quantiles/QuantilesItemsCorruptImageTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantiles; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import java.lang.foreign.MemorySegment; +import java.util.Comparator; + +import org.apache.datasketches.common.AllocationTracker; +import org.apache.datasketches.common.ArrayOfLongsSerDe; +import org.apache.datasketches.common.ByteArrayUtil; +import org.apache.datasketches.common.SketchesArgumentException; +import org.testng.annotations.Test; + +/** + * Checks that a corrupt QuantilesItemsSketch image is rejected. + */ +public class QuantilesItemsCorruptImageTest { + private static final int N_LONG = 8; + private final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe(); + + private byte[] image(final int n) { + final QuantilesItemsSketch<Long> sk = QuantilesItemsSketch.getInstance(Long.class, 32768, Comparator.naturalOrder()); + for (long i = 0; i < n; i++) { sk.update(i); } + return sk.toByteArray(serDe); + } + + @Test + public void checkLargeN() { + final byte[] bytes = image(1); + ByteArrayUtil.putLongLE(bytes, N_LONG, 1L << 60); + final MemorySegment seg = MemorySegment.ofArray(bytes); + final long allocated = AllocationTracker.allocatedBytes( + () -> QuantilesItemsSketch.heapify(Long.class, seg, Comparator.naturalOrder(), serDe)); + assertTrue(allocated < (1 << 20), "allocated " + allocated); + expectThrows(SketchesArgumentException.class, + () -> QuantilesItemsSketch.heapify(Long.class, seg, Comparator.naturalOrder(), serDe)); + final byte[] negative = image(1); + ByteArrayUtil.putLongLE(negative, N_LONG, -1L); + expectThrows(SketchesArgumentException.class, + () -> QuantilesItemsSketch.heapify(Long.class, MemorySegment.ofArray(negative), Comparator.naturalOrder(), serDe)); + } + + @Test + public void checkValidImagesStillWork() { + for (final int n : new int[] {1, 100, 100_000}) { + final QuantilesItemsSketch<Long> sk = + QuantilesItemsSketch.heapify(Long.class, MemorySegment.ofArray(image(n)), Comparator.naturalOrder(), serDe); + assertEquals(sk.getN(), n); + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
