This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch 9.1.X in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 9cf8254cd577a779ee924e8f4c4dfe989ddd77a5 Author: Lee Rhodes <[email protected]> AuthorDate: Thu Oct 29 15:09:39 2026 -0700 CPC: check stream lengths and coupon counts - Compute the required image size in long arithmetic, and bounds-check each stream against the image. - Check that 0 <= numCoupons <= 64k (k rows of 64 columns), that 0 <= numSv <= numCoupons, and that numSv <= 32 * svLengthInts, since each surprising value takes at least one bit of the compressed stream. Co-Authored-By: Claude Opus 5.5 <[email protected]> --- .../apache/datasketches/cpc/CompressedState.java | 23 +++++- .../org/apache/datasketches/cpc/PreambleUtil.java | 2 + .../datasketches/cpc/CpcCorruptImageTest.java | 86 ++++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/datasketches/cpc/CompressedState.java b/src/main/java/org/apache/datasketches/cpc/CompressedState.java index a0e88df50..cde401d7c 100644 --- a/src/main/java/org/apache/datasketches/cpc/CompressedState.java +++ b/src/main/java/org/apache/datasketches/cpc/CompressedState.java @@ -49,6 +49,7 @@ import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssert; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.Util; /** @@ -115,7 +116,7 @@ final class CompressedState { long getRequiredSerializedBytes() { final Format format = getFormat(); final int preInts = getDefinedPreInts(format); - return 4L * (preInts + csvLengthInts + cwLengthInts); + return 4L * ((long) preInts + csvLengthInts + cwLengthInts); //long arithmetic, so it cannot overflow } static CompressedState importFromSegment(final MemorySegment seg) { @@ -216,10 +217,28 @@ final class CompressedState { } } checkCapacity(seg.byteSize(), - 4L * (getPreInts(seg) + state.csvLengthInts + state.cwLengthInts)); + 4L * ((long) getPreInts(seg) + state.csvLengthInts + state.cwLengthInts)); + checkCounts(state); return state; } + /** + * Checks the coupon and surprising-value counts read from an image, before they are used to size + * the arrays that hold the uncompressed values. The matrix has k rows of 64 columns, so there are + * at most 64k coupons. The surprising values are a subset of the coupons, and each one takes at + * least one bit of the compressed stream. + * @param state the imported state + */ + private static void checkCounts(final CompressedState state) { + final long k = 1L << state.lgK; + if ((state.numCoupons < 0) || (state.numCoupons > (64L * k)) + || (state.numCsv < 0) || (state.numCsv > state.numCoupons) + || (state.numCsv > (32L * state.csvLengthInts))) { + throw new SketchesArgumentException("Possible corruption: numCoupons: " + state.numCoupons + + ", numSv: " + state.numCsv + ", svLengthInts: " + state.csvLengthInts); + } + } + void exportToSegment(final MemorySegment wseg) { final Format format = getFormat(); switch (format) { diff --git a/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java b/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java index f24c86dd1..89cb71aca 100644 --- a/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java @@ -375,6 +375,7 @@ final class PreambleUtil { static int[] getSvStream(final MemorySegment seg) { final long offset = getSvStreamOffset(seg); final int svLengthInts = getSvLengthInts(seg); + checkBounds(offset, (long) svLengthInts * Integer.BYTES, seg.byteSize()); //before allocating final int[] svStream = new int[svLengthInts]; MemorySegment.copy(seg, JAVA_INT_UNALIGNED, offset, svStream, 0, svLengthInts); return svStream; @@ -383,6 +384,7 @@ final class PreambleUtil { static int[] getWStream(final MemorySegment seg) { final long offset = getWStreamOffset(seg); final int wLength = getWLengthInts(seg); + checkBounds(offset, (long) wLength * Integer.BYTES, seg.byteSize()); //before allocating final int[] wStream = new int[wLength]; MemorySegment.copy(seg, JAVA_INT_UNALIGNED, offset, wStream, 0, wLength); return wStream; diff --git a/src/test/java/org/apache/datasketches/cpc/CpcCorruptImageTest.java b/src/test/java/org/apache/datasketches/cpc/CpcCorruptImageTest.java new file mode 100644 index 000000000..9dca351a8 --- /dev/null +++ b/src/test/java/org/apache/datasketches/cpc/CpcCorruptImageTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.cpc; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.AllocationTracker; +import org.apache.datasketches.common.ByteArrayUtil; +import org.apache.datasketches.common.SketchesArgumentException; +import org.testng.annotations.Test; + +/** + * Checks that corrupt CpcSketch images are rejected without large allocations. + */ +public class CpcCorruptImageTest { + + private static byte[] image(final int n) { + final CpcSketch sk = new CpcSketch(11); + for (int i = 0; i < n; i++) { sk.update(i); } + return sk.toByteArray(); + } + + @Test + public void checkValidImagesStillWork() { + for (final int n : new int[] {0, 5, 100, 3000, 100_000}) { + final CpcSketch sk = new CpcSketch(11); + for (int i = 0; i < n; i++) { sk.update(i); } + assertEquals(CpcSketch.heapify(MemorySegment.ofArray(sk.toByteArray())).getEstimate(), sk.getEstimate()); + } + } + + private static void checkRejected(final byte[] bytes) { + final long allocated = AllocationTracker.allocatedBytes(() -> CpcSketch.heapify(MemorySegment.ofArray(bytes))); + assertTrue(allocated < (1 << 20), "allocated " + allocated); + expectThrows(SketchesArgumentException.class, () -> CpcSketch.heapify(MemorySegment.ofArray(bytes))); + } + + private static int offset(final byte[] bytes, final PreambleUtil.HiField field) { + return (int) PreambleUtil.getHiFieldOffset(PreambleUtil.getFormat(MemorySegment.ofArray(bytes)), field); + } + + @Test + public void checkNumCouponsTooLarge() { + for (final int n : new int[] {100, 3000, 100_000}) { //sparse, hybrid and pinned or sliding formats + for (final int numCoupons : new int[] {Integer.MAX_VALUE, 1 << 25, -1}) { + final byte[] bytes = image(n); + ByteArrayUtil.putIntLE(bytes, offset(bytes, PreambleUtil.HiField.NUM_COUPONS), numCoupons); + checkRejected(bytes); + } + } + } + + @Test + public void checkStreamLengthTooLarge() { + for (final int n : new int[] {100, 3000, 100_000}) { + for (final int len : new int[] {Integer.MAX_VALUE, 1 << 28, -1}) { + final byte[] bytes = image(n); + final PreambleUtil.HiField field = (n > 3000) ? PreambleUtil.HiField.W_LENGTH_INTS + : PreambleUtil.HiField.SV_LENGTH_INTS; + ByteArrayUtil.putIntLE(bytes, offset(bytes, field), len); + checkRejected(bytes); + } + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
