This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch hll-xlang-fixtures in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 45a11552f5cc4e7eab57ef48ca66e0d86466e8fd Author: Lee Rhodes <[email protected]> AuthorDate: Thu Sep 10 11:54:46 2026 -0700 Add union and updatable HLL fixtures to the cross-language test The existing fixtures are update sketches only, serialized compact, at a single lgK. That leaves two gaps, and both produced real bugs this month that these files would not have caught: - the deferred KxQ rebuild is reachable only through a union gadget, and the generator creates no unions at all; - the flags-byte bit 32 collision (REBUILD_CURMIN_NUM_KXQ here, FULL_SIZE in C++) lives in the flags byte, which nothing inspected, and only the compact form was ever emitted. Generates six more files from a union whose inputs span all three curModes, so the coupon-update path into a gadget with a pending rebuild is exercised, in both compact and updatable form for HLL_4/6/8. One lgK is deliberate: neither defect class is lgK-dependent, and the files are generated on demand into a gitignored directory, so this adds no bytes to the repository. The check is one body reused by all four languages, matching the existing pattern, and asserts three things the September fixes established: bit 32 is reserved and must never be written by any implementation, the compact flag must match the form actually produced (HLL_6 and HLL_8 used to omit it), and the estimate must be identical across target types for the same input, since an incorrect KxQ rebuild shifts it. Verified by corrupting a fixture: setting bit 32 and clearing the compact flag each fail with the intended message, and the suite is 4/4 green otherwise. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01BKM9nrFVMhH2gmFaag2JuC --- .../hll/HllSketchCrossLanguageTest.java | 97 ++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java index 8595391a4..bd158c7be 100644 --- a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java @@ -21,6 +21,7 @@ package org.apache.datasketches.hll; import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_RUST_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; import static org.apache.datasketches.common.UtilityIO.getFileBytes; @@ -28,7 +29,11 @@ import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.apache.datasketches.hll.TgtHllType.HLL_4; import static org.apache.datasketches.hll.TgtHllType.HLL_6; import static org.apache.datasketches.hll.TgtHllType.HLL_8; +import static org.apache.datasketches.hll.PreambleUtil.COMPACT_FLAG_MASK; +import static org.apache.datasketches.hll.PreambleUtil.FLAGS_BYTE; +import static org.apache.datasketches.hll.PreambleUtil.REBUILD_CURMIN_NUM_KXQ_MASK; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import java.io.IOException; @@ -43,6 +48,9 @@ import org.testng.annotations.Test; */ public class HllSketchCrossLanguageTest { + private static final int LG_K = HllSketch.DEFAULT_LG_K; + private static final int UNION_DISTINCT = 6000; //0..999 and 2000..6999; the SET-mode input overlaps + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateBinariesForCompatibilityTesting() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; @@ -97,4 +105,93 @@ public class HllSketchCrossLanguageTest { } } + /** + * The fixtures above are update sketches only, serialized compact, at a single lgK. That + * leaves two gaps a cross-language reader can fall through, both of which produced real bugs + * in 2026-09: the deferred KxQ rebuild is reachable only through a union gadget, and the + * flags-byte bit 32 collision is only visible in the flags byte itself. These fixtures close + * both. One lgK is deliberate; the defect classes are not lgK-dependent. + */ + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateUnionBinariesForCompatibilityTesting() throws IOException { + for (final TgtHllType type: new TgtHllType[] {HLL_4, HLL_6, HLL_8}) { + final HllUnion union = mixedModeUnion(); + final String tag = tagOf(type); + putBytesToJavaPath(tag + "_union_java.sk", union.getResult(type).toCompactByteArray()); + putBytesToJavaPath(tag + "_unionupd_java.sk", union.getResult(type).toUpdatableByteArray()); + } + } + + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkUnionJava() { deserializeUnionResults(GroupLanguage.JAVA); } + + @Test(groups = {CHECK_CPP_FILES}) + public void checkUnionCpp() { deserializeUnionResults(GroupLanguage.CPP); } + + @Test(groups = {CHECK_GO_FILES}) + public void checkUnionGo() { deserializeUnionResults(GroupLanguage.GO); } + + @Test(groups = {CHECK_RUST_FILES}) + public void checkUnionRust() { deserializeUnionResults(GroupLanguage.RUST); } + + /** + * A union whose inputs span all three curModes, so the coupon-update path into a gadget with + * a pending KxQ rebuild is exercised. The second sketch stays in SET mode and overlaps the + * first entirely, so the true distinct count is UNION_DISTINCT. + */ + private static HllUnion mixedModeUnion() { + final HllUnion union = new HllUnion(LG_K); + final HllSketch big = new HllSketch(LG_K, HLL_8); + for (int i = 0; i < 1000; i++) { big.update(i); } + final HllSketch small = new HllSketch(LG_K, HLL_8); + for (int i = 0; i < 20; i++) { small.update(500 + i); } //stays in SET mode, fully overlapping + final HllSketch other = new HllSketch(LG_K, HLL_8); + for (int i = 0; i < 5000; i++) { other.update(2000 + i); } + union.update(big); + union.update(small); + union.update(other); + return union; + } + + private static String tagOf(final TgtHllType type) { + return type == HLL_4 ? "hll4" : type == HLL_6 ? "hll6" : "hll8"; + } + + /** + * One body, reused by every language. Beyond deserializing, this asserts the two flags-byte + * invariants the 2026-09 fixes established: bit 32 is reserved and must never be written by + * any implementation, and the compact flag must reflect the form actually produced. + */ + private static void deserializeUnionResults(final GroupLanguage lang) { + final String[] tags = {"hll4", "hll6", "hll8"}; + final String[] forms = {"_union", "_unionupd"}; + double firstEstimate = Double.NaN; + for (final String form: forms) { + for (final String tag: tags) { + final String fileName = tag + form + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + final int flags = bytes[FLAGS_BYTE] & 0xFF; + + //bit 32 collided across implementations: REBUILD_CURMIN_NUM_KXQ here, FULL_SIZE in C++. + //Java clears it before serializing and C++ no longer writes it, so no image carries it. + assertFalse((flags & REBUILD_CURMIN_NUM_KXQ_MASK) > 0, + fileName + " has reserved flags bit 32 set"); + + //HLL_6 and HLL_8 used to omit this on toCompactByteArray() + assertEquals((flags & COMPACT_FLAG_MASK) > 0, form.equals("_union"), + fileName + " compact flag does not match its serialized form"); + + final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sketch.getLgConfigK(), LG_K); + assertTrue(!sketch.isEmpty()); + assertEquals(sketch.getEstimate(), UNION_DISTINCT, UNION_DISTINCT * 0.02); + + //an incorrect KxQ rebuild shifts the estimate, and it must not depend on target type + if (Double.isNaN(firstEstimate)) { firstEstimate = sketch.getEstimate(); } + else { assertEquals(sketch.getEstimate(), firstEstimate, 1e-9, fileName); } + } + } + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
