Repository: cassandra Updated Branches: refs/heads/trunk 716d08f90 -> d6cc59432
New Bloomfilter format without changing the byte ordering patch by Jay Zhuang; reviewed by jasobrown for CASSANDRA-9067 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d6cc5943 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d6cc5943 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d6cc5943 Branch: refs/heads/trunk Commit: d6cc5943250a7c19adb42ef86dc9a186d4e52166 Parents: 716d08f Author: Jay Zhuang <jay.zhu...@yahoo.com> Authored: Sun Jan 14 20:42:57 2018 -0800 Committer: Jason Brown <jasedbr...@gmail.com> Committed: Fri Jan 19 06:40:50 2018 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../io/sstable/format/SSTableReader.java | 8 +- .../cassandra/io/sstable/format/Version.java | 7 ++ .../io/sstable/format/big/BigFormat.java | 14 ++- .../io/sstable/format/big/BigTableWriter.java | 2 +- .../cassandra/utils/BloomFilterSerializer.java | 8 +- .../apache/cassandra/utils/FilterFactory.java | 14 --- .../org/apache/cassandra/utils/obs/IBitSet.java | 4 +- .../cassandra/utils/obs/OffHeapBitSet.java | 60 +++++++---- .../legacy_na_clust/na-1-big-CompressionInfo.db | Bin 87 -> 87 bytes .../legacy_na_clust/na-1-big-Data.db | Bin 5215 -> 5259 bytes .../legacy_na_clust/na-1-big-Digest.crc32 | 2 +- .../legacy_na_clust/na-1-big-Filter.db | Bin 24 -> 24 bytes .../legacy_na_clust/na-1-big-Index.db | Bin 157553 -> 157553 bytes .../legacy_na_clust/na-1-big-Statistics.db | Bin 7095 -> 7095 bytes .../legacy_na_clust/na-1-big-TOC.txt | 6 +- .../na-1-big-CompressionInfo.db | Bin 79 -> 79 bytes .../legacy_na_clust_counter/na-1-big-Data.db | Bin 5781 -> 5888 bytes .../na-1-big-Digest.crc32 | 2 +- .../legacy_na_clust_counter/na-1-big-Filter.db | Bin 24 -> 24 bytes .../legacy_na_clust_counter/na-1-big-Index.db | Bin 157553 -> 157553 bytes .../na-1-big-Statistics.db | Bin 7104 -> 7104 bytes .../legacy_na_clust_counter/na-1-big-TOC.txt | 6 +- .../legacy_na_simple/na-1-big-Data.db | Bin 89 -> 89 bytes .../legacy_na_simple/na-1-big-Digest.crc32 | 2 +- .../legacy_na_simple/na-1-big-Filter.db | Bin 24 -> 24 bytes .../legacy_na_simple/na-1-big-Statistics.db | Bin 4648 -> 4648 bytes .../legacy_na_simple/na-1-big-TOC.txt | 6 +- .../legacy_na_simple_counter/na-1-big-Data.db | Bin 146 -> 140 bytes .../na-1-big-Digest.crc32 | 2 +- .../legacy_na_simple_counter/na-1-big-Filter.db | Bin 24 -> 24 bytes .../na-1-big-Statistics.db | Bin 4657 -> 4657 bytes .../legacy_na_simple_counter/na-1-big-TOC.txt | 6 +- .../serialization/4.0/utils.BloomFilter1000.bin | Bin 0 -> 2500016 bytes .../microbench/BloomFilterSerializerBench.java | 99 +++++++++++++++++++ .../cassandra/io/sstable/LegacySSTableTest.java | 3 +- .../apache/cassandra/utils/BloomFilterTest.java | 24 +++-- .../cassandra/utils/SerializationsTest.java | 49 +++++++-- .../cassandra/utils/obs/OffHeapBitSetTest.java | 21 ++-- 39 files changed, 257 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 65a7a40..fd0acb9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067) * Remove unused on-heap BloomFilter implementation (CASSANDRA-14152) * Delete temp test files on exit (CASSANDRA-14153) * Make PartitionUpdate and Mutation immutable (CASSANDRA-13867) http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index 6722e1b..757b9a8 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -75,6 +75,7 @@ import org.apache.cassandra.utils.*; import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.utils.concurrent.Ref; import org.apache.cassandra.utils.concurrent.SelfRefCounted; +import org.apache.cassandra.utils.BloomFilterSerializer; import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR; @@ -728,7 +729,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS { // bf is enabled and fp chance matches the currently configured value. load(false, true); - loadBloomFilter(); + loadBloomFilter(descriptor.version.hasOldBfFormat()); } } @@ -736,12 +737,13 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS * Load bloom filter from Filter.db file. * * @throws IOException + * @param oldBfFormat */ - private void loadBloomFilter() throws IOException + private void loadBloomFilter(boolean oldBfFormat) throws IOException { try (DataInputStream stream = new DataInputStream(new BufferedInputStream(Files.newInputStream(Paths.get(descriptor.filenameFor(Component.FILTER)))))) { - bf = FilterFactory.deserialize(stream); + bf = BloomFilterSerializer.deserialize(stream, oldBfFormat); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/io/sstable/format/Version.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index e8721a6..1d965ce 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -57,6 +57,13 @@ public abstract class Version public abstract boolean hasMetadataChecksum(); + /** + * The old bloomfilter format serializes the data as BIG_ENDIAN long's, the new one uses the + * same format as in memory (serializes as bytes). + * @return True if the bloomfilter file is old serialization format + */ + public abstract boolean hasOldBfFormat(); + public String getVersion() { return version; http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index 5f6dd53..db73b4f 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -120,7 +120,7 @@ public class BigFormat implements SSTableFormat // mb (3.0.7, 3.7): commit log lower bound included // mc (3.0.8, 3.9): commit log intervals included - // na (4.0.0): uncompressed chunks, pending repair session, checksummed sstable metadata file + // na (4.0.0): uncompressed chunks, pending repair session, checksummed sstable metadata file, new Bloomfilter format // // NOTE: when adding a new version, please add that to LegacySSTableTest, too. @@ -131,6 +131,11 @@ public class BigFormat implements SSTableFormat public final boolean hasMaxCompressedLength; private final boolean hasPendingRepair; private final boolean hasMetadataChecksum; + /** + * CASSANDRA-9067: 4.0 bloom filter representation changed (two longs just swapped) + * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution. + */ + private final boolean hasOldBfFormat; BigVersion(String version) { @@ -144,6 +149,7 @@ public class BigFormat implements SSTableFormat hasMaxCompressedLength = version.compareTo("na") >= 0; hasPendingRepair = version.compareTo("na") >= 0; hasMetadataChecksum = version.compareTo("na") >= 0; + hasOldBfFormat = version.compareTo("na") < 0; } @Override @@ -197,5 +203,11 @@ public class BigFormat implements SSTableFormat { return hasMaxCompressedLength; } + + @Override + public boolean hasOldBfFormat() + { + return hasOldBfFormat; + } } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index 04c7bbf..b5488ed 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -484,7 +484,7 @@ public class BigTableWriter extends SSTableWriter DataOutputStreamPlus stream = new BufferedDataOutputStreamPlus(fos)) { // bloom filter - FilterFactory.serialize(bf, stream); + BloomFilterSerializer.serialize((BloomFilter) bf, stream); stream.flush(); SyncUtil.sync(fos); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java index 9e9d15a..d3c08b5 100644 --- a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java +++ b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java @@ -17,7 +17,7 @@ */ package org.apache.cassandra.utils; -import java.io.DataInput; +import java.io.DataInputStream; import java.io.IOException; import org.apache.cassandra.db.TypeSizes; @@ -25,7 +25,7 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.obs.IBitSet; import org.apache.cassandra.utils.obs.OffHeapBitSet; -final class BloomFilterSerializer +public final class BloomFilterSerializer { private BloomFilterSerializer() { @@ -38,10 +38,10 @@ final class BloomFilterSerializer } @SuppressWarnings("resource") - public static BloomFilter deserialize(DataInput in) throws IOException + public static BloomFilter deserialize(DataInputStream in, boolean oldBfFormat) throws IOException { int hashes = in.readInt(); - IBitSet bs = OffHeapBitSet.deserialize(in); + IBitSet bs = OffHeapBitSet.deserialize(in, oldBfFormat); return new BloomFilter(hashes, bs); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/utils/FilterFactory.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/FilterFactory.java b/src/java/org/apache/cassandra/utils/FilterFactory.java index 9479452..4cf0cbf 100644 --- a/src/java/org/apache/cassandra/utils/FilterFactory.java +++ b/src/java/org/apache/cassandra/utils/FilterFactory.java @@ -17,13 +17,9 @@ */ package org.apache.cassandra.utils; -import java.io.DataInput; -import java.io.IOException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.obs.IBitSet; import org.apache.cassandra.utils.obs.OffHeapBitSet; @@ -34,16 +30,6 @@ public class FilterFactory private static final Logger logger = LoggerFactory.getLogger(FilterFactory.class); private static final long BITSET_EXCESS = 20; - public static void serialize(IFilter bf, DataOutputPlus output) throws IOException - { - BloomFilterSerializer.serialize((BloomFilter) bf, output); - } - - public static IFilter deserialize(DataInput input) throws IOException - { - return BloomFilterSerializer.deserialize(input); - } - /** * @return A BloomFilter with the lowest practical false positive * probability for the given number of elements. http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/utils/obs/IBitSet.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/obs/IBitSet.java b/src/java/org/apache/cassandra/utils/obs/IBitSet.java index 15ff361..b262cf5 100644 --- a/src/java/org/apache/cassandra/utils/obs/IBitSet.java +++ b/src/java/org/apache/cassandra/utils/obs/IBitSet.java @@ -18,9 +18,9 @@ package org.apache.cassandra.utils.obs; import java.io.Closeable; -import java.io.DataOutput; import java.io.IOException; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.concurrent.Ref; public interface IBitSet extends Closeable @@ -44,7 +44,7 @@ public interface IBitSet extends Closeable */ public void clear(long index); - public void serialize(DataOutput out) throws IOException; + public void serialize(DataOutputPlus out) throws IOException; public long serializedSize(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java index d2c15ca..486ec38 100644 --- a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java +++ b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java @@ -18,11 +18,17 @@ package org.apache.cassandra.utils.obs; import java.io.DataInput; +import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; +import com.google.common.annotations.VisibleForTesting; + import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.Memory; +import org.apache.cassandra.io.util.MemoryOutputStream; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Ref; /** @@ -109,19 +115,26 @@ public class OffHeapBitSet implements IBitSet bytes.setMemory(0, bytes.size(), (byte) 0); } - public void serialize(DataOutput out) throws IOException + public void serialize(DataOutputPlus out) throws IOException { out.writeInt((int) (bytes.size() / 8)); - for (long i = 0; i < bytes.size();) + out.write(bytes, 0, bytes.size()); + } + + @VisibleForTesting + public void serializeOldBfFormat(DataOutputPlus out) throws IOException + { + out.writeInt((int) (bytes.size() / 8)); + for (long i = 0; i < bytes.size(); ) { long value = ((bytes.getByte(i++) & 0xff) << 0) - + ((bytes.getByte(i++) & 0xff) << 8) - + ((bytes.getByte(i++) & 0xff) << 16) - + ((long) (bytes.getByte(i++) & 0xff) << 24) - + ((long) (bytes.getByte(i++) & 0xff) << 32) - + ((long) (bytes.getByte(i++) & 0xff) << 40) - + ((long) (bytes.getByte(i++) & 0xff) << 48) - + ((long) bytes.getByte(i++) << 56); + + ((bytes.getByte(i++) & 0xff) << 8) + + ((bytes.getByte(i++) & 0xff) << 16) + + ((long) (bytes.getByte(i++) & 0xff) << 24) + + ((long) (bytes.getByte(i++) & 0xff) << 32) + + ((long) (bytes.getByte(i++) & 0xff) << 40) + + ((long) (bytes.getByte(i++) & 0xff) << 48) + + ((long) bytes.getByte(i++) << 56); out.writeLong(value); } } @@ -132,21 +145,28 @@ public class OffHeapBitSet implements IBitSet } @SuppressWarnings("resource") - public static OffHeapBitSet deserialize(DataInput in) throws IOException + public static OffHeapBitSet deserialize(DataInputStream in, boolean oldBfFormat) throws IOException { long byteCount = in.readInt() * 8L; Memory memory = Memory.allocate(byteCount); - for (long i = 0; i < byteCount;) + if (oldBfFormat) + { + for (long i = 0; i < byteCount; ) + { + long v = in.readLong(); + memory.setByte(i++, (byte) (v >>> 0)); + memory.setByte(i++, (byte) (v >>> 8)); + memory.setByte(i++, (byte) (v >>> 16)); + memory.setByte(i++, (byte) (v >>> 24)); + memory.setByte(i++, (byte) (v >>> 32)); + memory.setByte(i++, (byte) (v >>> 40)); + memory.setByte(i++, (byte) (v >>> 48)); + memory.setByte(i++, (byte) (v >>> 56)); + } + } + else { - long v = in.readLong(); - memory.setByte(i++, (byte) (v >>> 0)); - memory.setByte(i++, (byte) (v >>> 8)); - memory.setByte(i++, (byte) (v >>> 16)); - memory.setByte(i++, (byte) (v >>> 24)); - memory.setByte(i++, (byte) (v >>> 32)); - memory.setByte(i++, (byte) (v >>> 40)); - memory.setByte(i++, (byte) (v >>> 48)); - memory.setByte(i++, (byte) (v >>> 56)); + FBUtilities.copy(in, new MemoryOutputStream(memory), byteCount); } return new OffHeapBitSet(memory); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db index 935d863..ceaa5a3 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db index 9982578..6968720 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 index 1e6cbfc..f1c192b 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 @@ -1 +1 @@ -4285275084 \ No newline at end of file +4004129384 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db index 2e1d5d2..8868e5c 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db index 25b063c..af16195 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db index 59969c2..970e385 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt index 734a80d..bb800f8 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt @@ -1,8 +1,8 @@ -Index.db -TOC.txt +Digest.crc32 Filter.db CompressionInfo.db +Index.db Summary.db Data.db +TOC.txt Statistics.db -Digest.crc32 http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db index d6a1ff8..f5ad4d0 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db index 03babd4..7217716 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 index 7bd9e39..4f1391a 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 @@ -1 +1 @@ -1093063834 \ No newline at end of file +4072239034 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db index 2e1d5d2..8868e5c 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db index 5ad5400..6dd3da6 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db index 9347a2c..3a0e63f 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt index 734a80d..bb800f8 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt @@ -1,8 +1,8 @@ -Index.db -TOC.txt +Digest.crc32 Filter.db CompressionInfo.db +Index.db Summary.db Data.db +TOC.txt Statistics.db -Digest.crc32 http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db index 414ff07..c665dfb 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 index 1dfec4f..c6c24a7 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 @@ -1 +1 @@ -2354437953 \ No newline at end of file +3772296151 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db index 2e1d5d2..8868e5c 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db index 33f0516..6741430 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt index 734a80d..bb800f8 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt @@ -1,8 +1,8 @@ -Index.db -TOC.txt +Digest.crc32 Filter.db CompressionInfo.db +Index.db Summary.db Data.db +TOC.txt Statistics.db -Digest.crc32 http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db index ed0dadb..d9fe576 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 index 1480f5d..de7baed 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 @@ -1 +1 @@ -2023874595 \ No newline at end of file +4035692752 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db index 2e1d5d2..8868e5c 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db index 124dacd..e9556d1 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt index 734a80d..bb800f8 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt @@ -1,8 +1,8 @@ -Index.db -TOC.txt +Digest.crc32 Filter.db CompressionInfo.db +Index.db Summary.db Data.db +TOC.txt Statistics.db -Digest.crc32 http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/serialization/4.0/utils.BloomFilter1000.bin ---------------------------------------------------------------------- diff --git a/test/data/serialization/4.0/utils.BloomFilter1000.bin b/test/data/serialization/4.0/utils.BloomFilter1000.bin new file mode 100644 index 0000000..b1bfe99 Binary files /dev/null and b/test/data/serialization/4.0/utils.BloomFilter1000.bin differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java ---------------------------------------------------------------------- diff --git a/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java new file mode 100644 index 0000000..9222811 --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus; +import org.apache.cassandra.io.util.DataOutputStreamPlus; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.utils.BloomFilter; +import org.apache.cassandra.utils.BloomFilterSerializer; +import org.apache.cassandra.utils.FilterFactory; +import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.SerializationsTest; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 2, time = 4, timeUnit = TimeUnit.SECONDS) +@Fork(value = 2) +@State(Scope.Benchmark) +public class BloomFilterSerializerBench +{ + + @Param({"1", "10", "100", "1024"}) + private long numElemsInK; + + @Param({"true", "false"}) + public boolean oldBfFormat; + + static final IFilter.FilterKey wrap(ByteBuffer buf) + { + return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(0L), buf); + } + + private ByteBuffer testVal = ByteBuffer.wrap(new byte[] { 0, 1}); + + @Benchmark + public void serializationTest() throws IOException + { + File file = FileUtils.createTempFile("bloomFilterTest-", ".dat"); + try + { + BloomFilter filter = (BloomFilter) FilterFactory.getFilter(numElemsInK * 1024, 0.01d); + filter.add(wrap(testVal)); + DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(file)); + if (oldBfFormat) + SerializationsTest.serializeOldBfFormat(filter, out); + else + BloomFilterSerializer.serialize(filter, out); + out.close(); + filter.close(); + + DataInputStream in = new DataInputStream(new FileInputStream(file)); + BloomFilter filter2 = BloomFilterSerializer.deserialize(in, oldBfFormat); + FileUtils.closeQuietly(in); + filter2.close(); + } + finally + { + file.delete(); + } + } + +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 4892184..6e7d173 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -94,7 +94,7 @@ public class LegacySSTableTest { String scp = System.getProperty(LEGACY_SSTABLE_PROP); Assert.assertNotNull("System property " + LEGACY_SSTABLE_ROOT + " not set", scp); - + LEGACY_SSTABLE_ROOT = new File(scp).getAbsoluteFile(); Assert.assertTrue("System property " + LEGACY_SSTABLE_ROOT + " does not specify a directory", LEGACY_SSTABLE_ROOT.isDirectory()); @@ -270,7 +270,6 @@ public class LegacySSTableTest logger.debug("for pk={} ck={}", pk, ck); String pkValue = Integer.toString(pk); - UntypedResultSet rs; if (ck == 0) { readSimpleTable(legacyVersion, pkValue); http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/unit/org/apache/cassandra/utils/BloomFilterTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java index 581248c..1c3afff 100644 --- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java +++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java @@ -43,19 +43,23 @@ public class BloomFilterTest { public IFilter bfInvHashes; - public BloomFilterTest() - { - } - public static IFilter testSerialize(IFilter f) throws IOException + public static IFilter testSerialize(IFilter f, boolean oldBfFormat) throws IOException { f.add(FilterTestHelper.bytes("a")); DataOutputBuffer out = new DataOutputBuffer(); - FilterFactory.serialize(f, out); + if (oldBfFormat) + { + SerializationsTest.serializeOldBfFormat((BloomFilter) f, out); + } + else + { + BloomFilterSerializer.serialize((BloomFilter) f, out); + } ByteArrayInputStream in = new ByteArrayInputStream(out.getData(), 0, out.getLength()); - IFilter f2 = FilterFactory.deserialize(new DataInputStream(in)); + IFilter f2 = BloomFilterSerializer.deserialize(new DataInputStream(in), oldBfFormat); assert f2.isPresent(FilterTestHelper.bytes("a")); assert !f2.isPresent(FilterTestHelper.bytes("b")); @@ -132,7 +136,8 @@ public class BloomFilterTest @Test public void testSerialize() throws IOException { - BloomFilterTest.testSerialize(bfInvHashes).close(); + BloomFilterTest.testSerialize(bfInvHashes, true).close(); + BloomFilterTest.testSerialize(bfInvHashes, false).close(); } @Test @@ -206,13 +211,12 @@ public class BloomFilterTest BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long) Integer.MAX_VALUE / 8) + 1, 0.01d); filter.add(FilterTestHelper.wrap(test)); DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(file)); - FilterFactory.serialize(filter, out); - filter.bitset.serialize(out); + BloomFilterSerializer.serialize(filter, out); out.close(); filter.close(); DataInputStream in = new DataInputStream(new FileInputStream(file)); - BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in); + BloomFilter filter2 = BloomFilterSerializer.deserialize(in, false); Assert.assertTrue(filter2.isPresent(FilterTestHelper.wrap(test))); FileUtils.closeQuietly(in); filter2.close(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/unit/org/apache/cassandra/utils/SerializationsTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/utils/SerializationsTest.java b/test/unit/org/apache/cassandra/utils/SerializationsTest.java index 8da4a92..f260428 100644 --- a/test/unit/org/apache/cassandra/utils/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/utils/SerializationsTest.java @@ -31,30 +31,42 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.DataOutputStreamPlus; -import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.utils.obs.OffHeapBitSet; import java.io.File; import java.io.FileInputStream; public class SerializationsTest extends AbstractSerializationsTester { + // Helper function to serialize old Bloomfilter format, should be removed once the old format is not supported + public static void serializeOldBfFormat(BloomFilter bf, DataOutputPlus out) throws IOException + { + out.writeInt(bf.hashCount); + Assert.assertTrue(bf.bitset instanceof OffHeapBitSet); + ((OffHeapBitSet) bf.bitset).serializeOldBfFormat(out); + } + @BeforeClass public static void initDD() { DatabaseDescriptor.daemonInitialization(); } - private static void testBloomFilterWrite1000() throws IOException + private static void testBloomFilterWrite1000(boolean oldBfFormat) throws IOException { try (IFilter bf = FilterFactory.getFilter(1000000, 0.0001)) { for (int i = 0; i < 1000; i++) bf.add(Util.dk(Int32Type.instance.decompose(i))); - try (DataOutputStreamPlus out = getOutput("3.0", "utils.BloomFilter1000.bin")) + try (DataOutputStreamPlus out = getOutput(oldBfFormat ? "3.0" : "4.0", "utils.BloomFilter1000.bin")) { - FilterFactory.serialize(bf, out); + if (oldBfFormat) + serializeOldBfFormat((BloomFilter) bf, out); + else + BloomFilterSerializer.serialize((BloomFilter) bf, out); } } } @@ -63,10 +75,29 @@ public class SerializationsTest extends AbstractSerializationsTester public void testBloomFilterRead1000() throws IOException { if (EXECUTE_WRITES) - testBloomFilterWrite1000(); + { + testBloomFilterWrite1000(false); + testBloomFilterWrite1000(true); + } + + try (DataInputStream in = getInput("4.0", "utils.BloomFilter1000.bin"); + IFilter filter = BloomFilterSerializer.deserialize(in, false)) + { + boolean present; + for (int i = 0 ; i < 1000 ; i++) + { + present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i))); + Assert.assertTrue(present); + } + for (int i = 1000 ; i < 2000 ; i++) + { + present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i))); + Assert.assertFalse(present); + } + } try (DataInputStream in = getInput("3.0", "utils.BloomFilter1000.bin"); - IFilter filter = FilterFactory.deserialize(in)) + IFilter filter = BloomFilterSerializer.deserialize(in, true)) { boolean present; for (int i = 0 ; i < 1000 ; i++) @@ -85,15 +116,15 @@ public class SerializationsTest extends AbstractSerializationsTester @Test public void testBloomFilterTable() throws Exception { - testBloomFilterTable("test/data/bloom-filter/la/foo/la-1-big-Filter.db"); + testBloomFilterTable("test/data/bloom-filter/la/foo/la-1-big-Filter.db", true); } - private static void testBloomFilterTable(String file) throws Exception + private static void testBloomFilterTable(String file, boolean oldBfFormat) throws Exception { Murmur3Partitioner partitioner = new Murmur3Partitioner(); try (DataInputStream in = new DataInputStream(new FileInputStream(new File(file))); - IFilter filter = FilterFactory.deserialize(in)) + IFilter filter = BloomFilterSerializer.deserialize(in, oldBfFormat)) { for (int i = 1; i <= 10; i++) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java b/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java index f0325da..49b4c94 100644 --- a/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java +++ b/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java @@ -25,11 +25,9 @@ import java.util.List; import java.util.Random; import com.google.common.collect.Lists; +import org.apache.cassandra.io.util.DataOutputBuffer; import org.junit.Assert; import org.junit.Test; -import org.junit.rules.ExpectedException; - -import org.apache.cassandra.io.util.DataOutputBuffer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -46,8 +44,7 @@ public class OffHeapBitSetTest Assert.assertEquals(bs.get(i), newbs.get(i)); } - @Test - public void testOffHeapSerialization() throws IOException + private void testOffHeapSerialization(boolean oldBfFormat) throws IOException { try (OffHeapBitSet bs = new OffHeapBitSet(100000)) { @@ -56,10 +53,13 @@ public class OffHeapBitSetTest bs.set(i); DataOutputBuffer out = new DataOutputBuffer(); - bs.serialize(out); + if (oldBfFormat) + bs.serializeOldBfFormat(out); + else + bs.serialize(out); DataInputStream in = new DataInputStream(new ByteArrayInputStream(out.getData())); - try (OffHeapBitSet newbs = OffHeapBitSet.deserialize(in)) + try (OffHeapBitSet newbs = OffHeapBitSet.deserialize(in, oldBfFormat)) { compare(bs, newbs); } @@ -67,6 +67,13 @@ public class OffHeapBitSetTest } @Test + public void testSerialization() throws IOException + { + testOffHeapSerialization(true); + testOffHeapSerialization(false); + } + + @Test public void testBitSetGetClear() { int size = Integer.MAX_VALUE / 4000; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
