This is an automated email from the ASF dual-hosted git repository. edimitrova pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push: new 600f4d9a69 Upgrade Jamm version to 0.4.0 This upgrade also fixes issues with PhantomReferences and the test problems from CASSANDRA-17884 anad CASSANDRA-16304 600f4d9a69 is described below commit 600f4d9a690dbd887d5e6298fe67e6bba982033d Author: Benjamin Lerer <b.le...@gmail.com> AuthorDate: Thu Jul 13 19:08:11 2023 +0200 Upgrade Jamm version to 0.4.0 This upgrade also fixes issues with PhantomReferences and the test problems from CASSANDRA-17884 anad CASSANDRA-16304 patch by Benjamin Lerer; reviewed by Ekaterina Dimitrova for CASSANDRA-18239 --- CHANGES.txt | 1 + bin/cassandra.in.sh | 2 +- build.xml | 2 +- conf/cassandra-env.sh | 2 +- redhat/cassandra.in.sh | 2 +- .../org/apache/cassandra/audit/BinAuditLogger.java | 7 +- .../apache/cassandra/cql3/ColumnIdentifier.java | 2 +- .../org/apache/cassandra/db/BufferClustering.java | 2 +- .../org/apache/cassandra/db/rows/BufferCell.java | 2 +- .../org/apache/cassandra/db/rows/CellPath.java | 2 +- .../apache/cassandra/db/tries/InMemoryTrie.java | 7 +- .../org/apache/cassandra/fql/FullQueryLogger.java | 93 ++++++++---- .../org/apache/cassandra/utils/ObjectSizes.java | 135 +++++++++-------- .../test/microbench/tries/ComparisonReadBench.java | 8 +- .../unit/org/apache/cassandra/db/CellSpecTest.java | 2 +- .../cassandra/db/memtable/MemtableSizeTest.java | 22 +-- .../apache/cassandra/utils/ObjectSizesTest.java | 165 ++++++++++----------- 17 files changed, 252 insertions(+), 204 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1b02aa51bd..f6632365b8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0 + * Upgrade Jamm version to 0.4.0 (CASSANDRA-17884, CASSANDRA-16304, CASSANDRA-18329) * Remove legacy 3.0/3.11 buffer pool metrics (CASSANDRA-18313) * Add AzureSnitch (CASSANDRA-18646) * Implementation of the Unified Compaction Strategy as described in CEP-26 (CASSANDRA-18397) diff --git a/bin/cassandra.in.sh b/bin/cassandra.in.sh index dfbb95955e..41f554d125 100644 --- a/bin/cassandra.in.sh +++ b/bin/cassandra.in.sh @@ -79,7 +79,7 @@ if [ -f "$CASSANDRA_HOME"/lib/jsr223/scala/scala-compiler.jar ] ; then fi # set JVM javaagent opts to avoid warnings/errors -JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar" +JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.4.0.jar" # Added sigar-bin to the java.library.path CASSANDRA-7838 JAVA_OPTS="$JAVA_OPTS:-Djava.library.path=$CASSANDRA_HOME/lib/sigar-bin" diff --git a/build.xml b/build.xml index 5c4206d1fc..56116adba1 100644 --- a/build.xml +++ b/build.xml @@ -142,7 +142,7 @@ <property name="jacoco.finalexecfile" value="${jacoco.export.dir}/jacoco.exec" /> <property name="jflex.version" value="1.8.2"/> - <property name="jamm.version" value="0.3.2"/> + <property name="jamm.version" value="0.4.0"/> <property name="ecj.version" value="4.6.1"/> <!-- When updating ASM, please, do consider whether you might need to update also FBUtilities#ASM_BYTECODE_VERSION and the simulator InterceptClasses#BYTECODE_VERSION, in particular if we are looking to provide Cassandra support diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh index 25ba5052d3..9c4d3e4de9 100644 --- a/conf/cassandra-env.sh +++ b/conf/cassandra-env.sh @@ -191,7 +191,7 @@ fi JVM_OPTS="$JVM_OPTS -XX:CompileCommandFile=$CASSANDRA_CONF/hotspot_compiler" # add the jamm javaagent -JVM_OPTS="$JVM_OPTS -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar" +JVM_OPTS="$JVM_OPTS -javaagent:$CASSANDRA_HOME/lib/jamm-0.4.0.jar" # set jvm HeapDumpPath with CASSANDRA_HEAPDUMP_DIR if [ "x$CASSANDRA_HEAPDUMP_DIR" != "x" ]; then diff --git a/redhat/cassandra.in.sh b/redhat/cassandra.in.sh index c06cf72a3f..fed5d4384e 100644 --- a/redhat/cassandra.in.sh +++ b/redhat/cassandra.in.sh @@ -40,7 +40,7 @@ CLASSPATH="$CLASSPATH:$EXTRA_CLASSPATH" # set JVM javaagent opts to avoid warnings/errors -JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar" +JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.4.0.jar" # diff --git a/src/java/org/apache/cassandra/audit/BinAuditLogger.java b/src/java/org/apache/cassandra/audit/BinAuditLogger.java index 607d9fee0b..e624446c74 100644 --- a/src/java/org/apache/cassandra/audit/BinAuditLogger.java +++ b/src/java/org/apache/cassandra/audit/BinAuditLogger.java @@ -98,6 +98,11 @@ public class BinAuditLogger implements IAuditLogger @VisibleForTesting public static class Message extends BinLog.ReleaseableWriteMarshallable implements WeightedQueue.Weighable { + /** + * The shallow size of a {@code Message} object. + */ + private static final long EMPTY_SIZE = ObjectSizes.measure(new Message("")); + private final String message; public Message(String message) @@ -130,7 +135,7 @@ public class BinAuditLogger implements IAuditLogger @Override public int weight() { - return Ints.checkedCast(ObjectSizes.sizeOf(message)); + return Ints.checkedCast(EMPTY_SIZE + ObjectSizes.sizeOf(message)); } } } diff --git a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java index dc0645fe60..26aeb85795 100644 --- a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java +++ b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java @@ -205,7 +205,7 @@ public class ColumnIdentifier implements IMeasurableMemory, Comparable<ColumnIde public long unsharedHeapSizeExcludingData() { return EMPTY_SIZE - + ObjectSizes.sizeOnHeapExcludingData(bytes) + + ObjectSizes.sizeOnHeapExcludingDataOf(bytes) + ObjectSizes.sizeOf(text); } diff --git a/src/java/org/apache/cassandra/db/BufferClustering.java b/src/java/org/apache/cassandra/db/BufferClustering.java index ed6d61c82c..6cacbd14c9 100644 --- a/src/java/org/apache/cassandra/db/BufferClustering.java +++ b/src/java/org/apache/cassandra/db/BufferClustering.java @@ -52,7 +52,7 @@ public class BufferClustering extends AbstractBufferClusteringPrefix implements { if (this == Clustering.EMPTY || this == Clustering.STATIC_CLUSTERING) return 0; - return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(values); + return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingDataOf(values); } public static BufferClustering make(ByteBuffer... values) diff --git a/src/java/org/apache/cassandra/db/rows/BufferCell.java b/src/java/org/apache/cassandra/db/rows/BufferCell.java index 0491df9d31..d6918533e8 100644 --- a/src/java/org/apache/cassandra/db/rows/BufferCell.java +++ b/src/java/org/apache/cassandra/db/rows/BufferCell.java @@ -155,7 +155,7 @@ public class BufferCell extends AbstractCell<ByteBuffer> @Override public long unsharedHeapSizeExcludingData() { - return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(value) + (path == null ? 0 : path.unsharedHeapSizeExcludingData()); + return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingDataOf(value) + (path == null ? 0 : path.unsharedHeapSizeExcludingData()); } @Override diff --git a/src/java/org/apache/cassandra/db/rows/CellPath.java b/src/java/org/apache/cassandra/db/rows/CellPath.java index 8e018a7c74..8e1ce7fe2c 100644 --- a/src/java/org/apache/cassandra/db/rows/CellPath.java +++ b/src/java/org/apache/cassandra/db/rows/CellPath.java @@ -138,7 +138,7 @@ public abstract class CellPath implements IMeasurableMemory @Override public long unsharedHeapSizeExcludingData() { - return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(value); + return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingDataOf(value); } } diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java index 19f28c339b..9bda82057f 100644 --- a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java +++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java @@ -32,7 +32,8 @@ import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.bytecomparable.ByteSource; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.ObjectSizes; -import org.github.jamm.MemoryLayoutSpecification; + +import org.github.jamm.MemoryMeterStrategy; /** * In-memory trie built for fast modification and reads executing concurrently with writes from a single mutator thread. @@ -973,7 +974,7 @@ public class InMemoryTrie<T> extends InMemoryReadTrie<T> /** Returns the on heap size of the memtable trie itself, not counting any space taken by referenced content. */ public long sizeOnHeap() { - return contentCount * MemoryLayoutSpecification.SPEC.getReferenceSize() + + return contentCount * MemoryMeterStrategy.MEMORY_LAYOUT.getReferenceSize() + REFERENCE_ARRAY_ON_HEAP_SIZE * getChunkIdx(contentCount, CONTENTS_START_SHIFT, CONTENTS_START_SIZE) + (bufferType == BufferType.ON_HEAP ? allocatedPos + EMPTY_SIZE_ON_HEAP : EMPTY_SIZE_OFF_HEAP) + REFERENCE_ARRAY_ON_HEAP_SIZE * getChunkIdx(allocatedPos, BUF_START_SHIFT, BUF_START_SIZE); @@ -1021,7 +1022,7 @@ public class InMemoryTrie<T> extends InMemoryReadTrie<T> int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE); int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE); AtomicReferenceArray<T> contentArray = contentArrays[leadBit]; - int contentOverhead = ((contentArray != null ? contentArray.length() : 0) - ofs) * MemoryLayoutSpecification.SPEC.getReferenceSize(); + int contentOverhead = ((contentArray != null ? contentArray.length() : 0) - ofs) * MemoryMeterStrategy.MEMORY_LAYOUT.getReferenceSize(); return bufferOverhead + contentOverhead; } diff --git a/src/java/org/apache/cassandra/fql/FullQueryLogger.java b/src/java/org/apache/cassandra/fql/FullQueryLogger.java index 54c24c97df..1cd781391b 100644 --- a/src/java/org/apache/cassandra/fql/FullQueryLogger.java +++ b/src/java/org/apache/cassandra/fql/FullQueryLogger.java @@ -50,7 +50,6 @@ import org.apache.cassandra.utils.binlog.BinLog; import org.apache.cassandra.utils.binlog.BinLogOptions; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import org.apache.cassandra.utils.concurrent.WeightedQueue; -import org.github.jamm.MemoryLayoutSpecification; import static com.google.common.base.Preconditions.checkNotNull; @@ -85,9 +84,6 @@ public class FullQueryLogger implements QueryEvents.Listener private static final int EMPTY_LIST_SIZE = Ints.checkedCast(ObjectSizes.measureDeep(new ArrayList<>(0))); private static final int EMPTY_BYTEBUF_SIZE; - private static final int OBJECT_HEADER_SIZE = MemoryLayoutSpecification.SPEC.getObjectHeaderSize(); - private static final int OBJECT_REFERENCE_SIZE = MemoryLayoutSpecification.SPEC.getReferenceSize(); - public static final FullQueryLogger instance = new FullQueryLogger(); volatile BinLog binLog; @@ -332,6 +328,11 @@ public class FullQueryLogger implements QueryEvents.Listener public static class Query extends AbstractLogEntry { + /** + * The shallow size of a {@code Query} object. + */ + private static final long EMPTY_SIZE = ObjectSizes.measure(new Query()); + private final String query; public Query(String query, QueryOptions queryOptions, QueryState queryState, long queryStartTime) @@ -340,6 +341,14 @@ public class FullQueryLogger implements QueryEvents.Listener this.query = query; } + /** + * Constructor only use to compute this class shallow size. + */ + private Query() + { + this.query = null; + } + @Override protected String type() { @@ -356,12 +365,21 @@ public class FullQueryLogger implements QueryEvents.Listener @Override public int weight() { - return Ints.checkedCast(ObjectSizes.sizeOf(query)) + super.weight(); + // Object deep size = Object' shallow size + query field deep size + deep size of the parent fields + return Ints.checkedCast(EMPTY_SIZE + ObjectSizes.sizeOf(query) + super.fieldsSize()); } } public static class Batch extends AbstractLogEntry { + /** + * The shallow size of a {@code Batch} object (which includes primitive fields). + */ + private static final long EMPTY_SIZE = ObjectSizes.measure(new Batch()); + + /** + * The weight is pre-computed in the constructor and represent the object deep size. + */ private final int weight; private final BatchStatement.Type batchType; private final List<String> queries; @@ -380,25 +398,37 @@ public class FullQueryLogger implements QueryEvents.Listener this.values = values; this.batchType = batchType; - int weight = super.weight(); - - // weight, queries, values, batch type - weight += Integer.BYTES + // cached weight - 2 * EMPTY_LIST_SIZE + // queries + values lists - 3 * OBJECT_REFERENCE_SIZE; // batchType and two lists references + // We assume that all the lists are ArrayLists and that the size of each underlying array is the one of the list + // (which is obviously wrong but not worst than the previous computation that was ignoring part of the arrays size in the computation). + long queriesSize = EMPTY_LIST_SIZE + ObjectSizes.sizeOfReferenceArray(queries.size()); for (String query : queries) - weight += ObjectSizes.sizeOf(checkNotNull(query)) + OBJECT_REFERENCE_SIZE; + queriesSize += ObjectSizes.sizeOf(checkNotNull(query)); + long valuesSize = EMPTY_LIST_SIZE + ObjectSizes.sizeOfReferenceArray(values.size()); for (List<ByteBuffer> subValues : values) { - weight += EMPTY_LIST_SIZE + OBJECT_REFERENCE_SIZE; - - for (ByteBuffer value : subValues) - weight += ObjectSizes.sizeOnHeapOf(value) + OBJECT_REFERENCE_SIZE; + valuesSize += EMPTY_LIST_SIZE + ObjectSizes.sizeOfReferenceArray(subValues.size()); + for (ByteBuffer subValue : subValues) + valuesSize += ObjectSizes.sizeOnHeapOf(subValue); } - this.weight = weight; + // No need to add the batch type which is an enum. + this.weight = Ints.checkedCast(EMPTY_SIZE // Shallow size object + + super.fieldsSize() // deep size of the parent fields (non-primitives as they are included in the shallow size) + + queriesSize // deep size queries field + + valuesSize); // deep size values field + } + + /** + * Constructor only use to compute this class shallow size. + */ + private Batch() + { + this.weight = 0; + this.batchType = null; + this.queries = null; + this.values = null; } @Override @@ -483,6 +513,19 @@ public class FullQueryLogger implements QueryEvents.Listener } } + /** + * Constructor only use to compute sub-classes shallow size. + */ + private AbstractLogEntry() + { + this.queryStartTime = 0; + this.protocolVersion = 0; + this.queryOptionsBuffer = null; + this.generatedTimestamp = 0; + this.generatedNowInSeconds = 0; + this.keyspace = null; + } + @Override protected long version() { @@ -508,16 +551,14 @@ public class FullQueryLogger implements QueryEvents.Listener queryOptionsBuffer.release(); } - @Override - public int weight() + /** + * Returns the sum of the non-primitive fields' deep sizes. + * @return the sum of the non-primitive fields' deep sizes. + */ + protected long fieldsSize() { - return OBJECT_HEADER_SIZE - + Long.BYTES // queryStartTime - + Integer.BYTES // protocolVersion - + OBJECT_REFERENCE_SIZE + EMPTY_BYTEBUF_SIZE + queryOptionsBuffer.capacity() // queryOptionsBuffer - + Long.BYTES // generatedTimestamp - + Long.BYTES // generatedNowInSeconds - + OBJECT_REFERENCE_SIZE + Ints.checkedCast(ObjectSizes.sizeOf(keyspace)); // keyspace + return EMPTY_BYTEBUF_SIZE + queryOptionsBuffer.capacity() // queryOptionsBuffer + + ObjectSizes.sizeOf(keyspace); // keyspace } } diff --git a/src/java/org/apache/cassandra/utils/ObjectSizes.java b/src/java/org/apache/cassandra/utils/ObjectSizes.java index 2d94983d18..8eff7f77bb 100644 --- a/src/java/org/apache/cassandra/utils/ObjectSizes.java +++ b/src/java/org/apache/cassandra/utils/ObjectSizes.java @@ -24,23 +24,26 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import org.github.jamm.MemoryLayoutSpecification; import org.github.jamm.MemoryMeter; +import org.github.jamm.MemoryMeter.ByteBufferMode; +import org.github.jamm.MemoryMeter.Guess; + +import static org.github.jamm.MemoryMeterStrategy.MEMORY_LAYOUT; +import static org.github.jamm.utils.ArrayMeasurementUtils.computeArraySize; /** - * A convenience class for wrapping access to MemoryMeter + * A convenience class for wrapping access to MemoryMeter. Should be used instead of using a {@code MemoryMeter} directly. + * {@code MemoryMeter} can be used directly for testing as it allow a more fine tuned configuration for comparison. */ public class ObjectSizes { - private static final MemoryMeter meter = new MemoryMeter().withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE) - .ignoreKnownSingletons(); - private static final MemoryMeter omitSharedMeter = meter.omitSharedBufferOverhead(); - - private static final long EMPTY_HEAP_BUFFER_SIZE = measure(ByteBufferUtil.EMPTY_BYTE_BUFFER); - private static final long EMPTY_BYTE_ARRAY_SIZE = measure(new byte[0]); - private static final long EMPTY_STRING_SIZE = measure(""); + private static final MemoryMeter meter = MemoryMeter.builder().withGuessing(Guess.INSTRUMENTATION_AND_SPECIFICATION, + Guess.UNSAFE) + .build(); - private static final long DIRECT_BUFFER_HEAP_SIZE = measure(ByteBuffer.allocateDirect(0)); + private static final long HEAP_BUFFER_SHALLOW_SIZE = measure(ByteBufferUtil.EMPTY_BYTE_BUFFER); + private static final long DIRECT_BUFFER_SHALLOW_SIZE = measure(ByteBuffer.allocateDirect(0)); + private static final long DIRECT_BUFFER_DEEP_SIZE = measureDeep(ByteBuffer.allocateDirect(0)); public static final long IPV6_SOCKET_ADDRESS_SIZE = ObjectSizes.measureDeep(new InetSocketAddress(getIpvAddress(16), 42)); @@ -52,10 +55,7 @@ public class ObjectSizes */ public static long sizeOfArray(byte[] bytes) { - if (bytes == null) - return 0; - - return sizeOfArray(bytes.length, 1); + return meter.measureArray(bytes); } /** @@ -66,10 +66,7 @@ public class ObjectSizes */ public static long sizeOfArray(long[] longs) { - if (longs == null) - return 0; - - return sizeOfArray(longs.length, 8); + return meter.measureArray(longs); } /** @@ -80,10 +77,7 @@ public class ObjectSizes */ public static long sizeOfArray(int[] ints) { - if (ints == null) - return 0; - - return sizeOfArray(ints.length, 4); + return meter.measureArray(ints); } /** @@ -94,7 +88,7 @@ public class ObjectSizes */ public static long sizeOfReferenceArray(int length) { - return sizeOfArray(length, MemoryLayoutSpecification.SPEC.getReferenceSize()); + return sizeOfArray(length, MEMORY_LAYOUT.getReferenceSize()); } /** @@ -105,15 +99,12 @@ public class ObjectSizes */ public static long sizeOfArray(Object[] objects) { - if (objects == null) - return 0; - - return sizeOfReferenceArray(objects.length); + return meter.measureArray(objects); } - private static long sizeOfArray(int length, long elementSize) + private static long sizeOfArray(int length, int elementSize) { - return MemoryLayoutSpecification.sizeOfArray(length, elementSize); + return computeArraySize(MEMORY_LAYOUT.getArrayHeaderSize(), length, elementSize, MEMORY_LAYOUT.getObjectAlignment()); } /** @@ -134,65 +125,89 @@ public class ObjectSizes /** * Amount of non-data heap memory consumed by the array of byte buffers. It sums memory consumed - * by the array itself and for each included byte buffer using {@link #sizeOnHeapExcludingData(ByteBuffer)}. + * by the array itself and for each included byte buffer using {@link #sizeOnHeapExcludingDataOf(ByteBuffer)}. */ - public static long sizeOnHeapExcludingData(ByteBuffer[] array) + public static long sizeOnHeapExcludingDataOf(ByteBuffer[] array) { if (array == null) return 0; long sum = sizeOfArray(array); for (ByteBuffer b : array) - sum += sizeOnHeapExcludingData(b); + sum += sizeOnHeapExcludingDataOf(b); return sum; } /** - * @return heap memory consumed by the byte buffer. If it is a slice, it counts the data size, but it does not - * include the internal array overhead. + * Measures the heap memory used by the specified byte buffer. If the buffer is a slab only the data size will be + * counted but not the internal overhead. A SLAB is assumed to be created by: {@code buffer.duplicate().position(start).limit(end)} without the use of {@code slice()}. + * <p>This method makes a certain amount of assumptions: + * <ul> + * <li>That slabs are always created using: {@code buffer.duplicate().position(start).limit(end)} and not through slice</li> + * <li>That the input buffers are not read-only buffers</li> + * <li>That the direct buffers that are not slab are not duplicates</li> + * </ul> + * Non-respect of those assumptions can lead to an invalid value being returned. + * @param buffer the buffer to measure + * @return the heap memory used by the specified byte buffer. */ public static long sizeOnHeapOf(ByteBuffer buffer) { if (buffer == null) return 0; - if (buffer.isDirect()) - return DIRECT_BUFFER_HEAP_SIZE; + assert !buffer.isReadOnly(); - int arrayLen = buffer.array().length; - int bufLen = buffer.remaining(); + // We assume here that slabs are always created using: buffer.duplicate().position(start).limit(end) and not through slice + if (ByteBufferMode.SLAB_ALLOCATION_NO_SLICE.isSlab(buffer)) + { + if (buffer.isDirect()) + return DIRECT_BUFFER_SHALLOW_SIZE; // We ignore the underlying buffer + + return HEAP_BUFFER_SHALLOW_SIZE + buffer.remaining(); // We ignore the array overhead + } - // if we're only referencing a sub-portion of the ByteBuffer, don't count the array overhead (assume it is SLAB - // allocated - the overhead amortized over all the allocations is negligible and better to undercount than over) - if (arrayLen > bufLen) - return EMPTY_HEAP_BUFFER_SIZE + bufLen; + if (buffer.isDirect()) + return DIRECT_BUFFER_DEEP_SIZE; // That might not be true if the buffer is a view of another buffer so we could undercount - return EMPTY_HEAP_BUFFER_SIZE + (arrayLen == 0 ? EMPTY_BYTE_ARRAY_SIZE : sizeOfArray(arrayLen, 1)); + return HEAP_BUFFER_SHALLOW_SIZE + meter.measureArray(buffer.array()); } /** - * @return non-data heap memory consumed by the byte buffer. If it is a slice, it does not include the internal - * array overhead. + * Measures the heap memory used by the specified byte buffer excluding the data. If the buffer shallow size will be counted. + * A SLAB is assumed to be created by: {@code buffer.duplicate().position(start).limit(end)} without the use of {@code slice()}. + * <p>This method makes a certain amount of assumptions: + * <ul> + * <li>That slabs are always created using: {@code buffer.duplicate().position(start).limit(end)} and not through slice</li> + * <li>That the input buffers are not read-only buffers</li> + * <li>That the direct buffers that are not slab are not duplicates</li> + * </ul> + * Non-respect of those assumptions can lead to an invalid value being returned. T + * @param buffer the buffer to measure + * @return the heap memory used by the specified byte buffer excluding the data.. */ - public static long sizeOnHeapExcludingData(ByteBuffer buffer) + public static long sizeOnHeapExcludingDataOf(ByteBuffer buffer) { if (buffer == null) return 0; - if (buffer.isDirect()) - return DIRECT_BUFFER_HEAP_SIZE; + assert !buffer.isReadOnly(); - int arrayLen = buffer.array().length; - int bufLen = buffer.remaining(); + // We assume here that slabs are always created using: buffer.duplicate().position(start).limit(end) and not through slice + if (ByteBufferMode.SLAB_ALLOCATION_NO_SLICE.isSlab(buffer)) + { + if (buffer.isDirect()) + return DIRECT_BUFFER_SHALLOW_SIZE; // We ignore the underlying buffer - // if we're only referencing a sub-portion of the ByteBuffer, don't count the array overhead (assume it is SLAB - // allocated - the overhead amortized over all the allocations is negligible and better to undercount than over) - if (arrayLen > bufLen) - return EMPTY_HEAP_BUFFER_SIZE; + return HEAP_BUFFER_SHALLOW_SIZE; // We ignore the array overhead + } + + if (buffer.isDirect()) + return DIRECT_BUFFER_DEEP_SIZE; // That might not be true if the buffer is a view of another buffer so we could undercount - // If buffers are dedicated, account for byte array size and any padding overhead - return EMPTY_HEAP_BUFFER_SIZE + (arrayLen == 0 ? EMPTY_BYTE_ARRAY_SIZE : (sizeOfArray(arrayLen, 1) - arrayLen)); + byte[] bytes = buffer.array(); + return HEAP_BUFFER_SHALLOW_SIZE + meter.measureArray(bytes) - bytes.length; } /** @@ -201,13 +216,9 @@ public class ObjectSizes * @param str String to calculate memory size of * @return Total in-memory size of the String */ - // TODO hard coding this to 2 isn't necessarily correct in Java 11 public static long sizeOf(String str) { - if (str == null) - return 0; - - return EMPTY_STRING_SIZE + sizeOfArray(str.length(), Character.BYTES); + return meter.measureStringDeep(str); } /** @@ -230,7 +241,7 @@ public class ObjectSizes */ public static long measureDeepOmitShared(Object pojo) { - return omitSharedMeter.measureDeep(pojo); + return meter.measureDeep(pojo, ByteBufferMode.SLAB_ALLOCATION_NO_SLICE); } /** diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java index 1250512bc3..f52ab28d6b 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java @@ -43,6 +43,7 @@ import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; import org.github.jamm.MemoryMeter; +import org.github.jamm.MemoryMeter.Guess; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -66,10 +67,11 @@ import org.openjdk.jmh.annotations.Warmup; @State(Scope.Benchmark) public class ComparisonReadBench { - // Note: To see a printout of the usage for each object, add .enableDebug() here (most useful with smaller number of + // Note: To see a printout of the usage for each object, add .printVisitedTree() here (most useful with smaller number of // partitions). - static MemoryMeter meter = new MemoryMeter().ignoreKnownSingletons() - .withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE); + static MemoryMeter meter = MemoryMeter.builder() + .withGuessing(Guess.INSTRUMENTATION_AND_SPECIFICATION, Guess.UNSAFE) + .build(); @Param({"ON_HEAP"}) BufferType bufferType = BufferType.OFF_HEAP; diff --git a/test/unit/org/apache/cassandra/db/CellSpecTest.java b/test/unit/org/apache/cassandra/db/CellSpecTest.java index 3387c78e9e..b14b74be22 100644 --- a/test/unit/org/apache/cassandra/db/CellSpecTest.java +++ b/test/unit/org/apache/cassandra/db/CellSpecTest.java @@ -121,7 +121,7 @@ public class CellSpecTest private static long valuePtrSize(Object value) { if (value instanceof ByteBuffer) - return ObjectSizes.sizeOnHeapExcludingData((ByteBuffer) value); + return ObjectSizes.sizeOnHeapExcludingDataOf((ByteBuffer) value); else if (value instanceof byte[]) return ObjectSizes.sizeOfArray((byte[]) value) - ((byte[]) value).length; throw new IllegalArgumentException("Unsupported type: " + value.getClass()); diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTest.java b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTest.java index acbdc89993..ed7b90e382 100644 --- a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTest.java +++ b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTest.java @@ -39,16 +39,20 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.utils.FBUtilities; import org.github.jamm.MemoryMeter; +import org.github.jamm.MemoryMeter.Guess; // Note: This test can be run in idea with the allocation type configured in the test yaml and memtable using the // value memtableClass is initialized with. @RunWith(Parameterized.class) public class MemtableSizeTest extends CQLTester { - // Note: To see a printout of the usage for each object, add .enableDebug() here (most useful with smaller number of + // Note: To see a printout of the usage for each object, add .printVisitedTree() here (most useful with smaller number of // partitions). - static MemoryMeter meter = new MemoryMeter().ignoreKnownSingletons() - .withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE); + static MemoryMeter meter = MemoryMeter.builder() + .withGuessing(Guess.INSTRUMENTATION_AND_SPECIFICATION, + Guess.UNSAFE) +// .printVisitedTreeUpTo(1000) + .build(); static final Logger logger = LoggerFactory.getLogger(MemtableSizeTest.class); @@ -185,18 +189,6 @@ public class MemtableSizeTest extends CQLTester FBUtilities.prettyPrintMemory(actualHeap), FBUtilities.prettyPrintMemory(expectedHeap - actualHeap)); logger.info(message); - if (Math.abs(actualHeap - expectedHeap) > max_difference) - { - // Under Java 11, it seems the meter can reach into phantom reference queues and count more space than - // is actually reachable. Unfortunately ignoreNonStrongReferences() does not help (worse, it throws - // exceptions trying to get a phantom referrent). Retrying the measurement appears to clear these up. - Thread.sleep(50); - long secondPass = meter.measureDeep(memtable); - logger.error("Deep size first pass {} second pass {}", - FBUtilities.prettyPrintMemory(deepSizeAfter), - FBUtilities.prettyPrintMemory(secondPass)); - expectedHeap = secondPass - deepSizeBefore; - } Assert.assertTrue(message, Math.abs(actualHeap - expectedHeap) <= max_difference); } diff --git a/test/unit/org/apache/cassandra/utils/ObjectSizesTest.java b/test/unit/org/apache/cassandra/utils/ObjectSizesTest.java index a4c77bccc4..28d93eeb16 100644 --- a/test/unit/org/apache/cassandra/utils/ObjectSizesTest.java +++ b/test/unit/org/apache/cassandra/utils/ObjectSizesTest.java @@ -22,128 +22,123 @@ import java.nio.ByteBuffer; import org.junit.Test; -import org.github.jamm.MemoryLayoutSpecification; import org.github.jamm.MemoryMeter; +import org.github.jamm.MemoryMeter.Guess; -import static org.assertj.core.api.Assertions.assertThat; +import static org.github.jamm.MemoryMeter.ByteBufferMode.SLAB_ALLOCATION_NO_SLICE; +import static org.junit.Assert.assertEquals; public class ObjectSizesTest { - private static final MemoryMeter meter = new MemoryMeter().withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE).omitSharedBufferOverhead().ignoreKnownSingletons(); - - private static final long EMPTY_HEAP_BUFFER_RAW_SIZE = meter.measure(ByteBuffer.allocate(0)); - private static final long EMPTY_OFFHEAP_BUFFER_RAW_SIZE = meter.measure(ByteBuffer.allocateDirect(0)); - private static final ByteBuffer[] EMPTY_BYTE_BUFFER_ARRAY = new ByteBuffer[0]; - - public static final long REF_ARRAY_0_SIZE = MemoryLayoutSpecification.sizeOfArray(0, MemoryLayoutSpecification.SPEC.getReferenceSize()); - public static final long REF_ARRAY_1_SIZE = MemoryLayoutSpecification.sizeOfArray(1, MemoryLayoutSpecification.SPEC.getReferenceSize()); - public static final long REF_ARRAY_2_SIZE = MemoryLayoutSpecification.sizeOfArray(2, MemoryLayoutSpecification.SPEC.getReferenceSize()); - - public static final long BYTE_ARRAY_0_SIZE = MemoryLayoutSpecification.sizeOfArray(0, 1); - public static final long BYTE_ARRAY_10_SIZE = MemoryLayoutSpecification.sizeOfArray(10, 1); - public static final long BYTE_ARRAY_10_EXCEPT_DATA_SIZE = MemoryLayoutSpecification.sizeOfArray(10, 1) - 10; - - private ByteBuffer buf10 = ByteBuffer.allocate(10); - private ByteBuffer prefixBuf8 = buf10.duplicate(); - private ByteBuffer suffixBuf9 = buf10.duplicate(); - private ByteBuffer infixBuf7 = buf10.duplicate(); - - { - prefixBuf8.limit(8); - - suffixBuf9.position(1); - suffixBuf9 = suffixBuf9.slice(); - - infixBuf7.limit(8); - infixBuf7.position(1); - infixBuf7 = infixBuf7.slice(); - } + // We use INSTRUMENTATION as principal strategy as it is our reference strategy + private static final MemoryMeter meter = MemoryMeter.builder() + .withGuessing(Guess.INSTRUMENTATION, Guess.UNSAFE) + .build(); @Test public void testSizeOnHeapExcludingData() { - // empty array of byte buffers - ByteBuffer[] buffers = EMPTY_BYTE_BUFFER_ARRAY; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_0_SIZE); - // single empty heap buffer - buffers = new ByteBuffer[]{ ByteBuffer.allocate(0) }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_0_SIZE); + checkBufferSizeExcludingData(ByteBuffer.allocate(0), 0); // single non-empty heap buffer - buffers = new ByteBuffer[]{ buf10 }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_10_EXCEPT_DATA_SIZE); + checkBufferSizeExcludingData(ByteBuffer.allocate(10), 10); // single empty direct buffer - buffers = new ByteBuffer[]{ ByteBuffer.allocateDirect(0) }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE); + checkBufferSizeExcludingData(ByteBuffer.allocateDirect(0), 0); // single non-empty direct buffer - buffers = new ByteBuffer[]{ ByteBuffer.allocateDirect(10) }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE); + checkBufferSizeExcludingData(ByteBuffer.allocateDirect(10), 0); + + // heap buffer being a prefix slab + ByteBuffer buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().limit(8); + checkBufferSizeExcludingData(buffer, 8); - // two different empty byte buffers - buffers = new ByteBuffer[]{ ByteBuffer.allocate(0), ByteBuffer.allocateDirect(0) }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_2_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_0_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE); + // heap buffer being a suffix slab + buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1); + checkBufferSizeExcludingData(buffer, 9); + + // heap buffer being an infix slab + buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1).limit(8); + checkBufferSizeExcludingData(buffer, 7); + } - // two different non-empty byte buffers - buffers = new ByteBuffer[]{ buf10, ByteBuffer.allocateDirect(500) }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_2_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_10_EXCEPT_DATA_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE); + private void checkBufferSizeExcludingData(ByteBuffer buffer, int dataSize) + { + assertEquals(meter.measureDeep(buffer, SLAB_ALLOCATION_NO_SLICE) - dataSize, ObjectSizes.sizeOnHeapExcludingDataOf(buffer)); + } - // heap buffer being a prefix slice of other buffer - buffers = new ByteBuffer[]{ prefixBuf8 }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE); + @Test + public void testSizeOnHeapExcludingDataArray() + { + checkBufferSizeExcludingDataArray(0, new ByteBuffer[0]); - // heap buffer being a suffix slice of other buffer - buffers = new ByteBuffer[]{ suffixBuf9 }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE); + // single heap buffer + checkBufferSizeExcludingDataArray(0, ByteBuffer.allocate(0)); - // heap buffer being an infix slice of other buffer - buffers = new ByteBuffer[]{ infixBuf7 }; - assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE); + // multiple buffers + checkBufferSizeExcludingDataArray(10, ByteBuffer.allocate(0), ByteBuffer.allocate(10), ByteBuffer.allocateDirect(10)); + + // heap buffer being a prefix slab + ByteBuffer prefix = (ByteBuffer) ByteBuffer.allocate(10).duplicate().limit(8); + + // heap buffer being a suffix slab + ByteBuffer suffix = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1); + checkBufferSizeExcludingDataArray(8 + 9, prefix, suffix); + } + + private void checkBufferSizeExcludingDataArray(int dataSize, ByteBuffer... buffers) + { + assertEquals(meter.measureDeep(buffers, SLAB_ALLOCATION_NO_SLICE) - dataSize, ObjectSizes.sizeOnHeapExcludingDataOf(buffers)); } @Test public void testSizeOnHeapOf() { - // empty array of byte buffers - ByteBuffer[] buffers = EMPTY_BYTE_BUFFER_ARRAY; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_0_SIZE); - // single empty heap buffer - buffers = new ByteBuffer[]{ ByteBuffer.allocate(0) }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_0_SIZE); + checkBufferSize(ByteBuffer.allocate(0)); // single non-empty heap buffer - buffers = new ByteBuffer[]{ buf10 }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_10_SIZE); + checkBufferSize(ByteBuffer.allocate(10)); // single empty direct buffer - buffers = new ByteBuffer[]{ ByteBuffer.allocateDirect(0) }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE); + checkBufferSize(ByteBuffer.allocateDirect(0)); // single non-empty direct buffer - buffers = new ByteBuffer[]{ ByteBuffer.allocateDirect(10) }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE); + checkBufferSize(ByteBuffer.allocateDirect(10)); + + // heap buffer being a prefix slab + ByteBuffer buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().limit(8); + checkBufferSize(buffer); + + // heap buffer being a suffix slab + buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1); + checkBufferSize(buffer); - // two different empty byte buffers - buffers = new ByteBuffer[]{ ByteBuffer.allocate(0), ByteBuffer.allocateDirect(0) }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_2_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_0_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE); + // heap buffer being an infix slab + buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1).limit(8); + checkBufferSize(buffer); + } + + private void checkBufferSize(ByteBuffer buffer) + { + assertEquals(meter.measureDeep(buffer, SLAB_ALLOCATION_NO_SLICE), ObjectSizes.sizeOnHeapOf(buffer)); + } - // two different non-empty byte buffers - buffers = new ByteBuffer[]{ buf10, ByteBuffer.allocateDirect(500) }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_2_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_10_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE); + @Test + public void testSizeOnHeapOfArray() + { + checkBufferArraySize(new ByteBuffer[0]); - // heap buffer being a prefix slice of other buffer - buffers = new ByteBuffer[]{ prefixBuf8 }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + 8); + // single heap buffer + checkBufferArraySize(ByteBuffer.allocate(0)); - // heap buffer being a suffix slice of other buffer - buffers = new ByteBuffer[]{ suffixBuf9 }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + 9); + // multiple buffers + checkBufferArraySize(ByteBuffer.allocate(0), ByteBuffer.allocate(10), ByteBuffer.allocateDirect(10)); + } - // heap buffer being an infix slice of other buffer - buffers = new ByteBuffer[]{ infixBuf7 }; - assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + 7); + private void checkBufferArraySize(ByteBuffer... buffers) + { + assertEquals(meter.measureDeep(buffers, SLAB_ALLOCATION_NO_SLICE), ObjectSizes.sizeOnHeapOf(buffers)); } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org