This is an automated email from the ASF dual-hosted git repository. adelapena pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 0d8126dd143898588f4efcdc40b8e2bb10597185 Merge: ac4cd09 a690f33 Author: Andrés de la Peña <a.penya.gar...@gmail.com> AuthorDate: Mon Mar 7 16:30:02 2022 +0000 Merge branch 'cassandra-4.0' into trunk CHANGES.txt | 1 + .../apache/cassandra/cql3/ColumnIdentifier.java | 2 +- .../org/apache/cassandra/db/rows/ArrayCell.java | 2 +- .../org/apache/cassandra/db/rows/BufferCell.java | 2 +- .../org/apache/cassandra/db/rows/CellPath.java | 2 +- .../org/apache/cassandra/fql/FullQueryLogger.java | 49 ++++---- .../org/apache/cassandra/utils/ObjectSizes.java | 121 ++++++++++++++------ .../unit/org/apache/cassandra/db/CellSpecTest.java | 9 +- .../apache/cassandra/utils/ObjectSizesTest.java | 123 +++++++++++++++++++-- 9 files changed, 231 insertions(+), 80 deletions(-) diff --cc CHANGES.txt index 206ab5b,9320244..ff1d989 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,100 -1,5 +1,101 @@@ -4.0.4 +4.1 + * Expose streaming as a vtable (CASSANDRA-17390) + * Make startup checks configurable (CASSANDRA-17220) + * Add guardrail for number of partition keys on IN queries (CASSANDRA-17186) + * update Python test framework from nose to pytest (CASSANDRA-17293) + * Fix improper CDC commit log segments deletion in non-blocking mode (CASSANDRA-17233) + * Add support for string concatenations through the + operator (CASSANDRA-17190) + * Limit the maximum hints size per host (CASSANDRA-17142) + * Add a virtual table for exposing batch metrics (CASSANDRA-17225) + * Flatten guardrails config (CASSANDRA-17353) + * Instance failed to start up due to NPE in StartupClusterConnectivityChecker (CASSANDRA-17347) + * add the shorter version of version flag (-v) in cqlsh (CASSANDRA-17236) + * Make vtables accessible via internode messaging (CASSANDRA-17295) + * Add support for PEM based key material for SSL (CASSANDRA-17031) + * Standardize storage configuration parameters' names. Support unit suffixes. (CASSANDRA-15234) + * Remove support for Windows (CASSANDRA-16956) + * Runtime-configurable YAML option to prohibit USE statements (CASSANDRA-17318) + * When streaming sees a ClosedChannelException this triggers the disk failure policy (CASSANDRA-17116) + * Add a virtual table for exposing prepared statements metrics (CASSANDRA-17224) + * Remove python 2.x support from cqlsh (CASSANDRA-17242) + * Prewarm role and credential caches to avoid timeouts at startup (CASSANDRA-16958) + * Make capacity/validity/updateinterval/activeupdate for Auth Caches configurable via nodetool (CASSANDRA-17063) + * Added startup check for read_ahead_kb setting (CASSANDRA-16436) + * Avoid unecessary array allocations and initializations when performing query checks (CASSANDRA-17209) + * Add guardrail for list operations that require read before write (CASSANDRA-17154) + * Migrate thresholds for number of keyspaces and tables to guardrails (CASSANDRA-17195) + * Remove self-reference in SSTableTidier (CASSANDRA-17205) + * Add guardrail for query page size (CASSANDRA-17189) + * Allow column_index_size_in_kb to be configurable through nodetool (CASSANDRA-17121) + * Emit a metric for number of local read and write calls + * Add non-blocking mode for CDC writes (CASSANDRA-17001) + * Add guardrails framework (CASSANDRA-17147) + * Harden resource management on SSTable components to prevent future leaks (CASSANDRA-17174) + * Make nodes more resilient to local unrelated files during startup (CASSANDRA-17082) + * repair prepare message would produce a wrong error message if network timeout happened rather than reply wait timeout (CASSANDRA-16992) + * Log queries that fail on timeout or unavailable errors up to once per minute by default (CASSANDRA-17159) + * Refactor normal/preview/IR repair to standardize repair cleanup and error handling of failed RepairJobs (CASSANDRA-17069) + * Log missing peers in StartupClusterConnectivityChecker (CASSANDRA-17130) + * Introduce separate rate limiting settings for entire SSTable streaming (CASSANDRA-17065) + * Implement Virtual Tables for Auth Caches (CASSANDRA-16914) + * Actively update auth cache in the background (CASSANDRA-16957) + * Add unix time conversion functions (CASSANDRA-17029) + * JVMStabilityInspector.forceHeapSpaceOomMaybe should handle all non-heap OOMs rather than only supporting direct only (CASSANDRA-17128) + * Forbid other Future implementations with checkstyle (CASSANDRA-17055) + * commit log was switched from non-daemon to daemon threads, which causes the JVM to exit in some case as no non-daemon threads are active (CASSANDRA-17085) + * Add a Denylist to block reads and writes on specific partition keys (CASSANDRA-12106) + * v4+ protocol did not clean up client warnings, which caused leaking the state (CASSANDRA-17054) + * Remove duplicate toCQLString in ReadCommand (CASSANDRA-17023) + * Ensure hint window is persistent across restarts of a node (CASSANDRA-14309) + * Allow to GRANT or REVOKE multiple permissions in a single statement (CASSANDRA-17030) + * Allow to grant permission for all tables in a keyspace (CASSANDRA-17027) + * Log time spent writing keys during compaction (CASSANDRA-17037) + * Make nodetool compactionstats and sstable_tasks consistent (CASSANDRA-16976) + * Add metrics and logging around index summary redistribution (CASSANDRA-17036) + * Add configuration options for minimum allowable replication factor and default replication factor (CASSANDRA-14557) + * Expose information about stored hints via a nodetool command and a virtual table (CASSANDRA-14795) + * Add broadcast_rpc_address to system.local (CASSANDRA-11181) + * Add support for type casting in WHERE clause components and in the values of INSERT/UPDATE statements (CASSANDRA-14337) + * add credentials file support to CQLSH (CASSANDRA-16983) + * Skip remaining bytes in the Envelope buffer when a ProtocolException is thrown to avoid double decoding (CASSANDRA-17026) + * Allow reverse iteration of resources during permissions checking (CASSANDRA-17016) + * Add feature to verify correct ownership of attached locations on disk at startup (CASSANDRA-16879) + * Make SSLContext creation pluggable/extensible (CASSANDRA-16666) + * Add soft/hard limits to local reads to protect against reading too much data in a single query (CASSANDRA-16896) + * Avoid token cache invalidation for removing a non-member node (CASSANDRA-15290) + * Allow configuration of consistency levels on auth operations (CASSANDRA-12988) + * Add number of sstables in a compaction to compactionstats output (CASSANDRA-16844) + * Upgrade Caffeine to 2.9.2 (CASSANDRA-15153) + * Allow DELETE and TRUNCATE to work on Virtual Tables if the implementation allows it (CASSANDRA-16806) + * Include SASI components to snapshots (CASSANDRA-15134) + * Fix missed wait latencies in the output of `nodetool tpstats -F` (CASSANDRA-16938) + * Reduce native transport max frame size to 16MB (CASSANDRA-16886) + * Add support for filtering using IN restrictions (CASSANDRA-14344) + * Provide a nodetool command to invalidate auth caches (CASSANDRA-16404) + * Catch read repair timeout exceptions and add metric (CASSANDRA-16880) + * Exclude Jackson 1.x transitive dependency of hadoop* provided dependencies (CASSANDRA-16854) + * Add client warnings and abort to tombstone and coordinator reads which go past a low/high watermark (CASSANDRA-16850) + * Add TTL support to nodetool snapshots (CASSANDRA-16789) + * Allow CommitLogSegmentReader to optionally skip sync marker CRC checks (CASSANDRA-16842) + * allow blocking IPs from updating metrics about traffic (CASSANDRA-16859) + * Request-Based Native Transport Rate-Limiting (CASSANDRA-16663) + * Implement nodetool getauditlog command (CASSANDRA-16725) + * Clean up repair code (CASSANDRA-13720) + * Background schedule to clean up orphaned hints files (CASSANDRA-16815) + * Modify SecondaryIndexManager#indexPartition() to retrieve only columns for which indexes are actually being built (CASSANDRA-16776) + * Batch the token metadata update to improve the speed (CASSANDRA-15291) + * Reduce the log level on "expected" repair exceptions (CASSANDRA-16775) + * Make JMXTimer expose attributes using consistent time unit (CASSANDRA-16760) + * Remove check on gossip status from DynamicEndpointSnitch::updateScores (CASSANDRA-11671) + * Fix AbstractReadQuery::toCQLString not returning valid CQL (CASSANDRA-16510) + * Log when compacting many tombstones (CASSANDRA-16780) + * Display bytes per level in tablestats for LCS tables (CASSANDRA-16799) + * Add isolated flush timer to CommitLogMetrics and ensure writes correspond to single WaitingOnCommit data points (CASSANDRA-16701) + * Add a system property to set hostId if not yet initialized (CASSANDRA-14582) + * GossiperTest.testHasVersion3Nodes didn't take into account trunk version changes, fixed to rely on latest version (CASSANDRA-16651) + * Update JNA library to 5.9.0 and snappy-java to version 1.1.8.4 (CASSANDRA-17040) +Merged from 4.0: + * Fix ObjectSizes implementation and usages (CASSANDRA-17402) * Fix race condition bug during local session repair (CASSANDRA-17335) * Fix ignored streaming encryption settings in sstableloader (CASSANDRA-17367) * Streaming tasks handle empty SSTables correctly (CASSANDRA-16349) diff --cc src/java/org/apache/cassandra/db/rows/ArrayCell.java index eddc11c,c097249..0e328bd --- a/src/java/org/apache/cassandra/db/rows/ArrayCell.java +++ b/src/java/org/apache/cassandra/db/rows/ArrayCell.java @@@ -110,15 -110,8 +110,15 @@@ public class ArrayCell extends Abstract return new BufferCell(column, timestamp, ttl, localDeletionTime, allocator.clone(value), path == null ? null : path.copy(allocator)); } + @Override + public long unsharedHeapSize() + { + return EMPTY_SIZE + ObjectSizes.sizeOfArray(value) + (path == null ? 0 : path.unsharedHeapSize()); + } + + @Override public long unsharedHeapSizeExcludingData() { - return EMPTY_SIZE + ObjectSizes.sizeOfEmptyByteArray() + (path == null ? 0 : path.unsharedHeapSizeExcludingData()); + return EMPTY_SIZE + ObjectSizes.sizeOfArray(value) - value.length + (path == null ? 0 : path.unsharedHeapSizeExcludingData()); } } diff --cc src/java/org/apache/cassandra/db/rows/BufferCell.java index 7870bf1,f5994f1..85d28f8 --- a/src/java/org/apache/cassandra/db/rows/BufferCell.java +++ b/src/java/org/apache/cassandra/db/rows/BufferCell.java @@@ -142,15 -142,8 +142,15 @@@ public class BufferCell extends Abstrac return new BufferCell(column, timestamp, ttl, localDeletionTime, allocator.clone(value), path == null ? null : path.copy(allocator)); } + @Override + public long unsharedHeapSize() + { + return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(value) + (path == null ? 0 : path.unsharedHeapSize()); + } + + @Override public long unsharedHeapSizeExcludingData() { - return EMPTY_SIZE + ObjectSizes.sizeOfEmptyHeapByteBuffer() + (path == null ? 0 : path.unsharedHeapSizeExcludingData()); + return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(value) + (path == null ? 0 : path.unsharedHeapSizeExcludingData()); } } diff --cc src/java/org/apache/cassandra/db/rows/CellPath.java index 27b6272,1bf8b8f..cee2484 --- a/src/java/org/apache/cassandra/db/rows/CellPath.java +++ b/src/java/org/apache/cassandra/db/rows/CellPath.java @@@ -126,16 -125,9 +126,16 @@@ public abstract class CellPath implemen return new SingleItemCellPath(allocator.clone(value)); } + @Override + public long unsharedHeapSize() + { + return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(value); + } + + @Override public long unsharedHeapSizeExcludingData() { - return EMPTY_SIZE + ObjectSizes.sizeOfEmptyHeapByteBuffer(); + return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(value); } } diff --cc src/java/org/apache/cassandra/utils/ObjectSizes.java index a8e8b6c,468522c..07066cf --- a/src/java/org/apache/cassandra/utils/ObjectSizes.java +++ b/src/java/org/apache/cassandra/utils/ObjectSizes.java @@@ -31,13 -29,15 +29,14 @@@ import org.github.jamm.MemoryMeter */ public class ObjectSizes { - private static final MemoryMeter meter = new MemoryMeter() - .withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE) - .ignoreKnownSingletons(); - private static final MemoryMeter meter = new MemoryMeter().omitSharedBufferOverhead() - .withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE) ++ private static final MemoryMeter meter = new MemoryMeter().withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE) + .ignoreKnownSingletons(); - private static final long BUFFER_EMPTY_SIZE = measure(ByteBufferUtil.EMPTY_BYTE_BUFFER); - private static final long BYTE_ARRAY_EMPTY_SIZE = measure(new byte[0]); - private static final long STRING_EMPTY_SIZE = measure(""); + private static final long EMPTY_HEAP_BUFFER_SIZE = measure(ByteBufferUtil.EMPTY_BYTE_BUFFER); + private static final long EMPTY_BYTE_ARRAY_SIZE = measure(new byte[0]); + private static final long EMPTY_STRING_SIZE = measure(""); + + private static final long DIRECT_BUFFER_HEAP_SIZE = measure(ByteBuffer.allocateDirect(0)); /** * Memory a byte array consumes diff --cc test/unit/org/apache/cassandra/db/CellSpecTest.java index 24a3b12,4cc886d..dc7ffa8 --- a/test/unit/org/apache/cassandra/db/CellSpecTest.java +++ b/test/unit/org/apache/cassandra/db/CellSpecTest.java @@@ -103,23 -81,6 +103,20 @@@ public class CellSpecTes .isEqualTo(expected); } + private long valueSizeOnHeapOf(Object value) + { + if (value instanceof ByteBuffer) + { + ByteBuffer bb = (ByteBuffer) value; - long size = ObjectSizes.sizeOfEmptyHeapByteBuffer(); - if (!bb.isDirect()) - size += ObjectSizes.sizeOfArray(bb.array()); - return size; ++ return ObjectSizes.sizeOnHeapOf(bb); + } + else if (value instanceof byte[]) + { + return ObjectSizes.sizeOfArray((byte[]) value); + } + throw new IllegalArgumentException("Unsupported type: " + value.getClass()); + } + private static long valuePtrSize(Object value) { if (value instanceof ByteBuffer) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org