Merge branch 'cassandra-3.11' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6ec44528 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6ec44528 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6ec44528 Branch: refs/heads/trunk Commit: 6ec445282ad2ed0620d7fd16f23fab256123e7cf Parents: bfbc527 a6a9dce Author: Blake Eggleston <bdeggles...@gmail.com> Authored: Tue Nov 6 11:19:55 2018 -0800 Committer: Blake Eggleston <bdeggles...@gmail.com> Committed: Tue Nov 6 11:20:52 2018 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + src/java/org/apache/cassandra/db/Slice.java | 25 +----- .../cassandra/io/sstable/format/Version.java | 2 + .../io/sstable/format/big/BigFormat.java | 10 +++ .../io/sstable/metadata/MetadataCollector.java | 25 +++--- .../io/sstable/metadata/StatsMetadata.java | 14 ++- .../mc-1-big-CompressionInfo.db | Bin 0 -> 43 bytes .../mc-1-big-Data.db | Bin 0 -> 65 bytes .../mc-1-big-Digest.crc32 | 1 + .../mc-1-big-Filter.db | Bin 0 -> 16 bytes .../mc-1-big-Index.db | Bin 0 -> 8 bytes .../mc-1-big-Statistics.db | Bin 0 -> 4789 bytes .../mc-1-big-Summary.db | Bin 0 -> 56 bytes .../mc-1-big-TOC.txt | 8 ++ .../db/SinglePartitionSliceCommandTest.java | 89 +++++++++++++++++++ .../cassandra/io/sstable/LegacySSTableTest.java | 35 +++++++- 16 files changed, 170 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ec44528/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 41b3da9,f923fa0..2373cb2 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,330 -1,6 +1,331 @@@ +4.0 + * Partitioned outbound internode TCP connections can occur when nodes restart (CASSANDRA-14358) + * Don't write to system_distributed.repair_history, system_traces.sessions, system_traces.events in mixed version 3.X/4.0 clusters (CASSANDRA-14841) + * Avoid running query to self through messaging service (CASSANDRA-14807) + * Allow using custom script for chronicle queue BinLog archival (CASSANDRA-14373) + * Transient->Full range movements mishandle consistency level upgrade (CASSANDRA-14759) + * ReplicaCollection follow-up (CASSANDRA-14726) + * Transient node receives full data requests (CASSANDRA-14762) + * Enable snapshot artifacts publish (CASSANDRA-12704) + * Introduce RangesAtEndpoint.unwrap to simplify StreamSession.addTransferRanges (CASSANDRA-14770) + * LOCAL_QUORUM may speculate to non-local nodes, resulting in Timeout instead of Unavailable (CASSANDRA-14735) + * Avoid creating empty compaction tasks after truncate (CASSANDRA-14780) + * Fail incremental repair prepare phase if it encounters sstables from un-finalized sessions (CASSANDRA-14763) + * Add a check for receiving digest response from transient node (CASSANDRA-14750) + * Fail query on transient replica if coordinator only expects full data (CASSANDRA-14704) + * Remove mentions of transient replication from repair path (CASSANDRA-14698) + * Fix handleRepairStatusChangedNotification to remove first then add (CASSANDRA-14720) + * Allow transient node to serve as a repair coordinator (CASSANDRA-14693) + * DecayingEstimatedHistogramReservoir.EstimatedHistogramReservoirSnapshot returns wrong value for size() and incorrectly calculates count (CASSANDRA-14696) + * AbstractReplicaCollection equals and hash code should throw due to conflict between order sensitive/insensitive uses (CASSANDRA-14700) + * Detect inconsistencies in repaired data on the read path (CASSANDRA-14145) + * Add checksumming to the native protocol (CASSANDRA-13304) + * Make AuthCache more easily extendable (CASSANDRA-14662) + * Extend RolesCache to include detailed role info (CASSANDRA-14497) + * Add fqltool compare (CASSANDRA-14619) + * Add fqltool replay (CASSANDRA-14618) + * Log keyspace in full query log (CASSANDRA-14656) + * Transient Replication and Cheap Quorums (CASSANDRA-14404) + * Log server-generated timestamp and nowInSeconds used by queries in FQL (CASSANDRA-14675) + * Add diagnostic events for read repairs (CASSANDRA-14668) + * Use consistent nowInSeconds and timestamps values within a request (CASSANDRA-14671) + * Add sampler for query time and expose with nodetool (CASSANDRA-14436) + * Clean up Message.Request implementations (CASSANDRA-14677) + * Disable old native protocol versions on demand (CASANDRA-14659) + * Allow specifying now-in-seconds in native protocol (CASSANDRA-14664) + * Improve BTree build performance by avoiding data copy (CASSANDRA-9989) + * Make monotonic read / read repair configurable (CASSANDRA-14635) + * Refactor CompactionStrategyManager (CASSANDRA-14621) + * Flush netty client messages immediately by default (CASSANDRA-13651) + * Improve read repair blocking behavior (CASSANDRA-10726) + * Add a virtual table to expose settings (CASSANDRA-14573) + * Fix up chunk cache handling of metrics (CASSANDRA-14628) + * Extend IAuthenticator to accept peer SSL certificates (CASSANDRA-14652) + * Incomplete handling of exceptions when decoding incoming messages (CASSANDRA-14574) + * Add diagnostic events for user audit logging (CASSANDRA-13668) + * Allow retrieving diagnostic events via JMX (CASSANDRA-14435) + * Add base classes for diagnostic events (CASSANDRA-13457) + * Clear view system metadata when dropping keyspace (CASSANDRA-14646) + * Allocate ReentrantLock on-demand in java11 AtomicBTreePartitionerBase (CASSANDRA-14637) + * Make all existing virtual tables use LocalPartitioner (CASSANDRA-14640) + * Revert 4.0 GC alg back to CMS (CASANDRA-14636) + * Remove hardcoded java11 jvm args in idea workspace files (CASSANDRA-14627) + * Update netty to 4.1.128 (CASSANDRA-14633) + * Add a virtual table to expose thread pools (CASSANDRA-14523) + * Add a virtual table to expose caches (CASSANDRA-14538, CASSANDRA-14626) + * Fix toDate function for timestamp arguments (CASSANDRA-14502) + * Revert running dtests by default in circleci (CASSANDRA-14614) + * Stream entire SSTables when possible (CASSANDRA-14556) + * Cell reconciliation should not depend on nowInSec (CASSANDRA-14592) + * Add experimental support for Java 11 (CASSANDRA-9608) + * Make PeriodicCommitLogService.blockWhenSyncLagsNanos configurable (CASSANDRA-14580) + * Improve logging in MessageInHandler's constructor (CASSANDRA-14576) + * Set broadcast address in internode messaging handshake (CASSANDRA-14579) + * Wait for schema agreement prior to building MVs (CASSANDRA-14571) + * Make all DDL statements idempotent and not dependent on global state (CASSANDRA-13426) + * Bump the hints messaging version to match the current one (CASSANDRA-14536) + * OffsetAwareConfigurationLoader doesn't set ssl storage port causing bind errors in CircleCI (CASSANDRA-14546) + * Report why native_transport_port fails to bind (CASSANDRA-14544) + * Optimize internode messaging protocol (CASSANDRA-14485) + * Internode messaging handshake sends wrong messaging version number (CASSANDRA-14540) + * Add a virtual table to expose active client connections (CASSANDRA-14458) + * Clean up and refactor client metrics (CASSANDRA-14524) + * Nodetool import row cache invalidation races with adding sstables to tracker (CASSANDRA-14529) + * Fix assertions in LWTs after TableMetadata was made immutable (CASSANDRA-14356) + * Abort compactions quicker (CASSANDRA-14397) + * Support light-weight transactions in cassandra-stress (CASSANDRA-13529) + * Make AsyncOneResponse use the correct timeout (CASSANDRA-14509) + * Add option to sanity check tombstones on reads/compactions (CASSANDRA-14467) + * Add a virtual table to expose all running sstable tasks (CASSANDRA-14457) + * Let nodetool import take a list of directories (CASSANDRA-14442) + * Avoid unneeded memory allocations / cpu for disabled log levels (CASSANDRA-14488) + * Implement virtual keyspace interface (CASSANDRA-7622) + * nodetool import cleanup and improvements (CASSANDRA-14417) + * Bump jackson version to >= 2.9.5 (CASSANDRA-14427) + * Allow nodetool toppartitions without specifying table (CASSANDRA-14360) + * Audit logging for database activity (CASSANDRA-12151) + * Clean up build artifacts in docs container (CASSANDRA-14432) + * Minor network authz improvements (Cassandra-14413) + * Automatic sstable upgrades (CASSANDRA-14197) + * Replace deprecated junit.framework.Assert usages with org.junit.Assert (CASSANDRA-14431) + * Cassandra-stress throws NPE if insert section isn't specified in user profile (CASSSANDRA-14426) + * List clients by protocol versions `nodetool clientstats --by-protocol` (CASSANDRA-14335) + * Improve LatencyMetrics performance by reducing write path processing (CASSANDRA-14281) + * Add network authz (CASSANDRA-13985) + * Use the correct IP/Port for Streaming when localAddress is left unbound (CASSANDRA-14389) + * nodetool listsnapshots is missing local system keyspace snapshots (CASSANDRA-14381) + * Remove StreamCoordinator.streamExecutor thread pool (CASSANDRA-14402) + * Rename nodetool --with-port to --print-port to disambiguate from --port (CASSANDRA-14392) + * Client TOPOLOGY_CHANGE messages have wrong port. (CASSANDRA-14398) + * Add ability to load new SSTables from a separate directory (CASSANDRA-6719) + * Eliminate background repair and probablistic read_repair_chance table options + (CASSANDRA-13910) + * Bind to correct local address in 4.0 streaming (CASSANDRA-14362) + * Use standard Amazon naming for datacenter and rack in Ec2Snitch (CASSANDRA-7839) + * Fix junit failure for SSTableReaderTest (CASSANDRA-14387) + * Abstract write path for pluggable storage (CASSANDRA-14118) + * nodetool describecluster should be more informative (CASSANDRA-13853) + * Compaction performance improvements (CASSANDRA-14261) + * Refactor Pair usage to avoid boxing ints/longs (CASSANDRA-14260) + * Add options to nodetool tablestats to sort and limit output (CASSANDRA-13889) + * Rename internals to reflect CQL vocabulary (CASSANDRA-14354) + * Add support for hybrid MIN(), MAX() speculative retry policies + (CASSANDRA-14293, CASSANDRA-14338, CASSANDRA-14352) + * Fix some regressions caused by 14058 (CASSANDRA-14353) + * Abstract repair for pluggable storage (CASSANDRA-14116) + * Add meaningful toString() impls (CASSANDRA-13653) + * Add sstableloader option to accept target keyspace name (CASSANDRA-13884) + * Move processing of EchoMessage response to gossip stage (CASSANDRA-13713) + * Add coordinator write metric per CF (CASSANDRA-14232) + * Correct and clarify SSLFactory.getSslContext method and call sites (CASSANDRA-14314) + * Handle static and partition deletion properly on ThrottledUnfilteredIterator (CASSANDRA-14315) + * NodeTool clientstats should show SSL Cipher (CASSANDRA-14322) + * Add ability to specify driver name and version (CASSANDRA-14275) + * Abstract streaming for pluggable storage (CASSANDRA-14115) + * Forced incremental repairs should promote sstables if they can (CASSANDRA-14294) + * Use Murmur3 for validation compactions (CASSANDRA-14002) + * Comma at the end of the seed list is interpretated as localhost (CASSANDRA-14285) + * Refactor read executor and response resolver, abstract read repair (CASSANDRA-14058) + * Add optional startup delay to wait until peers are ready (CASSANDRA-13993) + * Add a few options to nodetool verify (CASSANDRA-14201) + * CVE-2017-5929 Security vulnerability and redefine default log rotation policy (CASSANDRA-14183) + * Use JVM default SSL validation algorithm instead of custom default (CASSANDRA-13259) + * Better document in code InetAddressAndPort usage post 7544, incorporate port into UUIDGen node (CASSANDRA-14226) + * Fix sstablemetadata date string for minLocalDeletionTime (CASSANDRA-14132) + * Make it possible to change neverPurgeTombstones during runtime (CASSANDRA-14214) + * Remove GossipDigestSynVerbHandler#doSort() (CASSANDRA-14174) + * Add nodetool clientlist (CASSANDRA-13665) + * Revert ProtocolVersion changes from CASSANDRA-7544 (CASSANDRA-14211) + * Non-disruptive seed node list reload (CASSANDRA-14190) + * Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185) + * Migrate dtests to use pytest and python3 (CASSANDRA-14134) + * Allow storage port to be configurable per node (CASSANDRA-7544) + * Make sub-range selection for non-frozen collections return null instead of empty (CASSANDRA-14182) + * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067) + * Remove unused on-heap BloomFilter implementation (CASSANDRA-14152) + * Delete temp test files on exit (CASSANDRA-14153) + * Make PartitionUpdate and Mutation immutable (CASSANDRA-13867) + * Fix CommitLogReplayer exception for CDC data (CASSANDRA-14066) + * Fix cassandra-stress startup failure (CASSANDRA-14106) + * Remove initialDirectories from CFS (CASSANDRA-13928) + * Fix trivial log format error (CASSANDRA-14015) + * Allow sstabledump to do a json object per partition (CASSANDRA-13848) + * Add option to optimise merkle tree comparison across replicas (CASSANDRA-3200) + * Remove unused and deprecated methods from AbstractCompactionStrategy (CASSANDRA-14081) + * Fix Distribution.average in cassandra-stress (CASSANDRA-14090) + * Support a means of logging all queries as they were invoked (CASSANDRA-13983) + * Presize collections (CASSANDRA-13760) + * Add GroupCommitLogService (CASSANDRA-13530) + * Parallelize initial materialized view build (CASSANDRA-12245) + * Fix flaky SecondaryIndexManagerTest.assert[Not]MarkedAsBuilt (CASSANDRA-13965) + * Make LWTs send resultset metadata on every request (CASSANDRA-13992) + * Fix flaky indexWithFailedInitializationIsNotQueryableAfterPartialRebuild (CASSANDRA-13963) + * Introduce leaf-only iterator (CASSANDRA-9988) + * Upgrade Guava to 23.3 and Airline to 0.8 (CASSANDRA-13997) + * Allow only one concurrent call to StatusLogger (CASSANDRA-12182) + * Refactoring to specialised functional interfaces (CASSANDRA-13982) + * Speculative retry should allow more friendly params (CASSANDRA-13876) + * Throw exception if we send/receive repair messages to incompatible nodes (CASSANDRA-13944) + * Replace usages of MessageDigest with Guava's Hasher (CASSANDRA-13291) + * Add nodetool cmd to print hinted handoff window (CASSANDRA-13728) + * Fix some alerts raised by static analysis (CASSANDRA-13799) + * Checksum sstable metadata (CASSANDRA-13321, CASSANDRA-13593) + * Add result set metadata to prepared statement MD5 hash calculation (CASSANDRA-10786) + * Refactor GcCompactionTest to avoid boxing (CASSANDRA-13941) + * Expose recent histograms in JmxHistograms (CASSANDRA-13642) + * Fix buffer length comparison when decompressing in netty-based streaming (CASSANDRA-13899) + * Properly close StreamCompressionInputStream to release any ByteBuf (CASSANDRA-13906) + * Add SERIAL and LOCAL_SERIAL support for cassandra-stress (CASSANDRA-13925) + * LCS needlessly checks for L0 STCS candidates multiple times (CASSANDRA-12961) + * Correctly close netty channels when a stream session ends (CASSANDRA-13905) + * Update lz4 to 1.4.0 (CASSANDRA-13741) + * Optimize Paxos prepare and propose stage for local requests (CASSANDRA-13862) + * Throttle base partitions during MV repair streaming to prevent OOM (CASSANDRA-13299) + * Use compaction threshold for STCS in L0 (CASSANDRA-13861) + * Fix problem with min_compress_ratio: 1 and disallow ratio < 1 (CASSANDRA-13703) + * Add extra information to SASI timeout exception (CASSANDRA-13677) + * Add incremental repair support for --hosts, --force, and subrange repair (CASSANDRA-13818) + * Rework CompactionStrategyManager.getScanners synchronization (CASSANDRA-13786) + * Add additional unit tests for batch behavior, TTLs, Timestamps (CASSANDRA-13846) + * Add keyspace and table name in schema validation exception (CASSANDRA-13845) + * Emit metrics whenever we hit tombstone failures and warn thresholds (CASSANDRA-13771) + * Make netty EventLoopGroups daemon threads (CASSANDRA-13837) + * Race condition when closing stream sessions (CASSANDRA-13852) + * NettyFactoryTest is failing in trunk on macOS (CASSANDRA-13831) + * Allow changing log levels via nodetool for related classes (CASSANDRA-12696) + * Add stress profile yaml with LWT (CASSANDRA-7960) + * Reduce memory copies and object creations when acting on ByteBufs (CASSANDRA-13789) + * Simplify mx4j configuration (Cassandra-13578) + * Fix trigger example on 4.0 (CASSANDRA-13796) + * Force minumum timeout value (CASSANDRA-9375) + * Use netty for streaming (CASSANDRA-12229) + * Use netty for internode messaging (CASSANDRA-8457) + * Add bytes repaired/unrepaired to nodetool tablestats (CASSANDRA-13774) + * Don't delete incremental repair sessions if they still have sstables (CASSANDRA-13758) + * Fix pending repair manager index out of bounds check (CASSANDRA-13769) + * Don't use RangeFetchMapCalculator when RF=1 (CASSANDRA-13576) + * Don't optimise trivial ranges in RangeFetchMapCalculator (CASSANDRA-13664) + * Use an ExecutorService for repair commands instead of new Thread(..).start() (CASSANDRA-13594) + * Fix race / ref leak in anticompaction (CASSANDRA-13688) + * Expose tasks queue length via JMX (CASSANDRA-12758) + * Fix race / ref leak in PendingRepairManager (CASSANDRA-13751) + * Enable ppc64le runtime as unsupported architecture (CASSANDRA-13615) + * Improve sstablemetadata output (CASSANDRA-11483) + * Support for migrating legacy users to roles has been dropped (CASSANDRA-13371) + * Introduce error metrics for repair (CASSANDRA-13387) + * Refactoring to primitive functional interfaces in AuthCache (CASSANDRA-13732) + * Update metrics to 3.1.5 (CASSANDRA-13648) + * batch_size_warn_threshold_in_kb can now be set at runtime (CASSANDRA-13699) + * Avoid always rebuilding secondary indexes at startup (CASSANDRA-13725) + * Upgrade JMH from 1.13 to 1.19 (CASSANDRA-13727) + * Upgrade SLF4J from 1.7.7 to 1.7.25 (CASSANDRA-12996) + * Default for start_native_transport now true if not set in config (CASSANDRA-13656) + * Don't add localhost to the graph when calculating where to stream from (CASSANDRA-13583) + * Make CDC availability more deterministic via hard-linking (CASSANDRA-12148) + * Allow skipping equality-restricted clustering columns in ORDER BY clause (CASSANDRA-10271) + * Use common nowInSec for validation compactions (CASSANDRA-13671) + * Improve handling of IR prepare failures (CASSANDRA-13672) + * Send IR coordinator messages synchronously (CASSANDRA-13673) + * Flush system.repair table before IR finalize promise (CASSANDRA-13660) + * Fix column filter creation for wildcard queries (CASSANDRA-13650) + * Add 'nodetool getbatchlogreplaythrottle' and 'nodetool setbatchlogreplaythrottle' (CASSANDRA-13614) + * fix race condition in PendingRepairManager (CASSANDRA-13659) + * Allow noop incremental repair state transitions (CASSANDRA-13658) + * Run repair with down replicas (CASSANDRA-10446) + * Added started & completed repair metrics (CASSANDRA-13598) + * Added started & completed repair metrics (CASSANDRA-13598) + * Improve secondary index (re)build failure and concurrency handling (CASSANDRA-10130) + * Improve calculation of available disk space for compaction (CASSANDRA-13068) + * Change the accessibility of RowCacheSerializer for third party row cache plugins (CASSANDRA-13579) + * Allow sub-range repairs for a preview of repaired data (CASSANDRA-13570) + * NPE in IR cleanup when columnfamily has no sstables (CASSANDRA-13585) + * Fix Randomness of stress values (CASSANDRA-12744) + * Allow selecting Map values and Set elements (CASSANDRA-7396) + * Fast and garbage-free Streaming Histogram (CASSANDRA-13444) + * Update repairTime for keyspaces on completion (CASSANDRA-13539) + * Add configurable upper bound for validation executor threads (CASSANDRA-13521) + * Bring back maxHintTTL propery (CASSANDRA-12982) + * Add testing guidelines (CASSANDRA-13497) + * Add more repair metrics (CASSANDRA-13531) + * RangeStreamer should be smarter when picking endpoints for streaming (CASSANDRA-4650) + * Avoid rewrapping an exception thrown for cache load functions (CASSANDRA-13367) + * Log time elapsed for each incremental repair phase (CASSANDRA-13498) + * Add multiple table operation support to cassandra-stress (CASSANDRA-8780) + * Fix incorrect cqlsh results when selecting same columns multiple times (CASSANDRA-13262) + * Fix WriteResponseHandlerTest is sensitive to test execution order (CASSANDRA-13421) + * Improve incremental repair logging (CASSANDRA-13468) + * Start compaction when incremental repair finishes (CASSANDRA-13454) + * Add repair streaming preview (CASSANDRA-13257) + * Cleanup isIncremental/repairedAt usage (CASSANDRA-13430) + * Change protocol to allow sending key space independent of query string (CASSANDRA-10145) + * Make gc_log and gc_warn settable at runtime (CASSANDRA-12661) + * Take number of files in L0 in account when estimating remaining compaction tasks (CASSANDRA-13354) + * Skip building views during base table streams on range movements (CASSANDRA-13065) + * Improve error messages for +/- operations on maps and tuples (CASSANDRA-13197) + * Remove deprecated repair JMX APIs (CASSANDRA-11530) + * Fix version check to enable streaming keep-alive (CASSANDRA-12929) + * Make it possible to monitor an ideal consistency level separate from actual consistency level (CASSANDRA-13289) + * Outbound TCP connections ignore internode authenticator (CASSANDRA-13324) + * Upgrade junit from 4.6 to 4.12 (CASSANDRA-13360) + * Cleanup ParentRepairSession after repairs (CASSANDRA-13359) + * Upgrade snappy-java to 1.1.2.6 (CASSANDRA-13336) + * Incremental repair not streaming correct sstables (CASSANDRA-13328) + * Upgrade the jna version to 4.3.0 (CASSANDRA-13300) + * Add the currentTimestamp, currentDate, currentTime and currentTimeUUID functions (CASSANDRA-13132) + * Remove config option index_interval (CASSANDRA-10671) + * Reduce lock contention for collection types and serializers (CASSANDRA-13271) + * Make it possible to override MessagingService.Verb ids (CASSANDRA-13283) + * Avoid synchronized on prepareForRepair in ActiveRepairService (CASSANDRA-9292) + * Adds the ability to use uncompressed chunks in compressed files (CASSANDRA-10520) + * Don't flush sstables when streaming for incremental repair (CASSANDRA-13226) + * Remove unused method (CASSANDRA-13227) + * Fix minor bugs related to #9143 (CASSANDRA-13217) + * Output warning if user increases RF (CASSANDRA-13079) + * Remove pre-3.0 streaming compatibility code for 4.0 (CASSANDRA-13081) + * Add support for + and - operations on dates (CASSANDRA-11936) + * Fix consistency of incrementally repaired data (CASSANDRA-9143) + * Increase commitlog version (CASSANDRA-13161) + * Make TableMetadata immutable, optimize Schema (CASSANDRA-9425) + * Refactor ColumnCondition (CASSANDRA-12981) + * Parallelize streaming of different keyspaces (CASSANDRA-4663) + * Improved compactions metrics (CASSANDRA-13015) + * Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031) + * Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855) + * Thrift removal (CASSANDRA-11115) + * Remove pre-3.0 compatibility code for 4.0 (CASSANDRA-12716) + * Add column definition kind to dropped columns in schema (CASSANDRA-12705) + * Add (automate) Nodetool Documentation (CASSANDRA-12672) + * Update bundled cqlsh python driver to 3.7.0 (CASSANDRA-12736) + * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681) + * Clean up the SSTableReader#getScanner API wrt removal of RateLimiter (CASSANDRA-12422) + * Use new token allocation for non bootstrap case as well (CASSANDRA-13080) + * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084) + * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510) + * Allow IN restrictions on column families with collections (CASSANDRA-12654) + * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028) + * Add timeUnit Days for cassandra-stress (CASSANDRA-13029) + * Add mutation size and batch metrics (CASSANDRA-12649) + * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999) + * Expose time spent waiting in thread pool queue (CASSANDRA-8398) + * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969) + * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946) + * Add support for arithmetic operators (CASSANDRA-11935) + * Add histogram for delay to deliver hints (CASSANDRA-13234) + * Fix cqlsh automatic protocol downgrade regression (CASSANDRA-13307) + * Changing `max_hint_window_in_ms` at runtime (CASSANDRA-11720) + * Trivial format error in StorageProxy (CASSANDRA-13551) + * Nodetool repair can hang forever if we lose the notification for the repair completing/failing (CASSANDRA-13480) + * Anticompaction can cause noisy log messages (CASSANDRA-13684) + * Switch to client init for sstabledump (CASSANDRA-13683) + * CQLSH: Don't pause when capturing data (CASSANDRA-13743) + * nodetool clearsnapshot requires --all to clear all snapshots (CASSANDRA-13391) + * Correctly count range tombstones in traces and tombstone thresholds (CASSANDRA-8527) + * cqlshrc.sample uses incorrect option for time formatting (CASSANDRA-14243) + + 3.11.4 Merged from 3.0: + * Sstable min/max metadata can cause data loss (CASSANDRA-14861) * Dropped columns can cause reverse sstable iteration to return prematurely (CASSANDRA-14838) * Legacy sstables with multi block range tombstones create invalid bound sequences (CASSANDRA-14823) * Expand range tombstone validation checks to multiple interim request stages (CASSANDRA-14824) http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ec44528/src/java/org/apache/cassandra/db/Slice.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/Slice.java index 3fa8bf3,715c64d..2517186 --- a/src/java/org/apache/cassandra/db/Slice.java +++ b/src/java/org/apache/cassandra/db/Slice.java @@@ -234,31 -235,15 +234,10 @@@ public class Slic */ public boolean intersects(ClusteringComparator comparator, List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues) { - // If this slice start after max or end before min, it can't intersect - if (start.compareTo(comparator, maxClusteringValues) > 0 || end.compareTo(comparator, minClusteringValues) < 0) - return false; - - // We could safely return true here, but there's a minor optimization: if the first component - // of the slice is restricted to a single value (typically the slice is [4:5, 4:7]), we can - // check that the second component falls within the min/max for that component (and repeat for - // all components). - for (int j = 0; j < minClusteringValues.size() && j < maxClusteringValues.size(); j++) - { - ByteBuffer s = j < start.size() ? start.get(j) : null; - ByteBuffer f = j < end.size() ? end.get(j) : null; - - // we already know the first component falls within its min/max range (otherwise we wouldn't get here) - if (j > 0 && (j < end.size() && comparator.compareComponent(j, f, minClusteringValues.get(j)) < 0 || - j < start.size() && comparator.compareComponent(j, s, maxClusteringValues.get(j)) > 0)) - return false; - - // if this component isn't equal in the start and finish, we don't need to check any more - if (j >= start.size() || j >= end.size() || comparator.compareComponent(j, s, f) != 0) - break; - } - return true; + // If this slice starts after max clustering or ends before min clustering, it can't intersect + return start.compareTo(comparator, maxClusteringValues) <= 0 && end.compareTo(comparator, minClusteringValues) >= 0; } - public String toString(CFMetaData metadata) - { - return toString(metadata.comparator); - } - public String toString(ClusteringComparator comparator) { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ec44528/src/java/org/apache/cassandra/io/sstable/format/Version.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/format/Version.java index 9b82c14,2b9dcbd..0e9e303 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@@ -45,27 -46,36 +45,29 @@@ public abstract class Versio public abstract boolean isLatestVersion(); - public abstract boolean hasSamplingLevel(); - - public abstract boolean hasNewStatsFile(); - - public abstract ChecksumType compressedChecksumType(); - - public abstract ChecksumType uncompressedChecksumType(); - - public abstract boolean hasRepairedAt(); - - public abstract boolean tracksLegacyCounterShards(); - - public abstract boolean hasNewFileName(); + public abstract int correspondingMessagingVersion(); // Only use by storage that 'storeRows' so far - public abstract boolean storeRows(); + public abstract boolean hasCommitLogLowerBound(); - public abstract int correspondingMessagingVersion(); // Only use by storage that 'storeRows' so far + public abstract boolean hasCommitLogIntervals(); - public abstract boolean hasOldBfHashOrder(); + public abstract boolean hasMaxCompressedLength(); - public abstract boolean hasCompactionAncestors(); + public abstract boolean hasPendingRepair(); - public abstract boolean hasBoundaries(); + public abstract boolean hasIsTransient(); - public abstract boolean hasCommitLogLowerBound(); + public abstract boolean hasMetadataChecksum(); - public abstract boolean hasCommitLogIntervals(); + /** + * The old bloomfilter format serializes the data as BIG_ENDIAN long's, the new one uses the + * same format as in memory (serializes as bytes). + * @return True if the bloomfilter file is old serialization format + */ + public abstract boolean hasOldBfFormat(); + public abstract boolean hasAccurateMinMax(); + public String getVersion() { return version; http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ec44528/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index 8eb8603,ea0214b..d65a7c0 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@@ -124,39 -125,72 +124,43 @@@ public class BigFormat implements SSTab // store rows natively // mb (3.0.7, 3.7): commit log lower bound included // mc (3.0.8, 3.9): commit log intervals included + // md (3.0.18, 3.11.4): corrected sstable min/max clustering + + // na (4.0.0): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format // // NOTE: when adding a new version, please add that to LegacySSTableTest, too. private final boolean isLatestVersion; - private final boolean hasSamplingLevel; - private final boolean newStatsFile; - private final ChecksumType compressedChecksumType; - private final ChecksumType uncompressedChecksumType; - private final boolean hasRepairedAt; - private final boolean tracksLegacyCounterShards; - private final boolean newFileName; - public final boolean storeRows; - public final int correspondingMessagingVersion; // Only use by storage that 'storeRows' so far - public final boolean hasBoundaries; - /** - * CASSANDRA-8413: 3.0 bloom filter representation changed (two longs just swapped) - * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution. - */ - private final boolean hasOldBfHashOrder; + public final int correspondingMessagingVersion; private final boolean hasCommitLogLowerBound; private final boolean hasCommitLogIntervals; + private final boolean hasAccurateMinMax; + public final boolean hasMaxCompressedLength; + private final boolean hasPendingRepair; + private final boolean hasMetadataChecksum; + private final boolean hasIsTransient; + /** - * CASSANDRA-7066: compaction ancerstors are no longer used and have been removed. + * CASSANDRA-9067: 4.0 bloom filter representation changed (two longs just swapped) + * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution. */ - private final boolean hasCompactionAncestors; + private final boolean hasOldBfFormat; BigVersion(String version) { super(instance, version); isLatestVersion = version.compareTo(current_version) == 0; - hasSamplingLevel = version.compareTo("ka") >= 0; - newStatsFile = version.compareTo("ka") >= 0; - - //For a while Adler32 was in use, now the CRC32 instrinsic is very good especially after Haswell - //PureJavaCRC32 was always faster than Adler32. See CASSANDRA-8684 - ChecksumType checksumType = ChecksumType.CRC32; - if (version.compareTo("ka") >= 0 && version.compareTo("ma") < 0) - checksumType = ChecksumType.Adler32; - this.uncompressedChecksumType = checksumType; - - checksumType = ChecksumType.CRC32; - if (version.compareTo("jb") >= 0 && version.compareTo("ma") < 0) - checksumType = ChecksumType.Adler32; - this.compressedChecksumType = checksumType; - - hasRepairedAt = version.compareTo("ka") >= 0; - tracksLegacyCounterShards = version.compareTo("ka") >= 0; - - newFileName = version.compareTo("la") >= 0; - - hasOldBfHashOrder = version.compareTo("ma") < 0; - hasCompactionAncestors = version.compareTo("ma") < 0; - storeRows = version.compareTo("ma") >= 0; - correspondingMessagingVersion = storeRows - ? MessagingService.VERSION_30 - : MessagingService.VERSION_21; - - hasBoundaries = version.compareTo("ma") < 0; - hasCommitLogLowerBound = (version.compareTo("lb") >= 0 && version.compareTo("ma") < 0) - || version.compareTo("mb") >= 0; + correspondingMessagingVersion = MessagingService.VERSION_30; + + hasCommitLogLowerBound = version.compareTo("mb") >= 0; hasCommitLogIntervals = version.compareTo("mc") >= 0; + hasAccurateMinMax = version.compareTo("md") >= 0; + hasMaxCompressedLength = version.compareTo("na") >= 0; + hasPendingRepair = version.compareTo("na") >= 0; + hasIsTransient = version.compareTo("na") >= 0; + hasMetadataChecksum = version.compareTo("na") >= 0; + hasOldBfFormat = version.compareTo("na") < 0; } @Override @@@ -195,14 -260,27 +199,20 @@@ } @Override - public boolean hasCommitLogIntervals() + public boolean hasMetadataChecksum() { - return hasCommitLogIntervals; + return hasMetadataChecksum; } + @Override + public boolean hasAccurateMinMax() + { + return hasAccurateMinMax; + } + - @Override - public boolean storeRows() - { - return storeRows; - } - - @Override - public int correspondingMessagingVersion() + public boolean isCompatible() { - return correspondingMessagingVersion; + return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0); } @Override http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ec44528/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java index 36c218b,a618c96..d7c6b61 mode 100755,100644..100755 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java @@@ -23,8 -24,10 +24,11 @@@ import java.util.Collections import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.UUID; + import com.google.common.base.Preconditions; + import com.google.common.collect.Maps; + import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; import com.clearspring.analytics.stream.cardinality.ICardinality; import org.apache.cassandra.db.*; @@@ -95,10 -95,10 +99,10 @@@ public class MetadataCollector implemen protected final MinMaxIntTracker localDeletionTimeTracker = new MinMaxIntTracker(Cell.NO_DELETION_TIME, Cell.NO_DELETION_TIME); protected final MinMaxIntTracker ttlTracker = new MinMaxIntTracker(Cell.NO_TTL, Cell.NO_TTL); protected double compressionRatio = NO_COMPRESSION_RATIO; - protected StreamingHistogram.StreamingHistogramBuilder estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogramBuilder(); + protected StreamingTombstoneHistogramBuilder estimatedTombstoneDropTime = new StreamingTombstoneHistogramBuilder(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE, SSTable.TOMBSTONE_HISTOGRAM_SPOOL_SIZE, SSTable.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS); protected int sstableLevel; - protected ByteBuffer[] minClusteringValues; - protected ByteBuffer[] maxClusteringValues; + private ClusteringPrefix minClustering = ClusteringBound.TOP; + private ClusteringPrefix maxClustering = ClusteringBound.BOTTOM; protected boolean hasLegacyCounterShards = false; protected long totalColumnsSet; protected long totalRows; @@@ -273,8 -271,9 +269,9 @@@ this.hasLegacyCounterShards = this.hasLegacyCounterShards || hasLegacyCounterShards; } - public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, SerializationHeader header) + public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, UUID pendingRepair, boolean isTransient, SerializationHeader header) { + Preconditions.checkState(comparator.compare(maxClustering, minClustering) >= 0); Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class); components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance)); components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize, http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ec44528/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java index f14fb5d,94e8d41..5b06ada mode 100755,100644..100755 --- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java @@@ -341,15 -307,20 +341,17 @@@ public class StatsMetadata extends Meta commitLogUpperBound = CommitLogPosition.serializer.deserialize(in); long minTimestamp = in.readLong(); long maxTimestamp = in.readLong(); - // We use MAX_VALUE as that's the default value for "no deletion time" - int minLocalDeletionTime = version.storeRows() ? in.readInt() : Integer.MAX_VALUE; + int minLocalDeletionTime = in.readInt(); int maxLocalDeletionTime = in.readInt(); - int minTTL = version.storeRows() ? in.readInt() : 0; - int maxTTL = version.storeRows() ? in.readInt() : Integer.MAX_VALUE; + int minTTL = in.readInt(); + int maxTTL = in.readInt(); double compressionRatio = in.readDouble(); - StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in); + TombstoneHistogram tombstoneHistogram = TombstoneHistogram.serializer.deserialize(in); int sstableLevel = in.readInt(); - long repairedAt = 0; - if (version.hasRepairedAt()) - repairedAt = in.readLong(); + long repairedAt = in.readLong(); + // for legacy sstables, we skip deserializing the min and max clustering value + // to prevent erroneously excluding sstables from reads (see CASSANDRA-14861) int colCount = in.readInt(); List<ByteBuffer> minClusteringValues = new ArrayList<>(colCount); for (int i = 0; i < colCount; i++) @@@ -358,12 -333,18 +364,16 @@@ colCount = in.readInt(); List<ByteBuffer> maxClusteringValues = new ArrayList<>(colCount); for (int i = 0; i < colCount; i++) - maxClusteringValues.add(ByteBufferUtil.readWithShortLength(in)); + { + ByteBuffer val = ByteBufferUtil.readWithShortLength(in); + if (version.hasAccurateMinMax()) + maxClusteringValues.add(val); + } - boolean hasLegacyCounterShards = true; - if (version.tracksLegacyCounterShards()) - hasLegacyCounterShards = in.readBoolean(); + boolean hasLegacyCounterShards = in.readBoolean(); - long totalColumnsSet = version.storeRows() ? in.readLong() : -1L; - long totalRows = version.storeRows() ? in.readLong() : -1L; + long totalColumnsSet = in.readLong(); + long totalRows = in.readLong(); if (version.hasCommitLogLowerBound()) commitLogLowerBound = CommitLogPosition.serializer.deserialize(in); http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ec44528/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java index d03d3bc,1bdbcb2..f28bf41 --- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java +++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java @@@ -32,14 -40,16 +40,16 @@@ import org.junit.Test import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.Schema; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.Util; + import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; + import org.apache.cassandra.cql3.statements.SelectStatement; import org.apache.cassandra.db.filter.AbstractClusteringIndexFilter; import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter; import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; @@@ -323,4 -397,81 +335,81 @@@ public class SinglePartitionSliceComman Assert.assertNotNull(ret); Assert.assertFalse(ret.isEmpty()); } + + + public static List<Unfiltered> getUnfilteredsFromSinglePartition(String q) + { - SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(q).prepare(ClientState.forInternalCalls()).statement; ++ SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(q).prepare(ClientState.forInternalCalls()); + + List<Unfiltered> unfiltereds = new ArrayList<>(); - SinglePartitionReadCommand.Group query = (SinglePartitionReadCommand.Group) stmt.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds()); - Assert.assertEquals(1, query.commands.size()); - SinglePartitionReadCommand command = Iterables.getOnlyElement(query.commands); ++ SinglePartitionReadQuery.Group<SinglePartitionReadCommand> query = (SinglePartitionReadQuery.Group<SinglePartitionReadCommand>) stmt.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds()); ++ Assert.assertEquals(1, query.queries.size()); ++ SinglePartitionReadCommand command = Iterables.getOnlyElement(query.queries); + try (ReadExecutionController controller = ReadExecutionController.forCommand(command); + UnfilteredPartitionIterator partitions = command.executeLocally(controller)) + { + assert partitions.hasNext(); + try (UnfilteredRowIterator partition = partitions.next()) + { + while (partition.hasNext()) + { + Unfiltered next = partition.next(); + unfiltereds.add(next); + } + } + assert !partitions.hasNext(); + } + return unfiltereds; + } + + private static void assertQueryReturnsSingleRT(String query) + { + List<Unfiltered> unfiltereds = getUnfilteredsFromSinglePartition(query); + Assert.assertEquals(2, unfiltereds.size()); + Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker()); + Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false)); + Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker()); + Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false)); + } + + private static ByteBuffer bb(int v) + { + return Int32Type.instance.decompose(v); + } + + /** + * tests the bug raised in CASSANDRA-14861, where the sstable min/max can + * exclude range tombstones for clustering ranges not also covered by rows + */ + @Test + public void sstableFiltering() + { + QueryProcessor.executeOnceInternal("CREATE TABLE ks.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))"); - CFMetaData metadata = Schema.instance.getCFMetaData("ks", "legacy_mc_inaccurate_min_max"); - ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId); ++ TableMetadata metadata = Schema.instance.getTableMetadata("ks", "legacy_mc_inaccurate_min_max"); ++ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id); + + QueryProcessor.executeOnceInternal("INSERT INTO ks.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES (100, 2, 2, 2, 2)"); + QueryProcessor.executeOnceInternal("DELETE FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1"); + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1"); + cfs.forceBlockingFlush(); + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1"); - + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1 AND c3=1"); // clustering names + + cfs.truncateBlocking(); + + long nowMillis = System.currentTimeMillis(); + Slice slice = Slice.make(Clustering.make(bb(2), bb(3)), Clustering.make(bb(10), bb(10))); + RangeTombstone rt = new RangeTombstone(slice, new DeletionTime(TimeUnit.MILLISECONDS.toMicros(nowMillis), + Ints.checkedCast(TimeUnit.MILLISECONDS.toSeconds(nowMillis)))); - PartitionUpdate update = new PartitionUpdate(cfs.metadata, bb(100), cfs.metadata.partitionColumns(), 1); - update.add(rt); - new Mutation(update).apply(); ++ ++ PartitionUpdate.Builder builder = new PartitionUpdate.Builder(metadata, bb(100), metadata.regularAndStaticColumns(), 1); ++ builder.add(rt); ++ new Mutation(builder.build()).apply(); + + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2"); + cfs.forceBlockingFlush(); + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2"); + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2 AND c3=2"); // clustering names + + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ec44528/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index c4ccf48,bd51c0f..7c98e7e --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@@ -22,12 -22,10 +22,13 @@@ import java.io.FileInputStream import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Random; +import java.util.UUID; +import com.google.common.collect.Lists; + import com.google.common.collect.Iterables; import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; @@@ -39,14 -37,20 +40,24 @@@ import org.slf4j.LoggerFactory import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; + import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; + import org.apache.cassandra.cql3.statements.SelectStatement; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.compaction.AbstractCompactionTask; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.compaction.Verifier; +import org.apache.cassandra.db.streaming.CassandraOutgoingFile; + import org.apache.cassandra.db.ReadExecutionController; + import org.apache.cassandra.db.SinglePartitionReadCommand; + import org.apache.cassandra.db.SinglePartitionSliceCommandTest; + import org.apache.cassandra.db.compaction.Verifier; + import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; + import org.apache.cassandra.db.rows.RangeTombstoneMarker; + import org.apache.cassandra.db.rows.Unfiltered; + import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@@ -55,13 -59,11 +66,14 @@@ import org.apache.cassandra.io.sstable. import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.CacheService; + import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.streaming.OutgoingStream; import org.apache.cassandra.streaming.StreamPlan; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@@ -250,7 -290,28 +262,28 @@@ public class LegacySSTableTes } @Test + public void testInaccurateSSTableMinMax() throws Exception + { + QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))"); - loadLegacyTable("legacy_%s_inaccurate_min_max%s", "mc", ""); ++ loadLegacyTable("legacy_%s_inaccurate_min_max%s", "mc"); + + /* + sstable has the following mutations: + INSERT INTO legacy_tables.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES (100, 4, 4, 4, 4) + DELETE FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1<3 + */ + + String query = "SELECT * FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1"; + List<Unfiltered> unfiltereds = SinglePartitionSliceCommandTest.getUnfilteredsFromSinglePartition(query); + Assert.assertEquals(2, unfiltereds.size()); + Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker()); + Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false)); + Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker()); + Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false)); + } + + @Test - public void testVerifyOldSSTables() throws Exception + public void testVerifyOldSSTables() throws IOException { for (String legacyVersion : legacyVersions) { @@@ -545,13 -592,16 +578,13 @@@ File ksDir = new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables", BigFormat.latestVersion)); ksDir.mkdirs(); - for (int compact = 0; compact <= 1; compact++) - { - copySstablesFromTestData(String.format("legacy_%s_simple%s", BigFormat.latestVersion, getCompactNameSuffix(compact)), ksDir); - copySstablesFromTestData(String.format("legacy_%s_simple_counter%s", BigFormat.latestVersion, getCompactNameSuffix(compact)), ksDir); - copySstablesFromTestData(String.format("legacy_%s_clust%s", BigFormat.latestVersion, getCompactNameSuffix(compact)), ksDir); - copySstablesFromTestData(String.format("legacy_%s_clust_counter%s", BigFormat.latestVersion, getCompactNameSuffix(compact)), ksDir); - } + copySstablesFromTestData(String.format("legacy_%s_simple", BigFormat.latestVersion), ksDir); + copySstablesFromTestData(String.format("legacy_%s_simple_counter", BigFormat.latestVersion), ksDir); + copySstablesFromTestData(String.format("legacy_%s_clust", BigFormat.latestVersion), ksDir); + copySstablesFromTestData(String.format("legacy_%s_clust_counter", BigFormat.latestVersion), ksDir); } - private void copySstablesFromTestData(String table, File ksDir) throws IOException + public static void copySstablesFromTestData(String table, File ksDir) throws IOException { File cfDir = new File(ksDir, table); cfDir.mkdir(); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
