Merge branch 'cassandra-1.0' into cassandra-1.1.0 Conflicts: CHANGES.txt src/java/org/apache/cassandra/db/ColumnFamilyStore.java src/java/org/apache/cassandra/db/DataTracker.java src/java/org/apache/cassandra/db/compaction/LeveledManifest.java src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d78db44c Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d78db44c Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d78db44c Branch: refs/heads/trunk Commit: d78db44cb6b77acaee3fe05711ad6e8a14bb4558 Parents: 5f31757 f239907 Author: Sylvain Lebresne <sylv...@datastax.com> Authored: Wed Mar 7 18:53:07 2012 +0100 Committer: Sylvain Lebresne <sylv...@datastax.com> Committed: Wed Mar 7 18:53:07 2012 +0100 ---------------------------------------------------------------------- CHANGES.txt | 7 +++ src/java/org/apache/cassandra/db/ColumnFamily.java | 20 +++++++ .../org/apache/cassandra/db/ColumnFamilyStore.java | 16 ++++-- src/java/org/apache/cassandra/db/DataTracker.java | 15 +++--- .../cassandra/db/compaction/CompactionManager.java | 8 ++- .../cassandra/db/compaction/CompactionTask.java | 16 ++++-- .../db/compaction/LeveledCompactionStrategy.java | 13 ++++- .../cassandra/db/compaction/LeveledManifest.java | 41 ++++++++++----- .../cassandra/db/compaction/OperationType.java | 1 + .../org/apache/cassandra/io/util/FileUtils.java | 17 ------ .../SSTableListChangedNotification.java | 11 +++- .../cassandra/db/compaction/CompactionsTest.java | 3 +- 12 files changed, 112 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index a9e1c10,925a4a9..d7a595b --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,31 -1,4 +1,35 @@@ ++<<<<<<< HEAD +1.1-dev + * start hint replay as soon as FD notifies that the target is back up + (CASSANDRA-3958) + * avoid unproductive deserializing of cached rows during compaction + (CASSANDRA-3921) + * fix concurrency issues with CQL keyspace creation (CASSANDRA-3903) + * Show Effective Owership via Nodetool ring <keyspace> (CASSANDRA-3412) + * Update ORDER BY syntax for CQL3 (CASSANDRA-3925) + * Fix BulkRecordWriter to not throw NPE if reducer gets no map data from Hadoop (CASSANDRA-3944) + * Fix bug with counters in super columns (CASSANDRA-3821) + * Remove deprecated merge_shard_chance (CASSANDRA-3940) + * add a convenient way to reset a node's schema (CASSANDRA-2963) + * fix for intermittent SchemaDisagreementException (CASSANDRA-3884) + * ignore deprecated KsDef/CfDef/ColumnDef fields in native schema (CASSANDRA-3963) + * CLI to report when unsupported column_metadata pair was given (CASSANDRA-3959) + * reincarnate removed and deprecated KsDef/CfDef attributes (CASSANDRA-3953) + * Fix race between writes and read for cache (CASSANDRA-3862) + * perform static initialization of StorageProxy on start-up (CASSANDRA-3797) + * support trickling fsync() on writes (CASSANDRA-3950) + * expose counters for unavailable/timeout exceptions given to thrift clients (CASSANDRA-3671) + * avoid quadratic startup time in LeveledManifest (CASSANDRA-3952) +Merged from 1.0: + * remove the wait on hint future during write (CASSANDRA-3870) + * (cqlsh) ignore missing CfDef opts (CASSANDRA-3933) + * (cqlsh) look for cqlshlib relative to realpath (CASSANDRA-3767) + * Fix short read protection (CASSANDRA-3934) + * don't change manifest level for cleanup, scrub, and upgradesstables + operations under LeveledCompactionStrategy (CASSANDRA-3989) ++======= + 1.0.9 ++>>>>>>> cassandra-1.0 * always compact away deleted hints immediately after handoff (CASSANDRA-3955) * delete hints from dropped ColumnFamilies on handoff instead of erroring out (CASSANDRA-3975) @@@ -34,89 -7,11 +38,92 @@@ * Pig: Composite column support (CASSANDRA-384) * Avoid NPE during repair when a keyspace has no CFs (CASSANDRA-3988) * Fix division-by-zero error on get_slice (CASSANDRA-4000) + * don't change manifest level for cleanup, scrub, and upgradesstables + operations under LeveledCompactionStrategy (CASSANDRA-3989) + * fix race leading to super columns assertion failure (CASSANDRA-3957) +1.1-beta1 + * add nodetool rebuild_index (CASSANDRA-3583) + * add nodetool rangekeysample (CASSANDRA-2917) + * Fix streaming too much data during move operations (CASSANDRA-3639) + * Nodetool and CLI connect to localhost by default (CASSANDRA-3568) + * Reduce memory used by primary index sample (CASSANDRA-3743) + * (Hadoop) separate input/output configurations (CASSANDRA-3197, 3765) + * avoid returning internal Cassandra classes over JMX (CASSANDRA-2805) + * add row-level isolation via SnapTree (CASSANDRA-2893) + * Optimize key count estimation when opening sstable on startup + (CASSANDRA-2988) + * multi-dc replication optimization supporting CL > ONE (CASSANDRA-3577) + * add command to stop compactions (CASSANDRA-1740, 3566, 3582) + * multithreaded streaming (CASSANDRA-3494) + * removed in-tree redhat spec (CASSANDRA-3567) + * "defragment" rows for name-based queries under STCS, again (CASSANDRA-2503) + * Recycle commitlog segments for improved performance + (CASSANDRA-3411, 3543, 3557, 3615) + * update size-tiered compaction to prioritize small tiers (CASSANDRA-2407) + * add message expiration logic to OutboundTcpConnection (CASSANDRA-3005) + * off-heap cache to use sun.misc.Unsafe instead of JNA (CASSANDRA-3271) + * EACH_QUORUM is only supported for writes (CASSANDRA-3272) + * replace compactionlock use in schema migration by checking CFS.isValid + (CASSANDRA-3116) + * recognize that "SELECT first ... *" isn't really "SELECT *" (CASSANDRA-3445) + * Use faster bytes comparison (CASSANDRA-3434) + * Bulk loader is no longer a fat client, (HADOOP) bulk load output format + (CASSANDRA-3045) + * (Hadoop) add support for KeyRange.filter + * remove assumption that keys and token are in bijection + (CASSANDRA-1034, 3574, 3604) + * always remove endpoints from delevery queue in HH (CASSANDRA-3546) + * fix race between cf flush and its 2ndary indexes flush (CASSANDRA-3547) + * fix potential race in AES when a repair fails (CASSANDRA-3548) + * Remove columns shadowed by a deleted container even when we cannot purge + (CASSANDRA-3538) + * Improve memtable slice iteration performance (CASSANDRA-3545) + * more efficient allocation of small bloom filters (CASSANDRA-3618) + * Use separate writer thread in SSTableSimpleUnsortedWriter (CASSANDRA-3619) + * fsync the directory after new sstable or commitlog segment are created (CASSANDRA-3250) + * fix minor issues reported by FindBugs (CASSANDRA-3658) + * global key/row caches (CASSANDRA-3143, 3849) + * optimize memtable iteration during range scan (CASSANDRA-3638) + * introduce 'crc_check_chance' in CompressionParameters to support + a checksum percentage checking chance similarly to read-repair (CASSANDRA-3611) + * a way to deactivate global key/row cache on per-CF basis (CASSANDRA-3667) + * fix LeveledCompactionStrategy broken because of generation pre-allocation + in LeveledManifest (CASSANDRA-3691) + * finer-grained control over data directories (CASSANDRA-2749) + * Fix ClassCastException during hinted handoff (CASSANDRA-3694) + * Upgrade Thrift to 0.7 (CASSANDRA-3213) + * Make stress.java insert operation to use microseconds (CASSANDRA-3725) + * Allows (internally) doing a range query with a limit of columns instead of + rows (CASSANDRA-3742) + * Allow rangeSlice queries to be start/end inclusive/exclusive (CASSANDRA-3749) + * Fix BulkLoader to support new SSTable layout and add stream + throttling to prevent an NPE when there is no yaml config (CASSANDRA-3752) + * Allow concurrent schema migrations (CASSANDRA-1391, 3832) + * Add SnapshotCommand to trigger snapshot on remote node (CASSANDRA-3721) + * Make CFMetaData conversions to/from thrift/native schema inverses + (CASSANDRA_3559) + * Add initial code for CQL 3.0-beta (CASSANDRA-3781, 3753) + * Add wide row support for ColumnFamilyInputFormat (CASSANDRA-3264) + * Allow extending CompositeType comparator (CASSANDRA-3657) + * Avoids over-paging during get_count (CASSANDRA-3798) + * Add new command to rebuild a node without (repair) merkle tree calculations + (CASSANDRA-3483, 3922) + * respect not only row cache capacity but caching mode when + trying to read data (CASSANDRA-3812) + * fix system tests (CASSANDRA-3827) + * CQL support for altering key_validation_class in ALTER TABLE (CASSANDRA-3781) + * turn compression on by default (CASSANDRA-3871) + * make hexToBytes refuse invalid input (CASSANDRA-2851) + * Make secondary indexes CF inherit compression and compaction from their + parent CF (CASSANDRA-3877) + * Finish cleanup up tombstone purge code (CASSANDRA-3872) + * Avoid NPE on aboarted stream-out sessions (CASSANDRA-3904) + * BulkRecordWriter throws NPE for counter columns (CASSANDRA-3906) + * Support compression using BulkWriter (CASSANDRA-3907) + + 1.0.8 * fix race between cleanup and flush on secondary index CFSes (CASSANDRA-3712) * avoid including non-queried nodes in rangeslice read repair http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/ColumnFamily.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java index a8d1fc8,60a3487..a69989e --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@@ -51,7 -46,7 +51,8 @@@ import org.apache.cassandra.db.commitlo import org.apache.cassandra.db.compaction.AbstractCompactionStrategy; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.LeveledCompactionStrategy; +import org.apache.cassandra.db.filter.ExtendedFilter; + import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.filter.IFilter; import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.db.filter.QueryPath; @@@ -713,26 -779,18 +714,29 @@@ public class ColumnFamilyStore implemen public void updateRowCache(DecoratedKey key, ColumnFamily columnFamily) { - if (rowCache.isPutCopying()) + if (metadata.cfId == null) + return; // secondary index + + RowCacheKey cacheKey = new RowCacheKey(metadata.cfId, key); + + // always invalidate a copying cache value + if (CacheService.instance.rowCache.isPutCopying()) { - invalidateCachedRow(key); + invalidateCachedRow(cacheKey); + return; } - else + + // invalidate a normal cache value if it's a sentinel, so the read will retry (and include the new update) + IRowCacheEntry cachedRow = getCachedRowInternal(cacheKey); + if (cachedRow != null) { - ColumnFamily cachedRow = getRawCachedRow(key); - if (cachedRow != null) + if (cachedRow instanceof RowCacheSentinel) + invalidateCachedRow(cacheKey); + else - ((ColumnFamily) cachedRow).addAll(columnFamily, HeapAllocator.instance); + // columnFamily is what is written in the commit log. Because of the PeriodicCommitLog, this can be done in concurrency + // with this. So columnFamily shouldn't be modified and if it contains super columns, neither should they. So for super + // columns, we must make sure to clone them when adding to the cache. That's what addAllWithSCCopy does (see #3957) - cachedRow.addAllWithSCCopy(columnFamily, HeapAllocator.instance); ++ ((ColumnFamily) cachedRow).addAllWithSCCopy(columnFamily, HeapAllocator.instance); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/DataTracker.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/DataTracker.java index 3b1db56,c4b8a7b..d6b5fe5 --- a/src/java/org/apache/cassandra/db/DataTracker.java +++ b/src/java/org/apache/cassandra/db/DataTracker.java @@@ -30,7 -30,11 +30,8 @@@ import com.google.common.collect.* import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.cache.AutoSavingCache; import org.apache.cassandra.config.DatabaseDescriptor; + import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableReader; import org.apache.cassandra.notifications.INotification; import org.apache.cassandra.notifications.INotificationConsumer; @@@ -290,8 -259,9 +291,8 @@@ public class DataTracke // notifySSTablesChanged -> LeveledManifest.promote doesn't like a no-op "promotion" return; } - notifySSTablesChanged(notCompacting, Collections.<SSTableReader>emptySet()); - - replace(sstables, Collections.<SSTableReader>emptyList()); - notifySSTablesChanged(sstables, Collections.<SSTableReader>emptyList(), OperationType.UNKNOWN); ++ notifySSTablesChanged(notCompacting, Collections.<SSTableReader>emptySet(), OperationType.UNKNOWN); + postReplace(notCompacting, Collections.<SSTableReader>emptySet()); } /** (Re)initializes the tracker, purging all references. */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/CompactionManager.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java index 94bc788,55fab3c..ff50db5 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@@ -775,10 -796,10 +777,10 @@@ public class CompactionManager implemen logger.info(String.format(format, writer.getFilename(), startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime)); } - // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd + // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd cfs.indexManager.flushIndexesBlocking(); - cfs.replaceCompactedSSTables(Arrays.asList(sstable), results); + cfs.replaceCompactedSSTables(Arrays.asList(sstable), results, OperationType.CLEANUP); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/CompactionTask.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 683241f,5847bf7..9c19070 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@@ -42,15 -41,19 +42,17 @@@ import org.apache.cassandra.utils.FBUti public class CompactionTask extends AbstractCompactionTask { protected static final Logger logger = LoggerFactory.getLogger(CompactionTask.class); - protected String compactionFileLocation; protected final int gcBefore; protected boolean isUserDefined; + protected OperationType compactionType; protected static long totalBytesCompacted = 0; public CompactionTask(ColumnFamilyStore cfs, Collection<SSTableReader> sstables, final int gcBefore) { super(cfs, sstables); - compactionFileLocation = null; this.gcBefore = gcBefore; this.isUserDefined = false; + this.compactionType = OperationType.COMPACTION; } public static synchronized long addToTotalBytesCompacted(long bytesCompacted) http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index d2edebe,6dc6ce9..32c0a9b --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@@ -166,11 -164,9 +166,10 @@@ public class LeveledManifes int maximumLevel = 0; for (SSTableReader sstable : removed) { - int thisLevel = levelOf(sstable); + int thisLevel = remove(sstable); + assert thisLevel >= 0; maximumLevel = Math.max(maximumLevel, thisLevel); minimumLevel = Math.min(minimumLevel, thisLevel); - remove(sstable); } // it's valid to do a remove w/o an add (e.g. on truncate) @@@ -286,19 -295,20 +298,20 @@@ private int levelOf(SSTableReader sstable) { - for (int level = 0; level < generations.length; level++) - { - if (generations[level].contains(sstable)) - return level; - } - return -1; + Integer level = sstableGenerations.get(sstable); + if (level == null) + return -1; + + return level.intValue(); } - private void remove(SSTableReader reader) + private int remove(SSTableReader reader) { int level = levelOf(reader); assert level >= 0 : reader + " not present in manifest"; generations[level].remove(reader); + sstableGenerations.remove(reader); + return level; } private void add(SSTableReader sstable, int level) http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/OperationType.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/io/util/FileUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java index 26a52b5,9f31b4e..92456ca --- a/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java +++ b/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java @@@ -23,11 -23,17 +23,15 @@@ package org.apache.cassandra.notificati import org.apache.cassandra.io.sstable.SSTableReader; -import java.util.List; - + import org.apache.cassandra.db.compaction.OperationType; + public class SSTableListChangedNotification implements INotification { - public Iterable<SSTableReader> removed; - public Iterable<SSTableReader> added; - public SSTableListChangedNotification(Iterable<SSTableReader> added, Iterable<SSTableReader> removed) + public final Iterable<SSTableReader> removed; + public final Iterable<SSTableReader> added; + public final OperationType compactionType; + + public SSTableListChangedNotification(Iterable<SSTableReader> added, Iterable<SSTableReader> removed, OperationType compactionType) { this.removed = removed; this.added = added; http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java ----------------------------------------------------------------------