merge from 2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/90e585dd Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/90e585dd Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/90e585dd Branch: refs/heads/trunk Commit: 90e585dde10189bc8e7044837ce2db91720ea2ce Parents: 216139f ecec863 Author: Jonathan Ellis <jbel...@apache.org> Authored: Tue Dec 17 14:35:34 2013 -0600 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Tue Dec 17 14:35:34 2013 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../compaction/AbstractCompactionStrategy.java | 2 +- .../cassandra/db/compaction/CompactionTask.java | 2 +- .../compaction/LeveledCompactionStrategy.java | 20 +++++++--------- .../db/compaction/LeveledCompactionTask.java | 8 +++---- .../db/compaction/LeveledManifest.java | 25 ++++++++++++++++---- .../SizeTieredCompactionStrategy.java | 2 +- .../cassandra/db/compaction/Upgrader.java | 2 +- .../cassandra/tools/StandaloneScrubber.java | 2 +- 9 files changed, 39 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/90e585dd/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index dcc7e33,c2cd052..6c9f2e1 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,25 -1,5 +1,26 @@@ +2.1 + * Multithreaded commitlog (CASSANDRA-3578) + * allocate fixed index summary memory pool and resample cold index summaries + to use less memory (CASSANDRA-5519) + * Removed multithreaded compaction (CASSANDRA-6142) + * Parallelize fetching rows for low-cardinality indexes (CASSANDRA-1337) + * change logging from log4j to logback (CASSANDRA-5883) + * switch to LZ4 compression for internode communication (CASSANDRA-5887) + * Stop using Thrift-generated Index* classes internally (CASSANDRA-5971) + * Remove 1.2 network compatibility code (CASSANDRA-5960) + * Remove leveled json manifest migration code (CASSANDRA-5996) + * Remove CFDefinition (CASSANDRA-6253) + * Use AtomicIntegerFieldUpdater in RefCountedMemory (CASSANDRA-6278) + * User-defined types for CQL3 (CASSANDRA-5590) + * Use of o.a.c.metrics in nodetool (CASSANDRA-5871, 6406) + * Batch read from OTC's queue and cleanup (CASSANDRA-1632) + * Secondary index support for collections (CASSANDRA-4511) + * SSTable metadata(Stats.db) format change (CASSANDRA-6356) + * Push composites support in the storage engine (CASSANDRA-5417) + + 2.0.4 + * Fix size-tiered compaction in LCS L0 (CASSANDRA-6496) * Fix assertion failure in filterColdSSTables (CASSANDRA-6483) * Fix row tombstones in larger-than-memory compactions (CASSANDRA-6008) * Fix cleanup ClassCastException (CASSANDRA-6462) http://git-wip-us.apache.org/repos/asf/cassandra/blob/90e585dd/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/90e585dd/src/java/org/apache/cassandra/db/compaction/CompactionTask.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 59f2f2f,2a23966..cabe486 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@@ -119,12 -118,14 +119,12 @@@ public class CompactionTask extends Abs long totalkeysWritten = 0; long estimatedTotalKeys = Math.max(cfs.metadata.getIndexInterval(), SSTableReader.getApproximateKeyCount(actuallyCompact, cfs.metadata)); - long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(actuallyCompact) / strategy.getMaxSSTableSize()); - long estimatedSSTables = Math.max(1, SSTable.getTotalBytes(actuallyCompact) / strategy.getMaxSSTableBytes()); ++ long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(actuallyCompact) / strategy.getMaxSSTableBytes()); long keysPerSSTable = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables); if (logger.isDebugEnabled()) - logger.debug("Expected bloom filter size : " + keysPerSSTable); + logger.debug("Expected bloom filter size : {}", keysPerSSTable); - AbstractCompactionIterable ci = DatabaseDescriptor.isMultithreadedCompaction() - ? new ParallelCompactionIterable(compactionType, strategy.getScanners(actuallyCompact), controller) - : new CompactionIterable(compactionType, strategy.getScanners(actuallyCompact), controller); + AbstractCompactionIterable ci = new CompactionIterable(compactionType, strategy.getScanners(actuallyCompact), controller); CloseableIterator<AbstractCompactedRow> iter = ci.iterator(); Map<DecoratedKey, RowIndexEntry> cachedKeys = new HashMap<DecoratedKey, RowIndexEntry>(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/90e585dd/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/90e585dd/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index 4dab156,92cd887..e08b9f7 --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@@ -566,4 -576,47 +569,18 @@@ public class LeveledManifes return newLevel; } + - /** - * Scary method mutating existing sstable component - * - * Tries to do it safely by moving the new file on top of the old one - * - * Caller needs to reload the sstable metadata (sstableReader.reloadSSTableMetadata()) - * - * @see org.apache.cassandra.io.sstable.SSTableReader#reloadSSTableMetadata() - * - * @param oldMetadata - * @param descriptor - * @param filename - * @param level - * @throws IOException - */ - public static synchronized void mutateLevel(Pair<SSTableMetadata, Set<Integer>> oldMetadata, Descriptor descriptor, String filename, int level) throws IOException - { - logger.debug("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), level); - SSTableMetadata metadata = SSTableMetadata.copyWithNewSSTableLevel(oldMetadata.left, level); - DataOutputStream out = new DataOutputStream(new FileOutputStream(filename + "-tmp")); - SSTableMetadata.serializer.legacySerialize(metadata, oldMetadata.right, descriptor, out); - out.flush(); - out.close(); - // we cant move a file on top of another file in windows: - if (!FBUtilities.isUnix()) - FileUtils.delete(filename); - FileUtils.renameWithConfirm(filename + "-tmp", filename); - } - + public static class CompactionCandidate + { + public final Collection<SSTableReader> sstables; + public final int level; + public final long maxSSTableBytes; + + public CompactionCandidate(Collection<SSTableReader> sstables, int level, long maxSSTableBytes) + { + this.sstables = sstables; + this.level = level; + this.maxSSTableBytes = maxSSTableBytes; + } + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/90e585dd/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/90e585dd/src/java/org/apache/cassandra/db/compaction/Upgrader.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/Upgrader.java index ef881c4,2805a52..e4d29e9 --- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java +++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java @@@ -57,7 -56,7 +57,7 @@@ public class Upgrade this.strategy = cfs.getCompactionStrategy(); long estimatedTotalKeys = Math.max(cfs.metadata.getIndexInterval(), SSTableReader.getApproximateKeyCount(toUpgrade, cfs.metadata)); - long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(this.toUpgrade) / strategy.getMaxSSTableSize()); - long estimatedSSTables = Math.max(1, SSTable.getTotalBytes(this.toUpgrade) / strategy.getMaxSSTableBytes()); ++ long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(this.toUpgrade) / strategy.getMaxSSTableBytes()); this.estimatedRows = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/90e585dd/src/java/org/apache/cassandra/tools/StandaloneScrubber.java ----------------------------------------------------------------------
