Merge branch 'cassandra-2.2' into cassandra-3.0 Conflicts: src/java/org/apache/cassandra/db/ColumnFamilyStore.java src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/136e7002 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/136e7002 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/136e7002 Branch: refs/heads/cassandra-3.0 Commit: 136e7002f0ceccb41513d8e54d21d0b0a30bab0c Parents: fb28cb7 31db093 Author: Marcus Eriksson <marc...@apache.org> Authored: Mon Aug 17 08:39:16 2015 +0200 Committer: Marcus Eriksson <marc...@apache.org> Committed: Mon Aug 17 08:39:16 2015 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 48 +++++++++++++++++--- .../DateTieredCompactionStrategy.java | 13 +++++- .../DateTieredCompactionStrategyTest.java | 1 + 4 files changed, 54 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/136e7002/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index bc17fcc,1299aa2..3686a3d --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -29,40 -16,6 +29,41 @@@ Merged from 2.1 when both exist (CASSANDRA-9777) * Release snapshot selfRef when doing snapshot repair (CASSANDRA-9998) * Cannot replace token does not exist - DN node removed as Fat Client (CASSANDRA-9871) +Merged from 2.0: + * Don't cast expected bf size to an int (CASSANDRA-9959) ++ * Make getFullyExpiredSSTables less expensive (CASSANDRA-9882) + + +3.0.0-alpha1 + * Implement proper sandboxing for UDFs (CASSANDRA-9402) + * Simplify (and unify) cleanup of compaction leftovers (CASSANDRA-7066) + * Allow extra schema definitions in cassandra-stress yaml (CASSANDRA-9850) + * Metrics should use up to date nomenclature (CASSANDRA-9448) + * Change CREATE/ALTER TABLE syntax for compression (CASSANDRA-8384) + * Cleanup crc and adler code for java 8 (CASSANDRA-9650) + * Storage engine refactor (CASSANDRA-8099, 9743, 9746, 9759, 9781, 9808, 9825, + 9848, 9705, 9859, 9867, 9874, 9828, 9801) + * Update Guava to 18.0 (CASSANDRA-9653) + * Bloom filter false positive ratio is not honoured (CASSANDRA-8413) + * New option for cassandra-stress to leave a ratio of columns null (CASSANDRA-9522) + * Change hinted_handoff_enabled yaml setting, JMX (CASSANDRA-9035) + * Add algorithmic token allocation (CASSANDRA-7032) + * Add nodetool command to replay batchlog (CASSANDRA-9547) + * Make file buffer cache independent of paths being read (CASSANDRA-8897) + * Remove deprecated legacy Hadoop code (CASSANDRA-9353) + * Decommissioned nodes will not rejoin the cluster (CASSANDRA-8801) + * Change gossip stabilization to use endpoit size (CASSANDRA-9401) + * Change default garbage collector to G1 (CASSANDRA-7486) + * Populate TokenMetadata early during startup (CASSANDRA-9317) + * Undeprecate cache recentHitRate (CASSANDRA-6591) + * Add support for selectively varint encoding fields (CASSANDRA-9499, 9865) + * Materialized Views (CASSANDRA-6477) +Merged from 2.2: + * Avoid grouping sstables for anticompaction with DTCS (CASSANDRA-9900) + * UDF / UDA execution time in trace (CASSANDRA-9723) + * Fix broken internode SSL (CASSANDRA-9884) +Merged from 2.1: + * Add new JMX methods to change local compaction strategy (CASSANDRA-9965) * Fix handling of enable/disable autocompaction (CASSANDRA-9899) * Add consistency level to tracing ouput (CASSANDRA-9827) * Remove repair snapshot leftover on startup (CASSANDRA-7357) http://git-wip-us.apache.org/repos/asf/cassandra/blob/136e7002/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 8d72ecf,343ecee..c7d12a2 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@@ -1174,17 -1389,58 +1174,51 @@@ public class ColumnFamilyStore implemen if (!sstables.iterator().hasNext()) return ImmutableSet.of(); - + View view = data.getView(); - Set<SSTableReader> results = null; - for (SSTableReader sstable : sstables) + List<SSTableReader> sortedByFirst = Lists.newArrayList(sstables); - Collections.sort(sortedByFirst, new Comparator<SSTableReader>() - { - @Override - public int compare(SSTableReader o1, SSTableReader o2) - { - return o1.first.compareTo(o2.first); - } - }); - List<Interval<RowPosition, SSTableReader>> intervals = new ArrayList<>(); ++ Collections.sort(sortedByFirst, (o1, o2) -> o1.first.compareTo(o2.first)); ++ ++ List<AbstractBounds<PartitionPosition>> bounds = new ArrayList<>(); + DecoratedKey first = null, last = null; + /* + normalize the intervals covered by the sstables + assume we have sstables like this (brackets representing first/last key in the sstable); + [ ] [ ] [ ] [ ] + [ ] [ ] + then we can, instead of searching the interval tree 6 times, normalize the intervals and + only query the tree 2 times, for these intervals; + [ ] [ ] + */ + for (SSTableReader sstable : sortedByFirst) { - Set<SSTableReader> overlaps = ImmutableSet.copyOf(view.sstablesInBounds(sstableSet, sstable.first, sstable.last)); - results = results == null ? overlaps : Sets.union(results, overlaps).immutableCopy(); + if (first == null) + { + first = sstable.first; + last = sstable.last; + } + else + { + if (sstable.first.compareTo(last) <= 0) // we do overlap + { + if (sstable.last.compareTo(last) > 0) + last = sstable.last; + } + else + { - intervals.add(Interval.<RowPosition, SSTableReader>create(first, last)); ++ bounds.add(AbstractBounds.bounds(first, true, last, true)); + first = sstable.first; + last = sstable.last; + } + } } - results = Sets.difference(results, ImmutableSet.copyOf(sstables)); - intervals.add(Interval.<RowPosition, SSTableReader>create(first, last)); - SSTableIntervalTree tree = data.getView().intervalTree; ++ bounds.add(AbstractBounds.bounds(first, true, last, true)); + Set<SSTableReader> results = new HashSet<>(); + - for (Interval<RowPosition, SSTableReader> interval : intervals) - results.addAll(tree.search(interval)); ++ for (AbstractBounds<PartitionPosition> bound : bounds) ++ Iterables.addAll(results, view.sstablesInBounds(sstableSet, bound.left, bound.right)); - return results; + return Sets.difference(results, ImmutableSet.copyOf(sstables)); } /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/136e7002/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java index 1d1faf5,c48713e..a908884 --- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java @@@ -82,13 -82,19 +85,19 @@@ public class DateTieredCompactionStrate */ private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore) { - if (cfs.getSSTables().isEmpty()) + if (Iterables.isEmpty(cfs.getSSTables(SSTableSet.LIVE))) return Collections.emptyList(); - Set<SSTableReader> uncompacting = Sets.intersection(sstables, cfs.getUncompactingSSTables()); + Set<SSTableReader> uncompacting = ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains)); - // Find fully expired SSTables. Those will be included no matter what. - Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(SSTableSet.CANONICAL, uncompacting), gcBefore); + Set<SSTableReader> expired = Collections.emptySet(); + // we only check for expired sstables every 10 minutes due to it being an expensive operation + if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10)) + { + // Find fully expired SSTables. Those will be included no matter what. - expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore); ++ expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(SSTableSet.CANONICAL, uncompacting), gcBefore); + lastExpiredCheck = System.currentTimeMillis(); + } Set<SSTableReader> candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting)); List<SSTableReader> compactionCandidates = new ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates, expired), gcBefore)); http://git-wip-us.apache.org/repos/asf/cassandra/blob/136e7002/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java ----------------------------------------------------------------------