[2/2] git commit: Ensure compacted sstables are never used
Ensure compacted sstables are never used
patch by slebresne; reviewed by jbellis for CASSANDRA-4436
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6a6b7ec1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6a6b7ec1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6a6b7ec1
Branch: refs/heads/trunk
Commit: 6a6b7ec1f00d8084fb29379111a798c09c35e6d6
Parents: cc0be1b
Author: Sylvain Lebresne
Authored: Wed Jul 18 19:54:20 2012 +0200
Committer: Sylvain Lebresne
Committed: Thu Jul 26 17:46:19 2012 +0200
--
CHANGES.txt|1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 32 ++--
.../org/apache/cassandra/io/sstable/Component.java |2 +
.../apache/cassandra/io/sstable/Descriptor.java| 10 +++-
.../org/apache/cassandra/io/sstable/SSTable.java |2 +
.../cassandra/io/sstable/SSTableMetadata.java | 35 +
.../apache/cassandra/io/sstable/SSTableReader.java | 39 +--
7 files changed, 80 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6a6b7ec1/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 169f66d..84db73d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -19,6 +19,7 @@
* fix "Can't Modify Index Name" problem on CF update (CASSANDRA-4439)
* Fix assertion error in getOverlappingSSTables during repair (CASSANDRA-4456)
* fix nodetool's setcompactionthreshold command (CASSANDRA-4455)
+ * Ensure compacted files are never used, to avoid counter overcount
(CASSANDRA-4436)
Merged from 1.0:
* allow dropping columns shadowed by not-yet-expired supercolumn or row
tombstones in PrecompactedRow (CASSANDRA-4396)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6a6b7ec1/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
--
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index b93adc1..a39530a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -234,8 +234,23 @@ public class ColumnFamilyStore implements
ColumnFamilyStoreMBean
if (loadSSTables)
{
-Directories.SSTableLister sstables =
directories.sstableLister().skipCompacted(true).skipTemporary(true);
-
data.addInitialSSTables(SSTableReader.batchOpen(sstables.list().entrySet(),
savedKeys, data, metadata, this.partitioner));
+Directories.SSTableLister sstableFiles =
directories.sstableLister().skipCompacted(true).skipTemporary(true);
+Collection sstables =
SSTableReader.batchOpen(sstableFiles.list().entrySet(), savedKeys, data,
metadata, this.partitioner);
+
+// Filter non-compacted sstables, remove compacted ones
+Set compactedSSTables = new HashSet();
+for (SSTableReader sstable : sstables)
+compactedSSTables.addAll(sstable.getAncestors());
+
+Set liveSSTables = new HashSet();
+for (SSTableReader sstable : sstables)
+{
+if (compactedSSTables.contains(sstable.descriptor.generation))
+sstable.releaseReference(); // this amount to deleting the
sstable
+else
+liveSSTables.add(sstable);
+}
+data.addInitialSSTables(liveSSTables);
}
// compaction strategy should be created after the CFS has been
prepared
@@ -492,7 +507,7 @@ public class ColumnFamilyStore implements
ColumnFamilyStoreMBean
SSTableReader reader;
try
{
-reader = SSTableReader.open(newDescriptor, entry.getValue(),
Collections.emptySet(), data, metadata, partitioner);
+reader = SSTableReader.open(newDescriptor, entry.getValue(),
Collections.emptySet(), metadata, partitioner);
}
catch (IOException e)
{
@@ -1969,9 +1984,18 @@ public class ColumnFamilyStore implements
ColumnFamilyStoreMBean
ReplayPosition rp = ReplayPosition.getReplayPosition(sstables);
SSTableMetadata.Collector sstableMetadataCollector =
SSTableMetadata.createCollector().replayPosition(rp);
-// get the max timestamp of the precompacted sstables
+// Get the max timestamp of the precompacted sstables
+// and adds generation of live ancestors
for (SSTableReader sstable : sstables)
+{
sstableMetadataCollector.updateMaxTimestamp(sstable.getMaxTimestamp());
+
sstableMetadataCollector.addAnces
git commit: Ensure compacted sstables are never used
Updated Branches:
refs/heads/cassandra-1.1 cc0be1b40 -> 6a6b7ec1f
Ensure compacted sstables are never used
patch by slebresne; reviewed by jbellis for CASSANDRA-4436
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6a6b7ec1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6a6b7ec1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6a6b7ec1
Branch: refs/heads/cassandra-1.1
Commit: 6a6b7ec1f00d8084fb29379111a798c09c35e6d6
Parents: cc0be1b
Author: Sylvain Lebresne
Authored: Wed Jul 18 19:54:20 2012 +0200
Committer: Sylvain Lebresne
Committed: Thu Jul 26 17:46:19 2012 +0200
--
CHANGES.txt|1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 32 ++--
.../org/apache/cassandra/io/sstable/Component.java |2 +
.../apache/cassandra/io/sstable/Descriptor.java| 10 +++-
.../org/apache/cassandra/io/sstable/SSTable.java |2 +
.../cassandra/io/sstable/SSTableMetadata.java | 35 +
.../apache/cassandra/io/sstable/SSTableReader.java | 39 +--
7 files changed, 80 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6a6b7ec1/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 169f66d..84db73d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -19,6 +19,7 @@
* fix "Can't Modify Index Name" problem on CF update (CASSANDRA-4439)
* Fix assertion error in getOverlappingSSTables during repair (CASSANDRA-4456)
* fix nodetool's setcompactionthreshold command (CASSANDRA-4455)
+ * Ensure compacted files are never used, to avoid counter overcount
(CASSANDRA-4436)
Merged from 1.0:
* allow dropping columns shadowed by not-yet-expired supercolumn or row
tombstones in PrecompactedRow (CASSANDRA-4396)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6a6b7ec1/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
--
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index b93adc1..a39530a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -234,8 +234,23 @@ public class ColumnFamilyStore implements
ColumnFamilyStoreMBean
if (loadSSTables)
{
-Directories.SSTableLister sstables =
directories.sstableLister().skipCompacted(true).skipTemporary(true);
-
data.addInitialSSTables(SSTableReader.batchOpen(sstables.list().entrySet(),
savedKeys, data, metadata, this.partitioner));
+Directories.SSTableLister sstableFiles =
directories.sstableLister().skipCompacted(true).skipTemporary(true);
+Collection sstables =
SSTableReader.batchOpen(sstableFiles.list().entrySet(), savedKeys, data,
metadata, this.partitioner);
+
+// Filter non-compacted sstables, remove compacted ones
+Set compactedSSTables = new HashSet();
+for (SSTableReader sstable : sstables)
+compactedSSTables.addAll(sstable.getAncestors());
+
+Set liveSSTables = new HashSet();
+for (SSTableReader sstable : sstables)
+{
+if (compactedSSTables.contains(sstable.descriptor.generation))
+sstable.releaseReference(); // this amount to deleting the
sstable
+else
+liveSSTables.add(sstable);
+}
+data.addInitialSSTables(liveSSTables);
}
// compaction strategy should be created after the CFS has been
prepared
@@ -492,7 +507,7 @@ public class ColumnFamilyStore implements
ColumnFamilyStoreMBean
SSTableReader reader;
try
{
-reader = SSTableReader.open(newDescriptor, entry.getValue(),
Collections.emptySet(), data, metadata, partitioner);
+reader = SSTableReader.open(newDescriptor, entry.getValue(),
Collections.emptySet(), metadata, partitioner);
}
catch (IOException e)
{
@@ -1969,9 +1984,18 @@ public class ColumnFamilyStore implements
ColumnFamilyStoreMBean
ReplayPosition rp = ReplayPosition.getReplayPosition(sstables);
SSTableMetadata.Collector sstableMetadataCollector =
SSTableMetadata.createCollector().replayPosition(rp);
-// get the max timestamp of the precompacted sstables
+// Get the max timestamp of the precompacted sstables
+// and adds generation of live ancestors
for (SSTableReader sstable : sstables)
+{
sstableMetadataCollector.updateMaxTimesta
