Scrub (recover) sstables even when -Index.db is missing patch by mck; reviewed by stefania for CASSANDRA-9591
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/452d6a44 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/452d6a44 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/452d6a44 Branch: refs/heads/cassandra-2.2 Commit: 452d6a445a6935d3e7d0e0fdf59e87d2a2ff95e7 Parents: 62714a9 Author: Stefania Alborghetti <stefania.alborghe...@datastax.com> Authored: Mon Jun 15 16:49:03 2015 +0800 Committer: Benedict Elliott Smith <bened...@apache.org> Committed: Tue Jul 7 14:27:25 2015 +0100 ---------------------------------------------------------------------- CHANGES.txt | 4 ++ .../cassandra/db/compaction/Scrubber.java | 36 +++++++++++---- .../cassandra/io/sstable/SSTableReader.java | 47 ++++++++++++++++---- .../cassandra/tools/StandaloneScrubber.java | 2 +- .../unit/org/apache/cassandra/db/ScrubTest.java | 25 +++++++++++ 5 files changed, 96 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/452d6a44/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index ca4d4b5..bd1db92 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,7 @@ +2.0.18 +* Scrub (recover) sstables even when -Index.db is missing, (CASSANDRA-9591) + + 2.0.17 * Avoid NPE in AuthSuccess#decode (CASSANDRA-9727) * Add listen_address to system.local (CASSANDRA-9603) http://git-wip-us.apache.org/repos/asf/cassandra/blob/452d6a44/src/java/org/apache/cassandra/db/compaction/Scrubber.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java index ea10855..dc60efa 100644 --- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java +++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java @@ -96,7 +96,16 @@ public class Scrubber implements Closeable ? new ScrubController(cfs) : new CompactionController(cfs, Collections.singleton(sstable), CompactionManager.getDefaultGcBefore(cfs)); this.isCommutative = cfs.metadata.getDefaultValidator().isCommutative(); - this.expectedBloomFilterSize = Math.max(cfs.metadata.getIndexInterval(), (int)(SSTableReader.getApproximateKeyCount(toScrub,cfs.metadata))); + + boolean hasIndexFile = (new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))).exists(); + if (!hasIndexFile) + { + // if there's any corruption in the -Data.db then rows can't be skipped over. but it's worth a shot. + outputHandler.warn("Missing component: " + sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)); + } + + this.expectedBloomFilterSize = Math.max(cfs.metadata.getIndexInterval(), + hasIndexFile ? (int)(SSTableReader.getApproximateKeyCount(toScrub,cfs.metadata)) : 0); // loop through each row, deserializing to check for damage. // we'll also loop through the index at the same time, using the position from the index to recover if the @@ -105,7 +114,11 @@ public class Scrubber implements Closeable this.dataFile = isOffline ? sstable.openDataReader() : sstable.openDataReader(CompactionManager.instance.getRateLimiter()); - this.indexFile = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))); + + this.indexFile = hasIndexFile + ? RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))) + : null; + this.scrubInfo = new ScrubInfo(dataFile, sstable); this.currentRowPositionFromIndex = 0; @@ -117,7 +130,8 @@ public class Scrubber implements Closeable outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length())); try { - nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile); + nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null; + if (indexAvailable()) { // throw away variable so we don't have a side effect in the assert long firstRowPositionFromIndex = RowIndexEntry.serializer.deserialize(indexFile, sstable.descriptor.version).position; @@ -181,7 +195,7 @@ public class Scrubber implements Closeable outputHandler.debug(String.format("Index doublecheck: row %s is %s bytes", ByteBufferUtil.bytesToHex(currentIndexKey), dataSizeFromIndex)); } - assert currentIndexKey != null || indexFile.isEOF(); + assert currentIndexKey != null || !indexAvailable(); writer.mark(); try @@ -198,10 +212,10 @@ public class Scrubber implements Closeable if (dataSize > dataFile.length()) throw new IOError(new IOException("Impossible row size (greater than file length): " + dataSize)); - if (dataStart != dataStartFromIndex) + if (indexFile != null && dataStart != dataStartFromIndex) outputHandler.warn(String.format("Data file row position %d differs from index file row position %d", dataStart, dataStartFromIndex)); - if (dataSize != dataSizeFromIndex) + if (indexFile != null && dataSize != dataSizeFromIndex) outputHandler.warn(String.format("Data file row size %d differs from index file row size %d", dataSize, dataSizeFromIndex)); SSTableIdentityIterator atoms = new SSTableIdentityIterator(sstable, dataFile, key, dataSize, validateColumns); @@ -317,8 +331,9 @@ public class Scrubber implements Closeable currentRowPositionFromIndex = nextRowPositionFromIndex; try { - nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile); - nextRowPositionFromIndex = indexFile.isEOF() + nextIndexKey = !indexAvailable() ? null : ByteBufferUtil.readWithShortLength(indexFile); + + nextRowPositionFromIndex = !indexAvailable() ? dataFile.length() : RowIndexEntry.serializer.deserialize(indexFile, sstable.descriptor.version).position; } @@ -330,6 +345,11 @@ public class Scrubber implements Closeable } } + private boolean indexAvailable() + { + return indexFile != null && !indexFile.isEOF(); + } + private void seekToNextRow() { while(nextRowPositionFromIndex < dataFile.length()) http://git-wip-us.apache.org/repos/asf/cassandra/blob/452d6a44/src/java/org/apache/cassandra/io/sstable/SSTableReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java index 15808e8..8919a09 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java @@ -157,7 +157,7 @@ public class SSTableReader extends SSTable implements Closeable public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, CFMetaData metadata, IPartitioner partitioner) throws IOException { - SSTableMetadata sstableMetadata = openMetadata(descriptor, components, partitioner); + SSTableMetadata sstableMetadata = openMetadata(descriptor, components, partitioner, true); SSTableReader sstable = new SSTableReader(descriptor, components, metadata, @@ -191,7 +191,7 @@ public class SSTableReader extends SSTable implements Closeable boolean validate) throws IOException { long start = System.nanoTime(); - SSTableMetadata sstableMetadata = openMetadata(descriptor, components, partitioner); + SSTableMetadata sstableMetadata = openMetadata(descriptor, components, partitioner, validate); SSTableReader sstable = new SSTableReader(descriptor, components, @@ -213,12 +213,17 @@ public class SSTableReader extends SSTable implements Closeable return sstable; } - private static SSTableMetadata openMetadata(Descriptor descriptor, Set<Component> components, IPartitioner partitioner) throws IOException + private static SSTableMetadata openMetadata(Descriptor descriptor, + Set<Component> components, + IPartitioner partitioner, + boolean primaryIndexRequired) throws IOException { assert partitioner != null; // Minimum components without which we can't do anything assert components.contains(Component.DATA) : "Data component is missing for sstable" + descriptor; - assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor; + + assert !primaryIndexRequired || components.contains(Component.PRIMARY_INDEX) + : "Primary index component is missing for sstable " + descriptor; logger.info("Opening {} ({} bytes)", descriptor, new File(descriptor.filenameFor(COMPONENT_DATA)).length()); @@ -382,11 +387,17 @@ public class SSTableReader extends SSTable implements Closeable readMeterSyncFuture.cancel(false); // Force finalizing mmapping if necessary - ifile.cleanup(); + + if (null != ifile) + ifile.cleanup(); + dfile.cleanup(); // close the BF so it can be opened later. - bf.close(); - indexSummary.close(); + if (null != bf) + bf.close(); + + if (null != indexSummary) + indexSummary.close(); } public void setTrackedBy(DataTracker tracker) @@ -406,6 +417,12 @@ public class SSTableReader extends SSTable implements Closeable load(false, true); bf = FilterFactory.AlwaysPresent; } + else if (!components.contains(Component.PRIMARY_INDEX)) + { + // avoid any reading of the missing primary index component. + // this should only happen during StandaloneScrubber + load(false, false); + } else if (!components.contains(Component.FILTER)) { // bf is enabled, but filter component is missing. @@ -454,7 +471,9 @@ public class SSTableReader extends SSTable implements Closeable if (recreateBloomFilter || !summaryLoaded) buildSummary(recreateBloomFilter, ibuilder, dbuilder, summaryLoaded); - ifile = ibuilder.complete(descriptor.filenameFor(Component.PRIMARY_INDEX)); + if (components.contains(Component.PRIMARY_INDEX)) + ifile = ibuilder.complete(descriptor.filenameFor(Component.PRIMARY_INDEX)); + dfile = dbuilder.complete(descriptor.filenameFor(Component.DATA)); if (saveSummaryIfCreated && (recreateBloomFilter || !summaryLoaded)) // save summary information to disk saveSummary(this, ibuilder, dbuilder); @@ -462,6 +481,9 @@ public class SSTableReader extends SSTable implements Closeable private void buildSummary(boolean recreateBloomFilter, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, boolean summaryLoaded) throws IOException { + if (!components.contains(Component.PRIMARY_INDEX)) + return; + // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary. RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))); @@ -965,6 +987,9 @@ public class SSTableReader extends SSTable implements Closeable } } + if (ifile == null) + return null; + // scan the on-disk index, starting at the nearest sampled position. // The check against IndexInterval is to be exit the loop in the EQ case when the key looked for is not present // (bloom filter false positive). But note that for non-EQ cases, we might need to check the first key of the @@ -1063,6 +1088,9 @@ public class SSTableReader extends SSTable implements Closeable if (sampledPosition == -1) sampledPosition = 0; + if (ifile == null) + return null; + Iterator<FileDataInput> segments = ifile.iterator(sampledPosition); while (segments.hasNext()) { @@ -1480,7 +1508,8 @@ public class SSTableReader extends SSTable implements Closeable private void dropPageCache() { dropPageCache(dfile.path); - dropPageCache(ifile.path); + if (null != ifile) + dropPageCache(ifile.path); } private void dropPageCache(String filePath) http://git-wip-us.apache.org/repos/asf/cassandra/blob/452d6a44/src/java/org/apache/cassandra/tools/StandaloneScrubber.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index 26768f3..8ace228 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -79,7 +79,7 @@ public class StandaloneScrubber for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet()) { Set<Component> components = entry.getValue(); - if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX)) + if (!components.contains(Component.DATA)) continue; try http://git-wip-us.apache.org/repos/asf/cassandra/blob/452d6a44/test/unit/org/apache/cassandra/db/ScrubTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java index 94c7e34..978b3e8 100644 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@ -270,6 +270,31 @@ public class ScrubTest extends SchemaLoader } @Test + public void testScrubNoIndex() throws IOException, ExecutionException, InterruptedException, ConfigurationException + { + CompactionManager.instance.disableAutoCompaction(); + Keyspace keyspace = Keyspace.open(KEYSPACE); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); + cfs.clearUnsafe(); + + List<Row> rows; + + // insert data and verify we get it back w/ range query + fillCF(cfs, 10); + rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + assertEquals(10, rows.size()); + + for (SSTableReader sstable : cfs.getSSTables()) + new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)).delete(); + + CompactionManager.instance.performScrub(cfs, false, true); + + // check data is still there + rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + assertEquals(10, rows.size()); + } + + @Test public void testScrubOutOfOrder() throws Exception { CompactionManager.instance.disableAutoCompaction();
