Updated Branches: refs/heads/cassandra-2.0 b5c23cf74 -> d49303078 refs/heads/trunk 5832cc839 -> 2c7b61b76
add back ScrubTest patch by Tyler Hobbs; reviewed by jbellis for CASSANDRA-5730 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d4930307 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d4930307 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d4930307 Branch: refs/heads/cassandra-2.0 Commit: d49303078459b0b2e0d40d9f79660ccdc0dc1fe0 Parents: b5c23cf Author: Jonathan Ellis <jbel...@apache.org> Authored: Thu Sep 26 13:18:02 2013 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Thu Sep 26 13:19:04 2013 -0500 ---------------------------------------------------------------------- .../Keyspace1-Standard3-ja-1-CRC.db | Bin 8 -> 0 bytes .../Keyspace1-Standard3-ja-1-Data.db | Bin 354 -> 0 bytes .../Keyspace1-Standard3-ja-1-Digest.sha1 | 1 - .../Keyspace1-Standard3-ja-1-Filter.db | Bin 176 -> 0 bytes .../Keyspace1-Standard3-ja-1-Index.db | Bin 90 -> 0 bytes .../Keyspace1-Standard3-ja-1-Statistics.db | Bin 4377 -> 0 bytes .../Keyspace1-Standard3-ja-1-Summary.db | Bin 83 -> 0 bytes .../Keyspace1-Standard3-ja-1-TOC.txt | 8 - .../Keyspace1-Standard3-jb-1-CompressionInfo.db | Bin 0 -> 43 bytes .../Keyspace1-Standard3-jb-1-Data.db | Bin 0 -> 133 bytes .../Keyspace1-Standard3-jb-1-Filter.db | Bin 0 -> 24 bytes .../Keyspace1-Standard3-jb-1-Index.db | Bin 0 -> 90 bytes .../Keyspace1-Standard3-jb-1-Statistics.db | Bin 0 -> 4390 bytes .../Keyspace1-Standard3-jb-1-Summary.db | Bin 0 -> 71 bytes .../Keyspace1-Standard3-jb-1-TOC.txt | 7 + .../unit/org/apache/cassandra/db/ScrubTest.java | 206 +++++++++++++++++++ 16 files changed, 213 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db deleted file mode 100644 index 3cc23b0..0000000 Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db deleted file mode 100644 index 70e64e0..0000000 Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1 ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1 b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1 deleted file mode 100644 index c53d478..0000000 --- a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1 +++ /dev/null @@ -1 +0,0 @@ -a9fbab0c12f097cfbf91a7b8731a20363daef547 Keyspace1-Standard3-ja-1-Data.db \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db deleted file mode 100644 index df0734d..0000000 Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db deleted file mode 100644 index 5da2914..0000000 Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db deleted file mode 100644 index 28e250b..0000000 Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db deleted file mode 100644 index 105e42c..0000000 Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt deleted file mode 100644 index 39df2ce..0000000 --- a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt +++ /dev/null @@ -1,8 +0,0 @@ -Statistics.db -Digest.sha1 -Filter.db -Data.db -Index.db -Summary.db -CRC.db -TOC.txt http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db new file mode 100644 index 0000000..44d2e59 Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db new file mode 100644 index 0000000..f75c4e6 Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db new file mode 100644 index 0000000..8f0a999 Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db new file mode 100644 index 0000000..da84fbc Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db new file mode 100644 index 0000000..0762615 Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db new file mode 100644 index 0000000..376ca9d Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt new file mode 100644 index 0000000..cf6efa8 --- /dev/null +++ b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt @@ -0,0 +1,7 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/unit/org/apache/cassandra/db/ScrubTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java new file mode 100644 index 0000000..f09aab3 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@ -0,0 +1,206 @@ +package org.apache.cassandra.db; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.apache.cassandra.OrderedJUnit4ClassRunner; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.compaction.Scrubber; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.db.columniterator.IdentityQueryFilter; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.Util.column; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(OrderedJUnit4ClassRunner.class) +public class ScrubTest extends SchemaLoader +{ + public String KEYSPACE = "Keyspace1"; + public String CF = "Standard1"; + public String CF3 = "Standard2"; + + @Test + public void testScrubOneRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException + { + CompactionManager.instance.disableAutoCompaction(); + Keyspace keyspace = Keyspace.open(KEYSPACE); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); + + List<Row> rows; + + // insert data and verify we get it back w/ range query + fillCF(cfs, 1); + rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + assertEquals(1, rows.size()); + + CompactionManager.instance.performScrub(cfs); + + // check data is still there + rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + assertEquals(1, rows.size()); + } + + @Test + public void testScrubDeletedRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException + { + CompactionManager.instance.disableAutoCompaction(); + Keyspace keyspace = Keyspace.open(KEYSPACE); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF3); + + ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(KEYSPACE, CF3); + cf.delete(new DeletionInfo(0, 1)); // expired tombstone + RowMutation rm = new RowMutation(KEYSPACE, ByteBufferUtil.bytes(1), cf); + rm.applyUnsafe(); + cfs.forceBlockingFlush(); + + CompactionManager.instance.performScrub(cfs); + assert cfs.getSSTables().isEmpty(); + } + + @Test + public void testScrubMultiRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException + { + CompactionManager.instance.disableAutoCompaction(); + Keyspace keyspace = Keyspace.open(KEYSPACE); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); + + List<Row> rows; + + // insert data and verify we get it back w/ range query + fillCF(cfs, 10); + rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + assertEquals(10, rows.size()); + + CompactionManager.instance.performScrub(cfs); + + // check data is still there + rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + assertEquals(10, rows.size()); + } + + @Test + public void testScrubOutOfOrder() throws Exception + { + CompactionManager.instance.disableAutoCompaction(); + Keyspace keyspace = Keyspace.open(KEYSPACE); + String columnFamily = "Standard3"; + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily); + + /* + * Code used to generate an outOfOrder sstable. The test for out-of-order key in SSTableWriter must also be commented out. + * The test also assumes an ordered partitioner. + * + ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.metadata); + cf.addColumn(new Column(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"), 0L)); + + SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))), + cfs.metadata.getIndexInterval(), + cfs.metadata, + cfs.partitioner, + SSTableMetadata.createCollector(BytesType.instance)); + writer.append(Util.dk("a"), cf); + writer.append(Util.dk("b"), cf); + writer.append(Util.dk("z"), cf); + writer.append(Util.dk("c"), cf); + writer.append(Util.dk("y"), cf); + writer.append(Util.dk("d"), cf); + writer.closeAndOpenReader(); + */ + + + String root = System.getProperty("corrupt-sstable-root"); + assert root != null; + File rootDir = new File(root); + assert rootDir.isDirectory(); + Descriptor desc = new Descriptor(new Descriptor.Version("jb"), rootDir, KEYSPACE, columnFamily, 1, false); + CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname); + + try + { + SSTableReader.open(desc, metadata); + fail("SSTR validation should have caught the out-of-order rows"); + } + catch (IllegalStateException ise) { /* this is expected */ } + + // open without validation for scrubbing + Set<Component> components = new HashSet<>(); + components.add(Component.COMPRESSION_INFO); + components.add(Component.DATA); + components.add(Component.PRIMARY_INDEX); + components.add(Component.FILTER); + components.add(Component.STATS); + components.add(Component.SUMMARY); + components.add(Component.TOC); + SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata); + + Scrubber scrubber = new Scrubber(cfs, sstable); + scrubber.scrub(); + + cfs.loadNewSSTables(); + List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + assert isRowOrdered(rows) : "Scrub failed: " + rows; + assert rows.size() == 6 : "Got " + rows.size(); + } + + private static boolean isRowOrdered(List<Row> rows) + { + DecoratedKey prev = null; + for (Row row : rows) + { + if (prev != null && prev.compareTo(row.key) > 0) + return false; + prev = row.key; + } + return true; + } + + protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) throws ExecutionException, InterruptedException, IOException + { + for (int i = 0; i < rowsPerSSTable; i++) + { + String key = String.valueOf(i); + // create a row and update the birthdate value, test that the index query fetches the new version + ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(KEYSPACE, CF); + cf.addColumn(column("c1", "1", 1L)); + cf.addColumn(column("c2", "2", 1L)); + RowMutation rm = new RowMutation(KEYSPACE, ByteBufferUtil.bytes(key), cf); + rm.applyUnsafe(); + } + + cfs.forceBlockingFlush(); + } +}