Fix "Unable to create hard link" SSTableReaderTest error messages
patch by jbellis and scode; reviewed by slebresne for CASSANDRA-3735 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/099f04e6 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/099f04e6 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/099f04e6 Branch: refs/heads/cassandra-1.1 Commit: 099f04e609487a4e01b2e21758593a820869f8d9 Parents: e6b3cbb Author: Sylvain Lebresne <sylv...@datastax.com> Authored: Mon Feb 6 16:05:23 2012 +0100 Committer: Sylvain Lebresne <sylv...@datastax.com> Committed: Mon Feb 6 17:04:07 2012 +0100 ---------------------------------------------------------------------- .../org/apache/cassandra/db/ColumnFamilyStore.java | 25 ++++++++++++- src/java/org/apache/cassandra/db/Directories.java | 13 ++++++- .../cassandra/io/sstable/SSTableReaderTest.java | 29 +++++++-------- 3 files changed, 49 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/099f04e6/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 65d7c60..33112c6 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -495,8 +495,18 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean SSTableReader.releaseReferences(sstables); } - logger.info("Setting up new generation: " + generation); - fileIndexGenerator.set(generation); + if (fileIndexGenerator.get() < generation) + { + // we don't bother with CAS here since if the generations used in the new files overlap with + // files that we create during load, we're already screwed + logger.info("Setting up new generation: " + generation); + fileIndexGenerator.set(generation); + } + else + { + logger.warn("Largest generation seen in loaded sstables was {}, which may overlap with native sstable files (generation {}).", + generation, fileIndexGenerator.get()); + } logger.info("Done loading load new SSTables for " + table.name + "/" + columnFamily); } @@ -1519,6 +1529,17 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean */ public void clearUnsafe() { + fileIndexGenerator.set(0); // Avoid unit test failures (see CASSANDRA-3735). + + // Clear backups + Directories.SSTableLister lister = directories.sstableLister().onlyBackups(true); + for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet()) + { + Descriptor desc = entry.getKey(); + for (Component comp : entry.getValue()) + FileUtils.delete(desc.filenameFor(comp)); + } + for (ColumnFamilyStore cfs : concatWithIndexes()) cfs.data.init(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/099f04e6/src/java/org/apache/cassandra/db/Directories.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java index e514fd9..2afefd2 100644 --- a/src/java/org/apache/cassandra/db/Directories.java +++ b/src/java/org/apache/cassandra/db/Directories.java @@ -181,6 +181,7 @@ public class Directories private boolean skipCompacted; private boolean skipTemporary; private boolean includeBackups; + private boolean onlyBackups; private int nbFiles; private final Map<Descriptor, Set<Component>> components = new HashMap<Descriptor, Set<Component>>(); private boolean filtered; @@ -209,6 +210,15 @@ public class Directories return this; } + public SSTableLister onlyBackups(boolean b) + { + if (filtered) + throw new IllegalStateException("list() has already been called"); + onlyBackups = b; + includeBackups = b; + return this; + } + public Map<Descriptor, Set<Component>> list() { filter(); @@ -236,7 +246,8 @@ public class Directories for (File location : sstableDirectories) { - location.listFiles(getFilter()); + if (!onlyBackups) + location.listFiles(getFilter()); if (includeBackups) new File(location, BACKUPS_SUBDIR).listFiles(getFilter()); http://git-wip-us.apache.org/repos/asf/cassandra/blob/099f04e6/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java index b801f4e..bcb761b 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java @@ -24,14 +24,17 @@ package org.apache.cassandra.io.sstable; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.concurrent.ExecutionException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; import org.junit.Test; import org.apache.cassandra.CleanupHelper; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; import org.apache.cassandra.db.columniterator.IdentityQueryFilter; @@ -51,10 +54,6 @@ import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CLibrary; import org.apache.cassandra.utils.Pair; -import org.apache.cassandra.Util; - -import static org.junit.Assert.assertEquals; - public class SSTableReaderTest extends CleanupHelper { static Token t(int i) @@ -157,11 +156,16 @@ public class SSTableReaderTest extends CleanupHelper } store.forceBlockingFlush(); - store.clearUnsafe(); - store.loadNewSSTables(); + clearAndLoad(store); assert store.getMaxRowSize() != 0; } + private void clearAndLoad(ColumnFamilyStore cfs) throws IOException + { + cfs.clearUnsafe(); + cfs.loadNewSSTables(); + } + @Test public void testGetPositionsForRangesWithKeyCache() throws IOException, ExecutionException, InterruptedException { @@ -240,18 +244,13 @@ public class SSTableReaderTest extends CleanupHelper assertIndexQueryWorks(store); } - private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) + private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) throws IOException { assert "Indexed1".equals(indexedCFS.getColumnFamilyName()); // make sure all sstables including 2ary indexes load from disk - indexedCFS.clearUnsafe(); - for (ColumnFamilyStore indexCfs : indexedCFS.indexManager.getIndexesBackedByCfs()) - { - indexCfs.clearUnsafe(); - indexCfs.loadNewSSTables(); // v1.0.4 would fail here (see CASSANDRA-3540) - } - indexedCFS.loadNewSSTables(); + for (ColumnFamilyStore cfs : indexedCFS.concatWithIndexes()) + clearAndLoad(cfs); // query using index to see if sstable for secondary index opens IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), IndexOperator.EQ, ByteBufferUtil.bytes(1L));
