Repository: cassandra Updated Branches: refs/heads/trunk e6bc6385a -> 3510bdb50
Fix various unit-test failures Patch by jmckenzie; reviewed by branimir for CASSANDRA-9104 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3510bdb5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3510bdb5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3510bdb5 Branch: refs/heads/trunk Commit: 3510bdb503c2a9d97cb12fd7d9e1a01fd91fa879 Parents: e6bc638 Author: Joshua McKenzie <jmcken...@apache.org> Authored: Wed Apr 22 16:42:59 2015 -0500 Committer: Joshua McKenzie <jmcken...@apache.org> Committed: Wed Apr 22 16:42:59 2015 -0500 ---------------------------------------------------------------------- .../apache/cassandra/config/DatabaseDescriptor.java | 2 +- .../apache/cassandra/db/commitlog/CommitLog.java | 9 ++++++++- .../db/commitlog/CommitLogSegmentManager.java | 4 ++-- .../cassandra/io/sstable/SSTableRewriter.java | 12 ++++++++++-- .../apache/cassandra/db/ColumnFamilyStoreTest.java | 4 +++- .../unit/org/apache/cassandra/db/CommitLogTest.java | 4 ++-- test/unit/org/apache/cassandra/db/KeyCacheTest.java | 10 +++++++--- .../apache/cassandra/db/RecoveryManager2Test.java | 3 +-- test/unit/org/apache/cassandra/db/ScrubTest.java | 16 ++++++++++++++++ 9 files changed, 50 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 76ef4c5..68d7e82 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -1479,7 +1479,7 @@ public class DatabaseDescriptor public static int getSSTablePreempiveOpenIntervalInMB() { - return conf.sstable_preemptive_open_interval_in_mb; + return FBUtilities.isWindows() ? -1 : conf.sstable_preemptive_open_interval_in_mb; } public static boolean getTrickleFsync() http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/src/java/org/apache/cassandra/db/commitlog/CommitLog.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java index cf38d44..b1670ac 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java @@ -119,6 +119,11 @@ public class CommitLog implements CommitLogMBean */ public int recover() throws IOException { + // Allocator could be in the process of initial startup with 0 active and available segments. We need to wait for + // the allocation manager to finish allocation and add it to available segments so we don't get an invalid response + // on allocator.manages(...) below by grabbing a file off the filesystem before it's added to the CLQ. + allocator.allocatingFrom(); + FilenameFilter unmanagedFilesFilter = new FilenameFilter() { public boolean accept(File dir, String name) @@ -372,11 +377,13 @@ public class CommitLog implements CommitLogMBean /** * FOR TESTING PURPOSES. See CommitLogAllocator. + * @return the number of files recovered */ - public void resetUnsafe(boolean deleteSegments) + public int resetUnsafe(boolean deleteSegments) throws IOException { stopUnsafe(deleteSegments); startUnsafe(); + return recover(); } /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java index af415a2..ac36d34 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java @@ -466,6 +466,7 @@ public class CommitLogSegmentManager public void stopUnsafe(boolean deleteSegments) { logger.debug("CLSM closing and clearing existing commit log segments..."); + createReserveSegments = false; while (!segmentManagementTasks.isEmpty()) Thread.yield(); @@ -509,7 +510,7 @@ public class CommitLogSegmentManager } catch (AssertionError ignored) { - // segment file does not exit + // segment file does not exist } } @@ -519,7 +520,6 @@ public class CommitLogSegmentManager public void startUnsafe() { start(); - wakeManager(); } /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java index 618b1e4..8890659 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java @@ -57,16 +57,21 @@ public class SSTableRewriter static { long interval = DatabaseDescriptor.getSSTablePreempiveOpenIntervalInMB() * (1L << 20); - if (interval < 0 || FBUtilities.isWindows()) + if (interval < 0) interval = Long.MAX_VALUE; preemptiveOpenInterval = interval; } @VisibleForTesting - static void overrideOpenInterval(long size) + public static void overrideOpenInterval(long size) { preemptiveOpenInterval = size; } + @VisibleForTesting + public static long getOpenInterval() + { + return preemptiveOpenInterval; + } private final DataTracker dataTracker; private final ColumnFamilyStore cfs; @@ -280,6 +285,9 @@ public class SSTableRewriter { if (isOffline) return; + if (preemptiveOpenInterval == Long.MAX_VALUE) + return; + List<SSTableReader> toReplace = new ArrayList<>(); List<SSTableReader> replaceWith = new ArrayList<>(); final List<DecoratedKey> invalidateKeys = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 7d15c44..0c221ac 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -115,6 +115,7 @@ public class ColumnFamilyStoreTest public static final String CF_STANDARD2 = "Standard2"; public static final String CF_STANDARD3 = "Standard3"; public static final String CF_STANDARD4 = "Standard4"; + public static final String CF_STANDARD5 = "Standard5"; public static final String CF_STANDARDINT = "StandardInteger1"; public static final String CF_SUPER1 = "Super1"; public static final String CF_SUPER6 = "Super6"; @@ -142,6 +143,7 @@ public class ColumnFamilyStoreTest SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD5), SchemaLoader.indexCFMD(KEYSPACE1, CF_INDEX1, true), SchemaLoader.indexCFMD(KEYSPACE1, CF_INDEX2, false), SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, LongType.instance), @@ -1874,7 +1876,7 @@ public class ColumnFamilyStoreTest public void testLoadNewSSTablesAvoidsOverwrites() throws Throwable { String ks = KEYSPACE1; - String cf = CF_STANDARD1; + String cf = CF_STANDARD5; ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore(cf); cfs.truncateBlocking(); SSTableDeletingTask.waitForDeletions(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/test/unit/org/apache/cassandra/db/CommitLogTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/CommitLogTest.java b/test/unit/org/apache/cassandra/db/CommitLogTest.java index 683d3f9..4db6057 100644 --- a/test/unit/org/apache/cassandra/db/CommitLogTest.java +++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java @@ -311,7 +311,7 @@ public class CommitLogTest } @Test - public void testTruncateWithoutSnapshot() + public void testTruncateWithoutSnapshot() throws IOException { CommitLog.instance.resetUnsafe(true); boolean prev = DatabaseDescriptor.isAutoSnapshot(); @@ -340,7 +340,7 @@ public class CommitLogTest } @Test - public void testTruncateWithoutSnapshotNonDurable() + public void testTruncateWithoutSnapshotNonDurable() throws IOException { CommitLog.instance.resetUnsafe(true); boolean prevAutoSnapshot = DatabaseDescriptor.isAutoSnapshot(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/test/unit/org/apache/cassandra/db/KeyCacheTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java index 9695b4a..e5fd470 100644 --- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java +++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java @@ -25,7 +25,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import com.google.common.util.concurrent.Uninterruptibles; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -33,15 +32,18 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.db.composites.*; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Refs; import static org.junit.Assert.assertEquals; @@ -175,10 +177,12 @@ public class KeyCacheTest throw new IllegalStateException(); Util.compactAll(cfs, Integer.MAX_VALUE).get(); + boolean noEarlyOpen = DatabaseDescriptor.getSSTablePreempiveOpenIntervalInMB() < 0; + // after compaction cache should have entries for new SSTables, // but since we have kept a reference to the old sstables, // if we had 2 keys in cache previously it should become 4 - assertKeyCacheSize(4, KEYSPACE1, COLUMN_FAMILY1); + assertKeyCacheSize(noEarlyOpen ? 2 : 4, KEYSPACE1, COLUMN_FAMILY1); refs.release(); @@ -205,7 +209,7 @@ public class KeyCacheTest 10, System.currentTimeMillis())); - assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1); + assertKeyCacheSize(noEarlyOpen ? 4 : 2, KEYSPACE1, COLUMN_FAMILY1); } private void assertKeyCacheSize(int expected, String keyspace, String columnFamily) http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java b/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java index 772c827..13c3452 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java @@ -82,8 +82,7 @@ public class RecoveryManager2Test logger.debug("begin manual replay"); // replay the commit log (nothing on Standard1 should be replayed since everything was flushed, so only the row on Standard2 // will be replayed) - CommitLog.instance.resetUnsafe(false); - int replayed = CommitLog.instance.recover(); + int replayed = CommitLog.instance.resetUnsafe(false); assert replayed == 1 : "Expecting only 1 replayed mutation, got " + replayed; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/3510bdb5/test/unit/org/apache/cassandra/db/ScrubTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java index f91ba64..dce8d14 100644 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@ -54,6 +54,7 @@ import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableRewriter; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; @@ -169,6 +170,21 @@ public class ScrubTest } @Test + public void testScrubCorruptedCounterRowNoEarlyOpen() throws IOException, WriteTimeoutException + { + long oldOpenVal = SSTableRewriter.getOpenInterval(); + try + { + SSTableRewriter.overrideOpenInterval(Long.MAX_VALUE); + testScrubCorruptedCounterRow(); + } + finally + { + SSTableRewriter.overrideOpenInterval(oldOpenVal); + } + } + + @Test public void testScrubDeletedRow() throws ExecutionException, InterruptedException { CompactionManager.instance.disableAutoCompaction();
