This is an automated email from the ASF dual-hosted git repository. marcuse pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push: new 5463310 Make it possible to disable STCS-in-L0 during runtime 5463310 is described below commit 546331037ee8218208b61a26ecbc12ab8dd7eace Author: Marcus Eriksson <marc...@apache.org> AuthorDate: Mon Dec 9 14:29:20 2019 +0100 Make it possible to disable STCS-in-L0 during runtime Patch by marcuse; reviewed by Jordan West for CASSANDRA-15445 --- CHANGES.txt | 1 + .../cassandra/config/DatabaseDescriptor.java | 7 ++- .../cassandra/db/compaction/CompactionManager.java | 12 ++++ .../db/compaction/CompactionManagerMBean.java | 6 ++ .../compaction/LeveledCompactionStrategyTest.java | 70 ++++++++++++++++++++++ .../org/apache/cassandra/schema/MockSchema.java | 15 +++-- 6 files changed, 102 insertions(+), 9 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4c4af72..b88a65b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0-alpha3 + * Make it possible to disable STCS-in-L0 during runtime (CASSANDRA-15445) * Removed obsolete OldNetworkTopologyStrategy (CASSANDRA-13990) * Align record header of FQL and audit binary log (CASSANDRA-15076) * Shuffle forwarding replica for messages to non-local DC (CASSANDRA-15318) diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 3c184bd..e6fa280 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -145,7 +145,7 @@ public class DatabaseDescriptor private static final int searchConcurrencyFactor = Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "search_concurrency_factor", "1")); - private static final boolean disableSTCSInL0 = Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_stcs_in_l0"); + private static volatile boolean disableSTCSInL0 = Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_stcs_in_l0"); private static final boolean unsafeSystem = Boolean.getBoolean(Config.PROPERTY_PREFIX + "unsafesystem"); // turns some warnings into exceptions for testing @@ -1737,6 +1737,11 @@ public class DatabaseDescriptor return disableSTCSInL0; } + public static void setDisableSTCSInL0(boolean disabled) + { + disableSTCSInL0 = disabled; + } + public static int getStreamThroughputOutboundMegabitsPerSec() { return conf.stream_throughput_outbound_megabits_per_sec; diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index 9dab6d7..28db027 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -2065,6 +2065,18 @@ public class CompactionManager implements CompactionManagerMBean validationExecutor.setMaximumPoolSize(number); } + public boolean getDisableSTCSInL0() + { + return DatabaseDescriptor.getDisableSTCSInL0(); + } + + public void setDisableSTCSInL0(boolean disabled) + { + if (disabled != DatabaseDescriptor.getDisableSTCSInL0()) + logger.info("Changing STCS in L0 disabled from {} to {}", DatabaseDescriptor.getDisableSTCSInL0(), disabled); + DatabaseDescriptor.setDisableSTCSInL0(disabled); + } + public int getCoreViewBuildThreads() { return viewBuildExecutor.getCorePoolSize(); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java index e4d5392..d298c8b 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java @@ -123,6 +123,12 @@ public interface CompactionManagerMBean public int getCoreViewBuildThreads(); /** + * Enable / disable STCS in L0 + */ + public boolean getDisableSTCSInL0(); + public void setDisableSTCSInL0(boolean disabled); + + /** * Allows user to resize maximum size of the view build thread pool. * @param number New maximum of view build threads */ diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java index 6c75e7b..b925bab 100644 --- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java @@ -17,13 +17,16 @@ */ package org.apache.cassandra.db.compaction; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.UUID; @@ -52,6 +55,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.notifications.SSTableAddedNotification; import org.apache.cassandra.notifications.SSTableRepairStatusChanged; import org.apache.cassandra.repair.ValidationManager; +import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.repair.Validator; @@ -490,4 +494,70 @@ public class LeveledCompactionStrategyTest lastMaxTimeStamp = sstable.getMaxTimestamp(); } } + + @Test + public void testDisableSTCSInL0() throws IOException + { + /* + First creates a bunch of sstables in L1, then overloads L0 with 50 sstables. Now with STCS in L0 enabled + we should get a compaction task where the target level is 0, then we disable STCS-in-L0 and make sure that + the compaction task we get targets L1 or higher. + */ + ColumnFamilyStore cfs = MockSchema.newCFS(); + Map<String, String> localOptions = new HashMap<>(); + localOptions.put("class", "LeveledCompactionStrategy"); + localOptions.put("sstable_size_in_mb", "1"); + cfs.setCompactionParameters(localOptions); + List<SSTableReader> sstables = new ArrayList<>(); + for (int i = 0; i < 11; i++) + { + SSTableReader l1sstable = MockSchema.sstable(i, 1 * 1024 * 1024, cfs); + l1sstable.descriptor.getMetadataSerializer().mutateLevel(l1sstable.descriptor, 1); + l1sstable.reloadSSTableMetadata(); + sstables.add(l1sstable); + } + + for (int i = 100; i < 150; i++) + sstables.add(MockSchema.sstable(i, 1 * 1024 * 1024, cfs)); + + cfs.disableAutoCompaction(); + cfs.addSSTables(sstables); + assertEquals(0, getTaskLevel(cfs)); + + try + { + CompactionManager.instance.setDisableSTCSInL0(true); + assertTrue(getTaskLevel(cfs) > 0); + } + finally + { + CompactionManager.instance.setDisableSTCSInL0(false); + } + } + + private int getTaskLevel(ColumnFamilyStore cfs) + { + int level = -1; + for (List<AbstractCompactionStrategy> strategies : cfs.getCompactionStrategyManager().getStrategies()) + { + for (AbstractCompactionStrategy strategy : strategies) + { + AbstractCompactionTask task = strategy.getNextBackgroundTask(0); + if (task != null) + { + try + { + assertTrue(task instanceof LeveledCompactionTask); + LeveledCompactionTask lcsTask = (LeveledCompactionTask) task; + level = Math.max(level, lcsTask.getLevel()); + } + finally + { + task.transaction.abort(); + } + } + } + } + return level; + } } diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index dfa8731..40b0f87 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import java.util.function.Function; import com.google.common.collect.ImmutableSet; @@ -40,14 +39,10 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.ChannelProxy; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.Memory; -import org.apache.cassandra.schema.CachingParams; -import org.apache.cassandra.schema.KeyspaceMetadata; -import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.AlwaysPresentFilter; import org.apache.cassandra.utils.ByteBufferUtil; @@ -65,7 +60,8 @@ public class MockSchema public static final Keyspace ks = Keyspace.mockKS(KeyspaceMetadata.create("mockks", KeyspaceParams.simpleTransient(1))); public static final IndexSummary indexSummary; - private static final FileHandle RANDOM_ACCESS_READER_FACTORY = new FileHandle.Builder(temp("mocksegmentedfile").getAbsolutePath()).complete(); + + private static final File tempFile = temp("mocksegmentedfile"); public static Memtable memtable(ColumnFamilyStore cfs) { @@ -114,6 +110,9 @@ public class MockSchema { } } + // .complete() with size to make sstable.onDiskLength work + @SuppressWarnings("resource") + FileHandle fileHandle = new FileHandle.Builder(new ChannelProxy(tempFile)).bufferSize(size).complete(size); if (size > 0) { try @@ -134,7 +133,7 @@ public class MockSchema .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header) .get(MetadataType.STATS); SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata, - RANDOM_ACCESS_READER_FACTORY.sharedCopy(), RANDOM_ACCESS_READER_FACTORY.sharedCopy(), indexSummary.sharedCopy(), + fileHandle.sharedCopy(), fileHandle.sharedCopy(), indexSummary.sharedCopy(), new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header); reader.first = readerBounds(firstToken); reader.last = readerBounds(lastToken); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org