Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/764f3d3a Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/764f3d3a Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/764f3d3a Branch: refs/heads/cassandra-3.0 Commit: 764f3d3a1dd4d643cd3178967f14b45d155f4fd6 Parents: a3d0556 4aa859e Author: Yuki Morishita <yu...@apache.org> Authored: Thu May 26 09:53:39 2016 -0500 Committer: Yuki Morishita <yu...@apache.org> Committed: Thu May 26 09:53:39 2016 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/io/FSErrorHandler.java | 30 ++++++ .../org/apache/cassandra/io/util/FileUtils.java | 79 ++++----------- .../cassandra/service/CassandraDaemon.java | 2 + .../service/DefaultFSErrorHandler.java | 101 +++++++++++++++++++ .../apache/cassandra/db/DirectoriesTest.java | 2 + 6 files changed, 153 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/764f3d3a/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 062bb7b,f2276f0..75ce90d --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,44 -1,9 +1,45 @@@ -2.2.7 +3.0.7 + * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530) + * Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705) + * Allow compaction strategies to disable early open (CASSANDRA-11754) + * Refactor Materialized View code (CASSANDRA-11475) + * Update Java Driver (CASSANDRA-11615) +Merged from 2.2: * Enable client encryption in sstableloader with cli options (CASSANDRA-11708) * Possible memory leak in NIODataInputStream (CASSANDRA-11867) - * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669) * Add seconds to cqlsh tracing session duration (CASSANDRA-11753) - * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395) + * Prohibit Reversed Counter type as part of the PK (CASSANDRA-9395) +Merged from 2.1: ++ * Backport CASSANDRA-11578 (CASSANDRA-11750) + * Clear out parent repair session if repair coordinator dies (CASSANDRA-11824) + * Set default streaming_socket_timeout_in_ms to 24 hours (CASSANDRA-11840) + * Do not consider local node a valid source during replace (CASSANDRA-11848) + * Add message dropped tasks to nodetool netstats (CASSANDRA-11855) + * Avoid holding SSTableReaders for duration of incremental repair (CASSANDRA-11739) + + +3.0.6 + * Disallow creating view with a static column (CASSANDRA-11602) + * Reduce the amount of object allocations caused by the getFunctions methods (CASSANDRA-11593) + * Potential error replaying commitlog with smallint/tinyint/date/time types (CASSANDRA-11618) + * Fix queries with filtering on counter columns (CASSANDRA-11629) + * Improve tombstone printing in sstabledump (CASSANDRA-11655) + * Fix paging for range queries where all clustering columns are specified (CASSANDRA-11669) + * Don't require HEAP_NEW_SIZE to be set when using G1 (CASSANDRA-11600) + * Fix sstabledump not showing cells after tombstone marker (CASSANDRA-11654) + * Ignore all LocalStrategy keyspaces for streaming and other related + operations (CASSANDRA-11627) + * Ensure columnfilter covers indexed columns for thrift 2i queries (CASSANDRA-11523) + * Only open one sstable scanner per sstable (CASSANDRA-11412) + * Option to specify ProtocolVersion in cassandra-stress (CASSANDRA-11410) + * ArithmeticException in avgFunctionForDecimal (CASSANDRA-11485) + * LogAwareFileLister should only use OLD sstable files in current folder to determine disk consistency (CASSANDRA-11470) + * Notify indexers of expired rows during compaction (CASSANDRA-11329) + * Properly respond with ProtocolError when a v1/v2 native protocol + header is received (CASSANDRA-11464) + * Validate that num_tokens and initial_token are consistent with one another (CASSANDRA-10120) +Merged from 2.2: + * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669) * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626) * Exit JVM if JMX server fails to startup (CASSANDRA-11540) * Produce a heap dump when exiting on OOM (CASSANDRA-9861) http://git-wip-us.apache.org/repos/asf/cassandra/blob/764f3d3a/src/java/org/apache/cassandra/io/util/FileUtils.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/util/FileUtils.java index 46f2de5,3124fcd..9e81da5 --- a/src/java/org/apache/cassandra/io/util/FileUtils.java +++ b/src/java/org/apache/cassandra/io/util/FileUtils.java @@@ -25,19 -23,17 +25,17 @@@ import java.nio.charset.StandardCharset import java.nio.file.*; import java.text.DecimalFormat; import java.util.Arrays; +import java.util.Collections; +import java.util.List; - - import sun.nio.ch.DirectBuffer; + import java.util.concurrent.atomic.AtomicReference; -import org.apache.cassandra.config.Config; -import sun.nio.ch.DirectBuffer; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; ++import sun.nio.ch.DirectBuffer; import org.apache.cassandra.concurrent.ScheduledExecutors; - import org.apache.cassandra.config.DatabaseDescriptor; - import org.apache.cassandra.db.BlacklistedDirectories; - import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.io.FSError; + import org.apache.cassandra.io.FSErrorHandler; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.CorruptSSTableException; @@@ -47,10 -42,8 +44,10 @@@ import org.apache.cassandra.utils.JVMSt import static org.apache.cassandra.utils.Throwables.maybeFail; import static org.apache.cassandra.utils.Throwables.merge; - public class FileUtils + public final class FileUtils { + public static final Charset CHARSET = StandardCharsets.UTF_8; + private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); private static final double KB = 1024d; private static final double MB = 1024*1024d; @@@ -458,65 -396,30 +456,18 @@@ dir.deleteOnExit(); } - public static void skipBytesFully(DataInput in, int bytes) throws IOException - { - int n = 0; - while (n < bytes) - { - int skipped = in.skipBytes(bytes - n); - if (skipped == 0) - throw new EOFException("EOF after " + n + " bytes out of " + bytes); - n += skipped; - } - } - public static void handleCorruptSSTable(CorruptSSTableException e) { - if (!StorageService.instance.isSetupCompleted()) - handleStartupFSError(e); - - JVMStabilityInspector.inspectThrowable(e); - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - StorageService.instance.stopTransports(); - break; - } + FSErrorHandler handler = fsErrorHandler.get(); + if (handler != null) + handler.handleCorruptSSTable(e); } - + public static void handleFSError(FSError e) { - if (!StorageService.instance.isSetupCompleted()) - handleStartupFSError(e); - - JVMStabilityInspector.inspectThrowable(e); - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - case stop: - StorageService.instance.stopTransports(); - break; - case best_effort: - // for both read and write errors mark the path as unwritable. - BlacklistedDirectories.maybeMarkUnwritable(e.path); - if (e instanceof FSReadError) - { - File directory = BlacklistedDirectories.maybeMarkUnreadable(e.path); - if (directory != null) - Keyspace.removeUnreadableSSTables(directory); - } - break; - case ignore: - // already logged, so left nothing to do - break; - default: - throw new IllegalStateException(); - } - } - - private static void handleStartupFSError(Throwable t) - { - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - case stop: - case die: - logger.error("Exiting forcefully due to file system exception on startup, disk failure policy \"{}\"", - DatabaseDescriptor.getDiskFailurePolicy(), - t); - JVMStabilityInspector.killCurrentJVM(t, true); - break; - default: - break; - } + FSErrorHandler handler = fsErrorHandler.get(); + if (handler != null) + handler.handleFSError(e); } /** * Get the size of a directory in bytes @@@ -572,54 -474,8 +522,59 @@@ return false; } + public static void append(File file, String ... lines) + { + if (file.exists()) + write(file, Arrays.asList(lines), StandardOpenOption.APPEND); + else + write(file, Arrays.asList(lines), StandardOpenOption.CREATE); + } + + public static void appendAndSync(File file, String ... lines) + { + if (file.exists()) + write(file, Arrays.asList(lines), StandardOpenOption.APPEND, StandardOpenOption.SYNC); + else + write(file, Arrays.asList(lines), StandardOpenOption.CREATE, StandardOpenOption.SYNC); + } + + public static void replace(File file, String ... lines) + { + write(file, Arrays.asList(lines), StandardOpenOption.TRUNCATE_EXISTING); + } + + public static void write(File file, List<String> lines, StandardOpenOption ... options) + { + try + { + Files.write(file.toPath(), + lines, + CHARSET, + options); + } + catch (IOException ex) + { + throw new RuntimeException(ex); + } + } + + public static List<String> readLines(File file) + { + try + { + return Files.readAllLines(file.toPath(), CHARSET); + } + catch (IOException ex) + { + if (ex instanceof NoSuchFileException) + return Collections.emptyList(); + + throw new RuntimeException(ex); + } + } ++ + public static void setFSErrorHandler(FSErrorHandler handler) + { + fsErrorHandler.getAndSet(handler); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/764f3d3a/src/java/org/apache/cassandra/service/CassandraDaemon.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/764f3d3a/test/unit/org/apache/cassandra/db/DirectoriesTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/DirectoriesTest.java index f864bbc,008e343..f8d01a8 --- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java +++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java @@@ -42,8 -36,9 +42,9 @@@ import org.apache.cassandra.io.FSWriteE import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.IndexMetadata; + import org.apache.cassandra.service.DefaultFSErrorHandler; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.utils.Pair; import static org.junit.Assert.assertEquals; @@@ -65,14 -63,10 +66,15 @@@ public class DirectoriesTes @BeforeClass public static void beforeClass() throws IOException { + FileUtils.setFSErrorHandler(new DefaultFSErrorHandler()); - for (String cf : CFS) + for (String table : TABLES) { - CFM.add(new CFMetaData(KS, cf, ColumnFamilyType.Standard, null)); + UUID tableID = CFMetaData.generateLegacyCfId(KS, table); + CFM.add(CFMetaData.Builder.create(KS, table) + .withId(tableID) + .addPartitionKey("thekey", UTF8Type.instance) + .addClusteringColumn("thecolumn", UTF8Type.instance) + .build()); } tempDataDir = File.createTempFile("cassandra", "unittest");
