Backport CASSANDRA-11578 patch by yukim; reviewed by Paulo Motta for CASSANDRA-11750
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b851792c Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b851792c Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b851792c Branch: refs/heads/cassandra-2.2 Commit: b851792c4e3ae32b8d863d9079cca6d135f1cf23 Parents: 5dc7414 Author: Yuki Morishita <yu...@apache.org> Authored: Wed May 18 17:03:39 2016 -0500 Committer: Yuki Morishita <yu...@apache.org> Committed: Thu May 26 09:40:40 2016 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/io/FSErrorHandler.java | 30 ++++++ .../org/apache/cassandra/io/util/FileUtils.java | 75 +++----------- .../cassandra/service/CassandraDaemon.java | 2 + .../service/DefaultFSErrorHandler.java | 101 +++++++++++++++++++ .../apache/cassandra/db/DirectoriesTest.java | 2 + 6 files changed, 152 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/b851792c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index f73db6e..ad9d00c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.1.15 + * Backport CASSANDRA-11578 (CASSANDRA-11750) * Clear out parent repair session if repair coordinator dies (CASSANDRA-11824) * Set default streaming_socket_timeout_in_ms to 24 hours (CASSANDRA-11840) * Do not consider local node a valid source during replace (CASSANDRA-11848) http://git-wip-us.apache.org/repos/asf/cassandra/blob/b851792c/src/java/org/apache/cassandra/io/FSErrorHandler.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/FSErrorHandler.java b/src/java/org/apache/cassandra/io/FSErrorHandler.java new file mode 100644 index 0000000..081ec0b --- /dev/null +++ b/src/java/org/apache/cassandra/io/FSErrorHandler.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io; + +import org.apache.cassandra.io.sstable.CorruptSSTableException; + +/** + * Interface for handling file system errors. + */ +public interface FSErrorHandler +{ + void handleCorruptSSTable(CorruptSSTableException e); + void handleFSError(FSError e); +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/b851792c/src/java/org/apache/cassandra/io/util/FileUtils.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java index 3be7c99..f69ed01 100644 --- a/src/java/org/apache/cassandra/io/util/FileUtils.java +++ b/src/java/org/apache/cassandra/io/util/FileUtils.java @@ -32,6 +32,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.text.DecimalFormat; import java.util.Arrays; +import java.util.concurrent.atomic.AtomicReference; import sun.nio.ch.DirectBuffer; @@ -39,17 +40,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutors; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.BlacklistedDirectories; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.io.FSError; +import org.apache.cassandra.io.FSErrorHandler; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.JVMStabilityInspector; -public class FileUtils +public final class FileUtils { private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); private static final double KB = 1024d; @@ -59,6 +57,7 @@ public class FileUtils private static final DecimalFormat df = new DecimalFormat("#.##"); private static final boolean canCleanDirectBuffers; + private static final AtomicReference<FSErrorHandler> fsErrorHandler = new AtomicReference<>(); static { @@ -395,63 +394,16 @@ public class FileUtils public static void handleCorruptSSTable(CorruptSSTableException e) { - if (!StorageService.instance.isSetupCompleted()) - handleStartupFSError(e); - - JVMStabilityInspector.inspectThrowable(e); - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - StorageService.instance.stopTransports(); - break; - } - } - - public static void handleFSError(FSError e) - { - if (!StorageService.instance.isSetupCompleted()) - handleStartupFSError(e); - - JVMStabilityInspector.inspectThrowable(e); - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - case stop: - StorageService.instance.stopTransports(); - break; - case best_effort: - // for both read and write errors mark the path as unwritable. - BlacklistedDirectories.maybeMarkUnwritable(e.path); - if (e instanceof FSReadError) - { - File directory = BlacklistedDirectories.maybeMarkUnreadable(e.path); - if (directory != null) - Keyspace.removeUnreadableSSTables(directory); - } - break; - case ignore: - // already logged, so left nothing to do - break; - default: - throw new IllegalStateException(); - } + FSErrorHandler handler = fsErrorHandler.get(); + if (handler != null) + handler.handleCorruptSSTable(e); } - private static void handleStartupFSError(Throwable t) + public static void handleFSError(FSError e) { - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - case stop: - case die: - logger.error("Exiting forcefully due to file system exception on startup, disk failure policy \"{}\"", - DatabaseDescriptor.getDiskFailurePolicy(), - t); - JVMStabilityInspector.killCurrentJVM(t, true); - break; - default: - break; - } + FSErrorHandler handler = fsErrorHandler.get(); + if (handler != null) + handler.handleFSError(e); } /** * Get the size of a directory in bytes @@ -470,4 +422,9 @@ public class FileUtils } return length; } + + public static void setFSErrorHandler(FSErrorHandler handler) + { + fsErrorHandler.getAndSet(handler); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/b851792c/src/java/org/apache/cassandra/service/CassandraDaemon.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index a27fa20..5a1fb14 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -148,6 +148,8 @@ public class CassandraDaemon */ protected void setup() { + FileUtils.setFSErrorHandler(new DefaultFSErrorHandler()); + try { logger.info("Hostname: {}", InetAddress.getLocalHost().getHostName()); http://git-wip-us.apache.org/repos/asf/cassandra/blob/b851792c/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java b/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java new file mode 100644 index 0000000..88a1fce --- /dev/null +++ b/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service; + +import java.io.File; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.BlacklistedDirectories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.io.FSError; +import org.apache.cassandra.io.FSErrorHandler; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.utils.JVMStabilityInspector; + +public class DefaultFSErrorHandler implements FSErrorHandler +{ + private static final Logger logger = LoggerFactory.getLogger(DefaultFSErrorHandler.class); + + @Override + public void handleCorruptSSTable(CorruptSSTableException e) + { + if (!StorageService.instance.isSetupCompleted()) + handleStartupFSError(e); + + JVMStabilityInspector.inspectThrowable(e); + switch (DatabaseDescriptor.getDiskFailurePolicy()) + { + case stop_paranoid: + StorageService.instance.stopTransports(); + break; + } + } + + @Override + public void handleFSError(FSError e) + { + if (!StorageService.instance.isSetupCompleted()) + handleStartupFSError(e); + + JVMStabilityInspector.inspectThrowable(e); + switch (DatabaseDescriptor.getDiskFailurePolicy()) + { + case stop_paranoid: + case stop: + StorageService.instance.stopTransports(); + break; + case best_effort: + // for both read and write errors mark the path as unwritable. + BlacklistedDirectories.maybeMarkUnwritable(e.path); + if (e instanceof FSReadError) + { + File directory = BlacklistedDirectories.maybeMarkUnreadable(e.path); + if (directory != null) + Keyspace.removeUnreadableSSTables(directory); + } + break; + case ignore: + // already logged, so left nothing to do + break; + default: + throw new IllegalStateException(); + } + } + + private static void handleStartupFSError(Throwable t) + { + switch (DatabaseDescriptor.getDiskFailurePolicy()) + { + case stop_paranoid: + case stop: + case die: + logger.error("Exiting forcefully due to file system exception on startup, disk failure policy \"{}\"", + DatabaseDescriptor.getDiskFailurePolicy(), + t); + JVMStabilityInspector.killCurrentJVM(t, true); + break; + default: + break; + } + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/b851792c/test/unit/org/apache/cassandra/db/DirectoriesTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java index b1c51ee..4267c1f 100644 --- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java +++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java @@ -44,6 +44,7 @@ import org.apache.cassandra.db.Directories.DataDirectory; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.DefaultFSErrorHandler; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.io.FSWriteError; @@ -65,6 +66,7 @@ public class DirectoriesTest @BeforeClass public static void beforeClass() throws IOException { + FileUtils.setFSErrorHandler(new DefaultFSErrorHandler()); for (String cf : CFS) { CFM.add(new CFMetaData(KS, cf, ColumnFamilyType.Standard, null));
