Author: cmccabe
Date: Thu Mar 6 08:27:55 2014
New Revision: 1574799
URL: http://svn.apache.org/r1574799
Log:
HDFS-6061. Allow dfs.datanode.shared.file.descriptor.path to contain multiple
entries and fall back when needed (cmccabe)
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java
Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1574796
Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1574796
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1574799&r1=1574798&r2=1574799&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Mar 6 08:27:55 2014
@@ -124,6 +124,9 @@ Release 2.4.0 - UNRELEASED
HDFS-6044. Add property for setting the NFS look up time for users
(brandonli)
+ HDFS-6061. Allow dfs.datanode.shared.file.descriptor.path to contain
+ multiple entries and fall back when needed (cmccabe)
+
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need
recovery
Propchange:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1574796
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1574799&r1=1574798&r2=1574799&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Thu Mar 6 08:27:55 2014
@@ -473,8 +473,8 @@ public class DFSConfigKeys extends Commo
public static final String DFS_NAMENODE_STARTUP_KEY =
"dfs.namenode.startup";
public static final String DFS_DATANODE_KEYTAB_FILE_KEY =
"dfs.datanode.keytab.file";
public static final String DFS_DATANODE_USER_NAME_KEY =
"dfs.datanode.kerberos.principal";
- public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH =
"dfs.datanode.shared.file.descriptor.path";
- public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH_DEFAULT
= "/dev/shm";
+ public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS =
"dfs.datanode.shared.file.descriptor.paths";
+ public static final String
DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT = "/dev/shm,/tmp";
public static final String
DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS =
"dfs.short.circuit.shared.memory.watcher.interrupt.check.ms";
public static final int
DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT = 60000;
public static final String DFS_NAMENODE_KEYTAB_FILE_KEY =
"dfs.namenode.keytab.file";
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java?rev=1574799&r1=1574798&r2=1574799&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
Thu Mar 6 08:27:55 2014
@@ -17,14 +17,15 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
-import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH;
-import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH_DEFAULT;
+import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS;
+import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT;
import java.io.Closeable;
import java.io.FileInputStream;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;
@@ -45,7 +46,9 @@ import org.apache.hadoop.net.unix.Domain
import org.apache.hadoop.net.unix.DomainSocketWatcher;
import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
/*
* Manages client short-circuit memory segments on the DataNode.
@@ -149,38 +152,35 @@ public class ShortCircuitRegistry {
SharedFileDescriptorFactory shmFactory = null;
DomainSocketWatcher watcher = null;
try {
- String loadingFailureReason =
- SharedFileDescriptorFactory.getLoadingFailureReason();
- if (loadingFailureReason != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Disabling ShortCircuitRegistry because " +
- loadingFailureReason);
- }
- return;
- }
- String shmPath = conf.get(DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH,
- DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH_DEFAULT);
- if (shmPath.isEmpty()) {
- LOG.debug("Disabling ShortCircuitRegistry because shmPath was not
set.");
- return;
- }
int interruptCheck = conf.getInt(
DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS,
DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT);
if (interruptCheck <= 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Disabling ShortCircuitRegistry because " +
- "interruptCheckMs was set to " + interruptCheck);
- }
- return;
+ throw new IOException(
+ DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS +
+ " was set to " + interruptCheck);
+ }
+ String shmPaths[] =
+ conf.getTrimmedStrings(DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS);
+ if (shmPaths.length == 0) {
+ shmPaths =
+ DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT.split(",");
+ }
+ shmFactory = SharedFileDescriptorFactory.
+ create("HadoopShortCircuitShm_", shmPaths);
+ String dswLoadingFailure = DomainSocketWatcher.getLoadingFailureReason();
+ if (dswLoadingFailure != null) {
+ throw new IOException(dswLoadingFailure);
}
- shmFactory =
- new SharedFileDescriptorFactory("HadoopShortCircuitShm_", shmPath);
watcher = new DomainSocketWatcher(interruptCheck);
enabled = true;
if (LOG.isDebugEnabled()) {
LOG.debug("created new ShortCircuitRegistry with interruptCheck=" +
- interruptCheck + ", shmPath=" + shmPath);
+ interruptCheck + ", shmPath=" + shmFactory.getPath());
+ }
+ } catch (IOException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Disabling ShortCircuitRegistry", e);
}
} finally {
this.enabled = enabled;
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1574799&r1=1574798&r2=1574799&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Thu Mar 6 08:27:55 2014
@@ -1138,13 +1138,13 @@
</property>
<property>
- <name>dfs.datanode.shared.file.descriptor.path</name>
- <value>/dev/shm</value>
+ <name>dfs.datanode.shared.file.descriptor.paths</name>
+ <value>/dev/shm,/tmp</value>
<description>
- The path to use when creating file descriptors that will be shared
- between the DataNode and the DFSClient. Typically we use /dev/shm, so
- that the file descriptors will not be written to disk. Systems that
- don't have /dev/shm should use /tmp.
+ A comma-separated list of paths to use when creating file descriptors that
+ will be shared between the DataNode and the DFSClient. Typically we use
+ /dev/shm, so that the file descriptors will not be written to disk.
+ Systems that don't have /dev/shm will fall back to /tmp by default.
</description>
</property>
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java?rev=1574799&r1=1574798&r2=1574799&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java
Thu Mar 6 08:27:55 2014
@@ -22,11 +22,9 @@ import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Iterator;
-import org.apache.commons.lang.SystemUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory;
import org.apache.hadoop.hdfs.ExtendedBlockId;
import org.apache.hadoop.hdfs.ShortCircuitShm;
@@ -45,8 +43,8 @@ public class TestShortCircuitShm {
@Before
public void before() {
- Assume.assumeTrue(NativeIO.isAvailable());
- Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
+ Assume.assumeTrue(null ==
+ SharedFileDescriptorFactory.getLoadingFailureReason());
}
@Test(timeout=60000)
@@ -54,7 +52,8 @@ public class TestShortCircuitShm {
File path = new File(TEST_BASE, "testStartupShutdown");
path.mkdirs();
SharedFileDescriptorFactory factory =
- new SharedFileDescriptorFactory("shm_", path.getAbsolutePath());
+ SharedFileDescriptorFactory.create("shm_",
+ new String[] { path.getAbsolutePath() } );
FileInputStream stream =
factory.createDescriptor("testStartupShutdown", 4096);
ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
@@ -68,7 +67,8 @@ public class TestShortCircuitShm {
File path = new File(TEST_BASE, "testAllocateSlots");
path.mkdirs();
SharedFileDescriptorFactory factory =
- new SharedFileDescriptorFactory("shm_", path.getAbsolutePath());
+ SharedFileDescriptorFactory.create("shm_",
+ new String[] { path.getAbsolutePath() });
FileInputStream stream =
factory.createDescriptor("testAllocateSlots", 4096);
ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
