Author: cmccabe
Date: Tue Mar 4 06:51:50 2014
New Revision: 1573889
URL: http://svn.apache.org/r1573889
Log:
HDFS-6046. add dfs.client.mmap.enabled (cmccabe)
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1573887
Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1573887
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1573889&r1=1573888&r2=1573889&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Mar 4 06:51:50 2014
@@ -114,6 +114,8 @@ Release 2.4.0 - UNRELEASED
HDFS-5950. The DFSClient and DataNode should use shared memory segments to
communicate short-circuit information. (cmccabe)
+ HDFS-6046. add dfs.client.mmap.enabled (cmccabe)
+
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need
recovery
Propchange:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1573887
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1573889&r1=1573888&r2=1573889&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
Tue Mar 4 06:51:50 2014
@@ -281,6 +281,7 @@ public class DFSClient implements java.i
final long shortCircuitStreamsCacheExpiryMs;
final int shortCircuitSharedMemoryWatcherInterruptCheckMs;
+ final boolean shortCircuitMmapEnabled;
final int shortCircuitMmapCacheSize;
final long shortCircuitMmapCacheExpiryMs;
final long shortCircuitMmapCacheRetryTimeout;
@@ -400,6 +401,9 @@ public class DFSClient implements java.i
shortCircuitStreamsCacheExpiryMs = conf.getLong(
DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY,
DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_DEFAULT);
+ shortCircuitMmapEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_CLIENT_MMAP_ENABLED,
+ DFSConfigKeys.DFS_CLIENT_MMAP_ENABLED_DEFAULT);
shortCircuitMmapCacheSize = conf.getInt(
DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE,
DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE_DEFAULT);
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1573889&r1=1573888&r2=1573889&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Tue Mar 4 06:51:50 2014
@@ -431,6 +431,8 @@ public class DFSConfigKeys extends Commo
public static final int DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_DEFAULT =
1024 * 1024;
public static final String DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC =
"dfs.client.domain.socket.data.traffic";
public static final boolean DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT =
false;
+ public static final String DFS_CLIENT_MMAP_ENABLED=
"dfs.client.mmap.enabled";
+ public static final boolean DFS_CLIENT_MMAP_ENABLED_DEFAULT = true;
public static final String DFS_CLIENT_MMAP_CACHE_SIZE =
"dfs.client.mmap.cache.size";
public static final int DFS_CLIENT_MMAP_CACHE_SIZE_DEFAULT = 256;
public static final String DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS =
"dfs.client.mmap.cache.timeout.ms";
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1573889&r1=1573888&r2=1573889&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
Tue Mar 4 06:51:50 2014
@@ -1571,7 +1571,10 @@ implements ByteBufferReadable, CanSetDro
"at position " + pos);
}
}
- ByteBuffer buffer = tryReadZeroCopy(maxLength, opts);
+ ByteBuffer buffer = null;
+ if (dfsClient.getConf().shortCircuitMmapEnabled) {
+ buffer = tryReadZeroCopy(maxLength, opts);
+ }
if (buffer != null) {
return buffer;
}
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1573889&r1=1573888&r2=1573889&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Tue Mar 4 06:51:50 2014
@@ -1521,25 +1521,33 @@
</property>
<property>
+ <name>dfs.client.mmap.enabled</name>
+ <value>true</value>
+ <description>
+ If this is set to false, the client won't attempt to perform memory-mapped
reads.
+ </description>
+</property>
+
+<property>
<name>dfs.client.mmap.cache.size</name>
- <value>1024</value>
+ <value>256</value>
<description>
When zero-copy reads are used, the DFSClient keeps a cache of recently used
memory mapped regions. This parameter controls the maximum number of
entries that we will keep in that cache.
- If this is set to 0, we will not allow mmap.
-
The larger this number is, the more file descriptors we will potentially
use for memory-mapped files. mmaped files also use virtual address space.
You may need to increase your ulimit virtual address space limits before
increasing the client mmap cache size.
+
+ Note that you can still do zero-copy reads when this size is set to 0.
</description>
</property>
<property>
<name>dfs.client.mmap.cache.timeout.ms</name>
- <value>900000</value>
+ <value>3600000</value>
<description>
The minimum length of time that we will keep an mmap entry in the cache
between uses. If an entry is in the cache longer than this, and nobody
@@ -1558,7 +1566,7 @@
<property>
<name>dfs.client.short.circuit.replica.stale.threshold.ms</name>
- <value>3000000</value>
+ <value>1800000</value>
<description>
The maximum amount of time that we will consider a short-circuit replica to
be valid, if there is no communication from the DataNode. After this time
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java?rev=1573889&r1=1573888&r2=1573889&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
Tue Mar 4 06:51:50 2014
@@ -21,6 +21,8 @@ import static org.apache.hadoop.hdfs.DFS
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MMAP_ENABLED;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE;
import java.io.File;
import java.io.FileInputStream;
@@ -698,4 +700,63 @@ public class TestEnhancedByteBufferAcces
}
}, 10, 60000);
}
+
+ @Test
+ public void testClientMmapDisable() throws Exception {
+ HdfsConfiguration conf = initZeroCopyTest();
+ conf.setBoolean(DFS_CLIENT_MMAP_ENABLED, false);
+ MiniDFSCluster cluster = null;
+ final Path TEST_PATH = new Path("/a");
+ final int TEST_FILE_LENGTH = 16385;
+ final int RANDOM_SEED = 23453;
+ final String CONTEXT = "testClientMmapDisable";
+ FSDataInputStream fsIn = null;
+ DistributedFileSystem fs = null;
+ conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT, CONTEXT);
+
+ try {
+ // With DFS_CLIENT_MMAP_ENABLED set to false, we should not do memory
+ // mapped reads.
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ DFSTestUtil.createFile(fs, TEST_PATH,
+ TEST_FILE_LENGTH, (short)1, RANDOM_SEED);
+ DFSTestUtil.waitReplication(fs, TEST_PATH, (short)1);
+ fsIn = fs.open(TEST_PATH);
+ try {
+ fsIn.read(null, 1, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
+ Assert.fail("expected zero-copy read to fail when client mmaps " +
+ "were disabled.");
+ } catch (UnsupportedOperationException e) {
+ }
+ } finally {
+ if (fsIn != null) fsIn.close();
+ if (fs != null) fs.close();
+ if (cluster != null) cluster.shutdown();
+ }
+
+ fsIn = null;
+ fs = null;
+ cluster = null;
+ try {
+ // Now try again with DFS_CLIENT_MMAP_CACHE_SIZE == 0. It should work.
+ conf.setBoolean(DFS_CLIENT_MMAP_ENABLED, true);
+ conf.setInt(DFS_CLIENT_MMAP_CACHE_SIZE, 0);
+ conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT, CONTEXT + ".1");
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ DFSTestUtil.createFile(fs, TEST_PATH,
+ TEST_FILE_LENGTH, (short)1, RANDOM_SEED);
+ DFSTestUtil.waitReplication(fs, TEST_PATH, (short)1);
+ fsIn = fs.open(TEST_PATH);
+ ByteBuffer buf = fsIn.read(null, 1,
EnumSet.of(ReadOption.SKIP_CHECKSUMS));
+ fsIn.releaseBuffer(buf);
+ } finally {
+ if (fsIn != null) fsIn.close();
+ if (fs != null) fs.close();
+ if (cluster != null) cluster.shutdown();
+ }
+ }
}
