This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e6d2dcc HDFS-15469. Dynamically configure the size of
PacketReceiver#MAX_PACKET_SIZE. (#2138)
e6d2dcc is described below
commit e6d2dccbef664e60e682eee31caf0b48e50a2251
Author: jianghuazhu <[email protected]>
AuthorDate: Wed Nov 11 08:34:17 2020 +0800
HDFS-15469. Dynamically configure the size of
PacketReceiver#MAX_PACKET_SIZE. (#2138)
---
.../org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java | 6 ++++++
.../hadoop/hdfs/protocol/datatransfer/PacketReceiver.java | 12 +++++++++++-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++
.../hdfs/protocol/datatransfer/TestPacketReceiver.java | 9 ++++++++-
4 files changed, 33 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 0e4cebf..f858080 100755
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -221,6 +221,12 @@ public interface HdfsClientConfigKeys {
"dfs.encrypt.data.transfer.cipher.key.bitlength";
int DFS_ENCRYPT_DATA_TRANSFER_CIPHER_KEY_BITLENGTH_DEFAULT = 128;
+ public static final String
+ DFS_DATA_TRANSFER_MAX_PACKET_SIZE =
+ "dfs.data.transfer.max.packet.size";
+ public static final int DFS_DATA_TRANSFER_MAX_PACKET_SIZE_DEFAULT =
+ 16 * 1024 * 1024;
+
String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS =
"dfs.trustedchannel.resolver.class";
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
index dc6d590..6949a9d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
@@ -25,6 +25,9 @@ import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.util.DirectBufferPool;
import org.apache.hadoop.io.IOUtils;
@@ -45,7 +48,7 @@ public class PacketReceiver implements Closeable {
* The max size of any single packet. This prevents OOMEs when
* invalid data is sent.
*/
- public static final int MAX_PACKET_SIZE = 16 * 1024 * 1024;
+ public static final int MAX_PACKET_SIZE;
static final Logger LOG = LoggerFactory.getLogger(PacketReceiver.class);
@@ -74,6 +77,13 @@ public class PacketReceiver implements Closeable {
*/
private PacketHeader curHeader;
+ static {
+ Configuration conf = new HdfsConfiguration();
+ MAX_PACKET_SIZE = conf.getInt(HdfsClientConfigKeys.
+ DFS_DATA_TRANSFER_MAX_PACKET_SIZE,
+ HdfsClientConfigKeys.DFS_DATA_TRANSFER_MAX_PACKET_SIZE_DEFAULT);
+ }
+
public PacketReceiver(boolean useDirectBuffers) {
this.useDirectBuffers = useDirectBuffers;
reallocPacketBuf(PacketHeader.PKT_LENGTHS_LEN);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 4a37d26..8d70c57 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4459,6 +4459,14 @@
</property>
<property>
+ <name>dfs.data.transfer.max.packet.size</name>
+ <value>16777216</value>
+ <description>
+ The max size of any single packet.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.balance.max.concurrent.moves</name>
<value>100</value>
<description>
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
index bec696c..f627f00 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hdfs.AppendTestUtil;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.junit.Test;
import org.mockito.Mockito;
@@ -56,7 +57,13 @@ public class TestPacketReceiver {
buf.get(b);
return b;
}
-
+
+ @Test
+ public void testPacketSize() {
+ assertEquals(PacketReceiver.MAX_PACKET_SIZE,
+ HdfsClientConfigKeys.DFS_DATA_TRANSFER_MAX_PACKET_SIZE_DEFAULT);
+ }
+
@Test
public void testReceiveAndMirror() throws IOException {
PacketReceiver pr = new PacketReceiver(false);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
