HDFS-10903. Replace config key literal strings with config key names II: hadoop hdfs. Contributed by Chen Liang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c9a0106 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c9a0106 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c9a0106 Branch: refs/heads/HADOOP-13037 Commit: 3c9a01062e9097c2ed1db75318482543db2e382f Parents: 61f0490 Author: Mingliang Liu <lium...@apache.org> Authored: Tue Oct 11 16:29:30 2016 -0700 Committer: Mingliang Liu <lium...@apache.org> Committed: Tue Oct 11 16:29:30 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/http/server/FSOperations.java | 9 +++++++-- .../hadoop/lib/service/hadoop/FileSystemAccessService.java | 6 ++++-- .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +++ .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++ .../test/java/org/apache/hadoop/hdfs/TestFileAppend4.java | 3 ++- .../hdfs/server/blockmanagement/TestBlockTokenWithDFS.java | 3 ++- 6 files changed, 26 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c9a0106/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 46948f9..001bc92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -48,6 +48,9 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import static org.apache.hadoop.hdfs.DFSConfigKeys.HTTPFS_BUFFER_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.HTTP_BUFFER_SIZE_DEFAULT; + /** * FileSystem operation executors used by {@link HttpFSServer}. */ @@ -462,7 +465,8 @@ public class FSOperations { blockSize = fs.getDefaultBlockSize(path); } FsPermission fsPermission = new FsPermission(permission); - int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); + int bufferSize = fs.getConf().getInt(HTTPFS_BUFFER_SIZE_KEY, + HTTP_BUFFER_SIZE_DEFAULT); OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null); IOUtils.copyBytes(is, os, bufferSize, true); os.close(); @@ -752,7 +756,8 @@ public class FSOperations { */ @Override public InputStream execute(FileSystem fs) throws IOException { - int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096); + int bufferSize = HttpFSServerWebApp.get().getConfig().getInt( + HTTPFS_BUFFER_SIZE_KEY, HTTP_BUFFER_SIZE_DEFAULT); return fs.open(path, bufferSize); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c9a0106/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java index 0b767be..61d3b45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java @@ -50,6 +50,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; + @InterfaceAudience.Private public class FileSystemAccessService extends BaseService implements FileSystemAccess { private static final Logger LOG = LoggerFactory.getLogger(FileSystemAccessService.class); @@ -159,7 +161,7 @@ public class FileSystemAccessService extends BaseService implements FileSystemAc throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_PRINCIPAL); } Configuration conf = new Configuration(); - conf.set("hadoop.security.authentication", "kerberos"); + conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); try { UserGroupInformation.loginUserFromKeytab(principal, keytab); @@ -169,7 +171,7 @@ public class FileSystemAccessService extends BaseService implements FileSystemAc LOG.info("Using FileSystemAccess Kerberos authentication, principal [{}] keytab [{}]", principal, keytab); } else if (security.equals("simple")) { Configuration conf = new Configuration(); - conf.set("hadoop.security.authentication", "simple"); + conf.set(HADOOP_SECURITY_AUTHENTICATION, "simple"); UserGroupInformation.setConfiguration(conf); LOG.info("Using FileSystemAccess simple/pseudo authentication, principal [{}]", System.getProperty("user.name")); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c9a0106/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index df45e2a..18209ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -992,6 +992,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.disk.balancer.plan.threshold.percent"; public static final int DFS_DISK_BALANCER_PLAN_THRESHOLD_DEFAULT = 10; + public static final String HTTPFS_BUFFER_SIZE_KEY = + "httpfs.buffer.size"; + public static final int HTTP_BUFFER_SIZE_DEFAULT = 4096; // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c9a0106/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 672b597..db4035d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4273,4 +4273,12 @@ consecutive warnings within this interval.</description> </property> + <property> + <name>httpfs.buffer.size</name> + <value>4096</value> + <description> + The size buffer to be used when creating or opening httpfs filesystem IO stream. + </description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c9a0106/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java index 4147851..ae0f0c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -89,7 +90,7 @@ public class TestFileAppend4 { // handle failures in the DFSClient pipeline quickly // (for cluster.shutdown(); fs.close() idiom) - conf.setInt("ipc.client.connect.max.retries", 1); + conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); } /* http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c9a0106/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java index 9374ae8..5a8a39a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -218,7 +219,7 @@ public class TestBlockTokenWithDFS { conf.setInt("io.bytes.per.checksum", BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, numDataNodes); - conf.setInt("ipc.client.connect.max.retries", 0); + conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); // Set short retry timeouts so this test runs faster conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10); return conf; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org