HDFS-6959 Make the HDFS home directory location customizable. Contributed by Yongjun Zhang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4caedfc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4caedfc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4caedfc Branch: refs/heads/MR-2841 Commit: f4caedfcbfeae7e2fe7c0e812ddbb087608a5ffd Parents: 3a0142b Author: Colin Patrick Mccabe <cmcc...@cloudera.com> Authored: Wed Sep 3 19:30:30 2014 -0700 Committer: Colin Patrick Mccabe <cmcc...@cloudera.com> Committed: Wed Sep 3 19:30:30 2014 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 ++ .../hadoop/hdfs/DistributedFileSystem.java | 11 ++++++-- .../src/main/resources/hdfs-default.xml | 8 ++++++ .../org/apache/hadoop/hdfs/TestLocalDFS.java | 29 +++++++++++++++++++- 5 files changed, 49 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 555f294..8498b00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -435,6 +435,9 @@ Release 2.6.0 - UNRELEASED HDFS-4257. The ReplaceDatanodeOnFailure policies could have a forgiving option (szetszwo via cmccabe) + HDFS-6959. Make the HDFS home directory location customizable. (yzhang via + cmccabe) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index ace2ae9..b5b4f3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -41,6 +41,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_STREAM_BUFFER_SIZE_DEFAULT = 4096; public static final String DFS_BYTES_PER_CHECKSUM_KEY = "dfs.bytes-per-checksum"; public static final int DFS_BYTES_PER_CHECKSUM_DEFAULT = 512; + public static final String DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix"; + public static final String DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user"; public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY = "dfs.client.retry.policy.enabled"; public static final boolean DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = false; public static final String DFS_CLIENT_RETRY_POLICY_SPEC_KEY = "dfs.client.retry.policy.spec"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index fc4bd84..bf7d62e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -102,6 +102,8 @@ import com.google.common.base.Preconditions; public class DistributedFileSystem extends FileSystem { private Path workingDir; private URI uri; + private String homeDirPrefix = + DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT; DFSClient dfs; private boolean verifyChecksum = true; @@ -136,7 +138,10 @@ public class DistributedFileSystem extends FileSystem { if (host == null) { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } - + homeDirPrefix = conf.get( + DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, + DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT); + this.dfs = new DFSClient(uri, conf, statistics); this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority()); this.workingDir = getHomeDirectory(); @@ -167,10 +172,10 @@ public class DistributedFileSystem extends FileSystem { workingDir = fixRelativePart(dir); } - @Override public Path getHomeDirectory() { - return makeQualified(new Path("/user/" + dfs.ugi.getShortUserName())); + return makeQualified(new Path(homeDirPrefix + "/" + + dfs.ugi.getShortUserName())); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index cdb0100..6d1eecc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2098,4 +2098,12 @@ </description> </property> +<property> + <name>dfs.user.home.dir.prefix</name> + <value>/user</value> + <description>The directory to prepend to user name to get the user's + home direcotry. + </description> +</property> + </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java index 7a5e3d0..1542962 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java @@ -90,7 +90,9 @@ public class TestLocalDFS { // test home directory Path home = - fileSys.makeQualified(new Path("/user/" + getUserName(fileSys))); + fileSys.makeQualified( + new Path(DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT + + "/" + getUserName(fileSys))); Path fsHome = fileSys.getHomeDirectory(); assertEquals(home, fsHome); @@ -99,4 +101,29 @@ public class TestLocalDFS { cluster.shutdown(); } } + + /** + * Tests get/set working directory in DFS. + */ + @Test(timeout=30000) + public void testHomeDirectory() throws IOException { + final String[] homeBases = new String[] {"/home", "/home/user"}; + Configuration conf = new HdfsConfiguration(); + for (final String homeBase : homeBases) { + conf.set(DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, homeBase); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + FileSystem fileSys = cluster.getFileSystem(); + try { + // test home directory + Path home = + fileSys.makeQualified( + new Path(homeBase + "/" + getUserName(fileSys))); + Path fsHome = fileSys.getHomeDirectory(); + assertEquals(home, fsHome); + } finally { + fileSys.close(); + cluster.shutdown(); + } + } + } }
