This is an automated email from the ASF dual-hosted git repository.
wwei pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 9c3806c HDFS-12748. NameNode memory leak when accessing webhdfs
GETHOMEDIRECTORY. Contributed by Weiwei Yang.
9c3806c is described below
commit 9c3806cf1b78521b9df3fcbfa7fa7be835484166
Author: Weiwei Yang <w...@apache.org>
AuthorDate: Thu Jul 11 09:46:27 2019 +0800
HDFS-12748. NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY.
Contributed by Weiwei Yang.
---
.../java/org/apache/hadoop/hdfs/DFSUtilClient.java | 20 ++++++++++++++++++++
.../apache/hadoop/hdfs/DistributedFileSystem.java | 8 +-------
.../web/resources/NamenodeWebHdfsMethods.java | 6 +++---
3 files changed, 24 insertions(+), 10 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index dc23283..7236247 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -911,4 +911,24 @@ public class DFSUtilClient {
return new Path(sb.toString());
}
+ /**
+ * Returns current user home directory under a home directory prefix.
+ * The home directory prefix can be defined by
+ * {@link HdfsClientConfigKeys#DFS_USER_HOME_DIR_PREFIX_KEY}.
+ * User info is obtained from given {@link UserGroupInformation}.
+ * @param conf configuration
+ * @param ugi {@link UserGroupInformation} of current user.
+ * @return the home directory of current user.
+ */
+ public static Path getHomeDirectory(Configuration conf,
+ UserGroupInformation ugi) {
+ String userHomePrefix = HdfsClientConfigKeys
+ .DFS_USER_HOME_DIR_PREFIX_DEFAULT;
+ if (conf != null) {
+ userHomePrefix = conf.get(
+ HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY,
+ HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT);
+ }
+ return new Path(userHomePrefix + "/" + ugi.getShortUserName());
+ }
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index f44eb4b..4cb20b3 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -132,8 +132,6 @@ public class DistributedFileSystem extends FileSystem
implements KeyProviderTokenIssuer {
private Path workingDir;
private URI uri;
- private String homeDirPrefix =
- HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT;
DFSClient dfs;
private boolean verifyChecksum = true;
@@ -170,9 +168,6 @@ public class DistributedFileSystem extends FileSystem
if (host == null) {
throw new IOException("Incomplete HDFS URI, no host: "+ uri);
}
- homeDirPrefix = conf.get(
- HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY,
- HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT);
this.dfs = new DFSClient(uri, conf, statistics);
this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority());
@@ -215,8 +210,7 @@ public class DistributedFileSystem extends FileSystem
@Override
public Path getHomeDirectory() {
- return makeQualified(new Path(homeDirPrefix + "/"
- + dfs.ugi.getShortUserName()));
+ return makeQualified(DFSUtilClient.getHomeDirectory(getConf(), dfs.ugi));
}
/**
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index fa4bd37..09ec31c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@@ -1151,9 +1152,8 @@ public class NamenodeWebHdfsMethods {
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
case GETHOMEDIRECTORY: {
- final String js = JsonUtil.toJsonString("Path",
- FileSystem.get(conf != null ? conf : new Configuration())
- .getHomeDirectory().toUri().getPath());
+ String userHome = DFSUtilClient.getHomeDirectory(conf, ugi).toString();
+ final String js = JsonUtil.toJsonString("Path", userHome);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
case GETACLSTATUS: {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
