HDFS-13453. RBF: getMountPointDates should fetch latest subdir time/date when parent dir is not present but /parent/child dirs are present in mount table. Contributed by Dibyendu Karmakar.
(cherry picked from commit 1134af9ad1daf683204df8f95a8f03d7baaa74d4) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3c17359 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3c17359 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3c17359 Branch: refs/heads/YARN-8200 Commit: a3c17359513f775e5768ad9c57c33f3e2fdc6bdd Parents: 2444d70 Author: Inigo Goiri <inigo...@apache.org> Authored: Thu Apr 19 14:56:36 2018 -0700 Committer: Inigo Goiri <inigo...@apache.org> Committed: Thu Apr 19 14:58:02 2018 -0700 ---------------------------------------------------------------------- .../federation/router/RouterRpcServer.java | 55 ++++++++++++++++---- .../federation/router/TestRouterMountTable.java | 21 +++++++- 2 files changed, 65 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c17359/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index d626699..8a0cf27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -2197,18 +2197,11 @@ public class RouterRpcServer extends AbstractService private Map<String, Long> getMountPointDates(String path) { Map<String, Long> ret = new TreeMap<>(); if (subclusterResolver instanceof MountTableResolver) { - MountTableResolver mountTable = (MountTableResolver)subclusterResolver; - String srcPath; try { final List<String> children = subclusterResolver.getMountPoints(path); for (String child : children) { - if (path.equals(Path.SEPARATOR)) { - srcPath = Path.SEPARATOR + child; - } else { - srcPath = path + Path.SEPARATOR + child; - } - MountTable entry = mountTable.getMountPoint(srcPath); - ret.put(child, entry.getDateModified()); + Long modTime = getModifiedTime(ret, path, child); + ret.put(child, modTime); } } catch (IOException e) { LOG.error("Cannot get mount point", e); @@ -2218,6 +2211,50 @@ public class RouterRpcServer extends AbstractService } /** + * Get modified time for child. If the child is present in mount table it + * will return the modified time. If the child is not present but subdirs of + * this child are present then it will return latest modified subdir's time + * as modified time of the requested child. + * @param ret contains children and modified times. + * @param mountTable. + * @param path Name of the path to start checking dates from. + * @param child child of the requested path. + * @return modified time. + */ + private long getModifiedTime(Map<String, Long> ret, String path, + String child) { + MountTableResolver mountTable = (MountTableResolver)subclusterResolver; + String srcPath; + if (path.equals(Path.SEPARATOR)) { + srcPath = Path.SEPARATOR + child; + } else { + srcPath = path + Path.SEPARATOR + child; + } + Long modTime = 0L; + try { + // Get mount table entry for the srcPath + MountTable entry = mountTable.getMountPoint(srcPath); + // if srcPath is not in mount table but its subdirs are in mount + // table we will display latest modified subdir date/time. + if (entry == null) { + List<MountTable> entries = mountTable.getMounts(srcPath); + for (MountTable eachEntry : entries) { + // Get the latest date + if (ret.get(child) == null || + ret.get(child) < eachEntry.getDateModified()) { + modTime = eachEntry.getDateModified(); + } + } + } else { + modTime = entry.getDateModified(); + } + } catch (IOException e) { + LOG.error("Cannot get mount point", e); + } + return modTime; + } + + /** * Create a new file status for a mount point. * * @param name Name of the mount point. http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c17359/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java index b33b998..4d8ffe1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java @@ -24,6 +24,7 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -167,6 +168,12 @@ public class TestRouterMountTable { addEntry = MountTable.newInstance( "/testdir/subdir", Collections.singletonMap("ns0", "/testdir/subdir")); assertTrue(addMountTable(addEntry)); + addEntry = MountTable.newInstance( + "/testdir3/subdir1", Collections.singletonMap("ns0", "/testdir3")); + assertTrue(addMountTable(addEntry)); + addEntry = MountTable.newInstance( + "/testA/testB/testC/testD", Collections.singletonMap("ns0", "/test")); + assertTrue(addMountTable(addEntry)); // Create test dir in NN final FileSystem nnFs = nnContext.getFileSystem(); @@ -174,8 +181,18 @@ public class TestRouterMountTable { Map<String, Long> pathModTime = new TreeMap<>(); for (String mount : mountTable.getMountPoints("/")) { - pathModTime.put(mount, mountTable.getMountPoint("/"+mount) - .getDateModified()); + if (mountTable.getMountPoint("/"+mount) != null) { + pathModTime.put(mount, mountTable.getMountPoint("/"+mount) + .getDateModified()); + } else { + List<MountTable> entries = mountTable.getMounts("/"+mount); + for (MountTable entry : entries) { + if (pathModTime.get(mount) == null || + pathModTime.get(mount) < entry.getDateModified()) { + pathModTime.put(mount, entry.getDateModified()); + } + } + } } FileStatus[] iterator = nnFs.listStatus(new Path("/")); for (FileStatus file : iterator) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org