Repository: hadoop Updated Branches: refs/heads/HDFS-1312 90a032653 -> 62e4dcd2d
YARN-4958. The file localization process should allow for wildcards to reduce the application footprint in the state store (Daniel Templeton via sjlee) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5107a967 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5107a967 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5107a967 Branch: refs/heads/HDFS-1312 Commit: 5107a967fa2558deba11c33a326d4d2e5748f452 Parents: fc6b50c Author: Sangjin Lee <sj...@twitter.com> Authored: Mon Jun 20 09:56:53 2016 -0700 Committer: Sangjin Lee <sj...@twitter.com> Committed: Mon Jun 20 09:56:53 2016 -0700 ---------------------------------------------------------------------- .../yarn/server/nodemanager/ContainerExecutor.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5107a967/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 98171af..98d45f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -61,7 +61,7 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; public abstract class ContainerExecutor implements Configurable { - + private static final String WILDCARD = "*"; private static final Log LOG = LogFactory.getLog(ContainerExecutor.class); final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION = FsPermission.createImmutable((short) 0700); @@ -281,7 +281,18 @@ public abstract class ContainerExecutor implements Configurable { if (resources != null) { for (Map.Entry<Path,List<String>> entry : resources.entrySet()) { for (String linkName : entry.getValue()) { - sb.symlink(entry.getKey(), new Path(linkName)); + if (new Path(linkName).getName().equals(WILDCARD)) { + // If this is a wildcarded path, link to everything in the + // directory from the working directory + File directory = new File(entry.getKey().toString()); + + for (File wildLink : directory.listFiles()) { + sb.symlink(new Path(wildLink.toString()), + new Path(wildLink.getName())); + } + } else { + sb.symlink(entry.getKey(), new Path(linkName)); + } } } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
