MAPREDUCE-6862. Fragments are not handled correctly by resource limit checking. (Chris Trezzo via mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ceab00ac Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ceab00ac Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ceab00ac Branch: refs/heads/HDFS-7240 Commit: ceab00ac62f8057a07b4b936799e6f04271e6e41 Parents: 6a5516c Author: Ming Ma <[email protected]> Authored: Wed Mar 29 17:41:58 2017 -0700 Committer: Ming Ma <[email protected]> Committed: Wed Mar 29 17:41:58 2017 -0700 ---------------------------------------------------------------------- .../hadoop/mapreduce/JobResourceUploader.java | 36 ++++++++++++------ .../mapreduce/TestJobResourceUploader.java | 40 +++++++++++++++++--- 2 files changed, 59 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceab00ac/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java index 4c48ff4..085c966 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java @@ -238,28 +238,42 @@ class JobResourceUploader { Collection<String> dcArchives = conf.getStringCollection(MRJobConfig.CACHE_ARCHIVES); - for (String path : dcFiles) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : dcFiles) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } - for (String path : dcArchives) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : dcArchives) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } - for (String path : files) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : files) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } - for (String path : libjars) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : libjars) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } - for (String path : archives) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : archives) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } if (jobJar != null) { - explorePath(conf, new Path(jobJar), limitChecker, statCache); + explorePath(conf, stringToPath(jobJar), limitChecker, statCache); + } + } + + /** + * Convert a String to a Path and gracefully remove fragments/queries if they + * exist in the String. + */ + @VisibleForTesting + Path stringToPath(String s) { + try { + URI uri = new URI(s); + return new Path(uri.getScheme(), uri.getAuthority(), uri.getPath()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceab00ac/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java index 36ea57a..8ba50a6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java @@ -40,6 +40,34 @@ import org.junit.Test; public class TestJobResourceUploader { @Test + public void testStringToPath() throws IOException { + Configuration conf = new Configuration(); + JobResourceUploader uploader = + new JobResourceUploader(FileSystem.getLocal(conf), false); + + Assert.assertEquals("Failed: absolute, no scheme, with fragment", + "/testWithFragment.txt", + uploader.stringToPath("/testWithFragment.txt#fragment.txt").toString()); + + Assert.assertEquals("Failed: absolute, with scheme, with fragment", + "file:/testWithFragment.txt", + uploader.stringToPath("file:///testWithFragment.txt#fragment.txt") + .toString()); + + Assert.assertEquals("Failed: relative, no scheme, with fragment", + "testWithFragment.txt", + uploader.stringToPath("testWithFragment.txt#fragment.txt").toString()); + + Assert.assertEquals("Failed: relative, no scheme, no fragment", + "testWithFragment.txt", + uploader.stringToPath("testWithFragment.txt").toString()); + + Assert.assertEquals("Failed: absolute, with scheme, no fragment", + "file:/testWithFragment.txt", + uploader.stringToPath("file:///testWithFragment.txt").toString()); + } + + @Test public void testAllDefaults() throws IOException { ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); runLimitsTest(b.build(), true, null); @@ -210,17 +238,17 @@ public class TestJobResourceUploader { rlConf.maxSingleResourceMB); conf.set("tmpfiles", - buildPathString("file://tmpFiles", rlConf.numOfTmpFiles)); + buildPathString("file:///tmpFiles", rlConf.numOfTmpFiles)); conf.set("tmpjars", - buildPathString("file://tmpjars", rlConf.numOfTmpLibJars)); + buildPathString("file:///tmpjars", rlConf.numOfTmpLibJars)); conf.set("tmparchives", - buildPathString("file://tmpArchives", rlConf.numOfTmpArchives)); + buildPathString("file:///tmpArchives", rlConf.numOfTmpArchives)); conf.set(MRJobConfig.CACHE_ARCHIVES, - buildPathString("file://cacheArchives", rlConf.numOfDCArchives)); + buildPathString("file:///cacheArchives", rlConf.numOfDCArchives)); conf.set(MRJobConfig.CACHE_FILES, - buildPathString("file://cacheFiles", rlConf.numOfDCFiles)); + buildPathString("file:///cacheFiles", rlConf.numOfDCFiles)); if (rlConf.jobJar) { - conf.setJar("file://jobjar.jar"); + conf.setJar("file:///jobjar.jar"); } return conf; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
