Repository: incubator-twill Updated Branches: refs/heads/master f88e18f75 -> ef8b1eae8
(TWILL-154) Remove hardcoded check for âhdfsâ or âmaprfsâ URI prefix Signed-off-by: Terence Yim <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/ef8b1eae Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/ef8b1eae Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/ef8b1eae Branch: refs/heads/master Commit: ef8b1eae804db155dd05b20163e507521694647b Parents: f88e18f Author: Terence Yim <[email protected]> Authored: Fri Oct 9 13:42:22 2015 -0700 Committer: Terence Yim <[email protected]> Committed: Fri Oct 9 15:36:40 2015 -0700 ---------------------------------------------------------------------- .../org/apache/twill/internal/ServiceMain.java | 21 +++++++++----------- .../apache/twill/yarn/YarnTwillPreparer.java | 5 +++-- 2 files changed, 12 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ef8b1eae/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java index f7bea24..a6af3d3 100644 --- a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java +++ b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java @@ -136,20 +136,17 @@ public abstract class ServiceMain { return new LocalLocationFactory().create(appDir); } - if ("hdfs".equals(appDir.getScheme()) || "maprfs".equals(appDir.getScheme())) { - if (UserGroupInformation.isSecurityEnabled()) { - return new HDFSLocationFactory(FileSystem.get(appDir, conf)).create(appDir); - } - - String fsUser = System.getenv(EnvKeys.TWILL_FS_USER); - if (fsUser == null) { - throw new IllegalStateException("Missing environment variable " + EnvKeys.TWILL_FS_USER); - } - return new HDFSLocationFactory(FileSystem.get(appDir, conf, fsUser)).create(appDir); + // If not file, assuming it is a FileSystem, hence construct with HDFSLocationFactory which wraps + // a FileSystem created from the Configuration + if (UserGroupInformation.isSecurityEnabled()) { + return new HDFSLocationFactory(FileSystem.get(appDir, conf)).create(appDir); } - LOG.warn("Unsupported location type {}.", appDir); - throw new IllegalArgumentException("Unsupported location type " + appDir); + String fsUser = System.getenv(EnvKeys.TWILL_FS_USER); + if (fsUser == null) { + throw new IllegalStateException("Missing environment variable " + EnvKeys.TWILL_FS_USER); + } + return new HDFSLocationFactory(FileSystem.get(appDir, conf, fsUser)).create(appDir); } catch (Exception e) { LOG.error("Failed to create application location for {}.", appDir); http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ef8b1eae/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java index a444dda..ea77116 100644 --- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java +++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java @@ -52,6 +52,7 @@ import org.apache.twill.api.TwillPreparer; import org.apache.twill.api.TwillSpecification; import org.apache.twill.api.logging.LogEntry; import org.apache.twill.api.logging.LogHandler; +import org.apache.twill.filesystem.HDFSLocationFactory; import org.apache.twill.filesystem.Location; import org.apache.twill.filesystem.LocationFactory; import org.apache.twill.internal.ApplicationBundler; @@ -451,8 +452,8 @@ final class YarnTwillPreparer implements TwillPreparer { Location location; URI uri = localFile.getURI(); - if ("hdfs".equals(uri.getScheme())) { - // Assuming the location factory is HDFS one. If it is not, it will failed, which is the correct behavior. + if (locationFactory.getHomeLocation().toURI().getScheme().equals(uri.getScheme())) { + // If the source file location is having the same scheme as the target location, no need to copy location = locationFactory.create(uri); } else { URL url = uri.toURL();
