(TWILL-144) No need to rename resource name when localization to YARN This closes #57 on GitHub
Signed-off-by: Terence Yim <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/bb0212d4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/bb0212d4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/bb0212d4 Branch: refs/heads/site Commit: bb0212d415be891b53b40e3ec928f742198762fd Parents: 0c20804 Author: Terence Yim <[email protected]> Authored: Mon Jul 27 21:15:46 2015 -0700 Committer: Terence Yim <[email protected]> Committed: Thu Jul 30 12:58:43 2015 -0700 ---------------------------------------------------------------------- .../ApplicationMasterProcessLauncher.java | 5 ----- .../internal/container/TwillContainerMain.java | 13 ------------- .../yarn/AbstractYarnProcessLauncher.java | 20 +------------------- 3 files changed, 1 insertion(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/bb0212d4/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java index 4112080..126ff97 100644 --- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java +++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java @@ -43,11 +43,6 @@ public final class ApplicationMasterProcessLauncher extends AbstractYarnProcessL } @Override - protected boolean useArchiveSuffix() { - return false; - } - - @Override @SuppressWarnings("unchecked") protected <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext) { final ApplicationId appId = getContainerInfo(); http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/bb0212d4/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java index e04c8f9..51837a7 100644 --- a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java +++ b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java @@ -86,7 +86,6 @@ public final class TwillContainerMain extends ServiceMain { ZKClient appRunZkClient = getAppRunZKClient(zkClientService, appRunId); TwillSpecification twillSpec = loadTwillSpec(twillSpecFile); - renameLocalFiles(twillSpec.getRunnables().get(runnableName)); TwillRunnableSpecification runnableSpec = twillSpec.getRunnables().get(runnableName).getRunnableSpecification(); ContainerInfo containerInfo = new EnvContainerInfo(); @@ -135,18 +134,6 @@ public final class TwillContainerMain extends ServiceMain { } } - private static void renameLocalFiles(RuntimeSpecification runtimeSpec) { - for (LocalFile file : runtimeSpec.getLocalFiles()) { - if (file.isArchive()) { - String path = file.getURI().toString(); - String name = file.getName() + (path.endsWith(".tar.gz") ? ".tar.gz" : path.substring(path.lastIndexOf('.'))); - Preconditions.checkState(new File(name).renameTo(new File(file.getName())), - "Fail to rename file from %s to %s.", - name, file.getName()); - } - } - } - /** * Returns a {@link ZKClient} that namespaced under the given run id. */ http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/bb0212d4/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java index ebbb559..2023b0e 100644 --- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java +++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java @@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.twill.api.LocalFile; import org.apache.twill.internal.ProcessController; import org.apache.twill.internal.ProcessLauncher; -import org.apache.twill.internal.utils.Paths; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,13 +64,6 @@ public abstract class AbstractYarnProcessLauncher<T> implements ProcessLauncher< } /** - * Tells whether to append suffix to localize resource name for archive file type. Default is true. - */ - protected boolean useArchiveSuffix() { - return true; - } - - /** * For children class to override to perform actual process launching. */ protected abstract <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext); @@ -100,17 +92,7 @@ public abstract class AbstractYarnProcessLauncher<T> implements ProcessLauncher< } private void addLocalFile(LocalFile localFile) { - String name = localFile.getName(); - // Always append the file extension as the resource name so that archive expansion by Yarn could work. - // Renaming would happen by the Container Launcher. - if (localFile.isArchive() && useArchiveSuffix()) { - String path = localFile.getURI().toString(); - String suffix = Paths.getExtension(path); - if (!suffix.isEmpty()) { - name += '.' + suffix; - } - } - localResources.put(name, YarnUtils.createLocalResource(localFile)); + localResources.put(localFile.getName(), YarnUtils.createLocalResource(localFile)); } @Override
