[
https://issues.apache.org/jira/browse/TWILL-63?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14997591#comment-14997591
]
ASF GitHub Bot commented on TWILL-63:
-------------------------------------
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/incubator-twill/pull/66#discussion_r44346230
--- Diff:
twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java ---
@@ -145,41 +145,63 @@ public void createBundle(Location target, Class<?>
clz, Class<?>...classes) thro
* @throws IOException
*/
public void createBundle(Location target, Iterable<Class<?>> classes,
Iterable<URI> resources) throws IOException {
- LOG.debug("start creating bundle {}. building a temporary file locally
at first", target.getName());
+ createBundleAndGetClassPathUrls(target, classes, resources);
+ }
+
+ public Set<URL> createBundleAndGetClassPathUrls(Location target,
Iterable<Class<?>> classes,
+ Iterable<URI> resources)
throws IOException {
+ return createBundleAndGetClassPathUrls(target.getName(),
target.toString(),
+ target.getOutputStream(),
classes, resources);
+ }
+
+ private Set<URL> createBundleAndGetClassPathUrls(String targetName,
String targetPath,
+ OutputStream
targetOutputStream,
+ Iterable<Class<?>>
classes,
+ Iterable<URI>
resources) throws IOException {
+ LOG.debug("start creating bundle {}. building a temporary file locally
at first", targetName);
// Write the jar to local tmp file first
- File tmpJar = File.createTempFile(target.getName(), ".tmp");
+ File tmpJar = File.createTempFile(targetName, ".tmp");
+ Set<URL> classPathUrls;
try {
Set<String> entries = Sets.newHashSet();
try (JarOutputStream jarOut = new JarOutputStream(new
FileOutputStream(tmpJar))) {
// Find class dependencies
- findDependencies(classes, entries, jarOut);
+ classPathUrls = findDependenciesAndGetClassPathUrls(classes,
entries, jarOut);
// Add extra resources
for (URI resource : resources) {
copyResource(resource, entries, jarOut);
}
}
- LOG.debug("copying temporary bundle to destination {} ({} bytes)",
target, tmpJar.length());
+ LOG.debug("copying temporary bundle to destination {} ({} bytes)",
targetPath, tmpJar.length());
// Copy the tmp jar into destination.
try {
- OutputStream os = new
BufferedOutputStream(target.getOutputStream());
+ OutputStream os = new BufferedOutputStream(targetOutputStream);
try {
Files.copy(tmpJar, os);
} finally {
Closeables.closeQuietly(os);
}
} catch (IOException e) {
- throw new IOException("failed to copy bundle from " +
tmpJar.toURI() + " to " + target, e);
+ throw new IOException("failed to copy bundle from " +
tmpJar.toURI() + " to " + targetPath, e);
}
- LOG.debug("finished creating bundle at {}", target);
+ LOG.debug("finished creating bundle at {}", targetPath);
} finally {
tmpJar.delete();
LOG.debug("cleaned up local temporary for bundle {}",
tmpJar.toURI());
}
+ return classPathUrls;
+
--- End diff --
remove extra new line.
> Speed up application launch time
> --------------------------------
>
> Key: TWILL-63
> URL: https://issues.apache.org/jira/browse/TWILL-63
> Project: Apache Twill
> Issue Type: Improvement
> Components: yarn
> Affects Versions: 0.2.0-incubating
> Reporter: Terence Yim
> Assignee: Shankar Selvam
> Fix For: 0.7.0-incubating
>
>
> Currently when launching an application, two new jars are always created
> locally, one for AM (appMaster.jar) and one for Container (container.jar) and
> copied to HDFS before submitting the application. The jar files could
> potentially be big and if it doesn't changed, it should require copying to
> HDFS again.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
