Repository: incubator-reef Updated Branches: refs/heads/master 429dc5e07 -> 8c4bb098c
[REEF-348] Ignore and log changed timestamps on global.jar This change checks the time stamp on the `global.jar` for each Evaluator launch. If it changed, the change will be logged as a WARNING. *Note:* This doesn't address the root cause of [REEF-348], but it should make the code more reliable. Also, the logging will allow us to debug this further. JIRA: [REEF-348](https://issues.apache.org/jira/browse/REEF-348) Pull Request: This closes #196 Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/8c4bb098 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/8c4bb098 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/8c4bb098 Branch: refs/heads/master Commit: 8c4bb098c248a484e4f1e6cf1fbb742f7a6fd7fe Parents: 429dc5e Author: afchung <[email protected]> Authored: Thu May 28 16:03:59 2015 -0700 Committer: Markus Weimer <[email protected]> Committed: Thu May 28 17:54:38 2015 -0700 ---------------------------------------------------------------------- .../runtime/yarn/driver/GlobalJarUploader.java | 52 +++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/8c4bb098/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java index c6ee91e..a0f4f81 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java @@ -29,6 +29,8 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Utility class that creates the JAR file with the global files on the driver and then uploads it to the job folder on @@ -36,22 +38,34 @@ import java.util.concurrent.Callable; */ final class GlobalJarUploader implements Callable<Map<String, LocalResource>> { + private static final Logger LOG = Logger.getLogger(GlobalJarUploader.class.getName()); + /** * Used for the file system constants. */ private final REEFFileNames fileNames; - /** - * This will hold the actuall map to be used as the "global" resources when submitting Evaluators. - */ - private final Map<String, LocalResource> globalResources = new HashMap<>(1); + /** * Utility to actually perform the update. */ private final UploaderToJobFolder uploader; + /** - * True, if globalResources contains the valid information which is cached after the first call to call(). + * True, if the global JAR has already been uploaded. */ - private boolean isDone; + private boolean isUploaded; + + /** + * Path to the uploaded global JAR. + */ + private Path pathToGlobalJar; + + /** + * The cached LocalResource for global JAR. The latest global JAR + * is still updated due to REEF-348. + * This is primarily used to detect change in JAR timestamps. + */ + private LocalResource globalJarResource; @Inject GlobalJarUploader(final REEFFileNames fileNames, @@ -69,13 +83,27 @@ final class GlobalJarUploader implements Callable<Map<String, LocalResource>> { */ @Override public synchronized Map<String, LocalResource> call() throws IOException { - if (!this.isDone) { - final Path pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar()); - globalResources.put(this.fileNames.getGlobalFolderPath(), - this.uploader.makeLocalResourceForJarFile(pathToGlobalJar)); - this.isDone = true; + final Map<String, LocalResource> globalResources = new HashMap<>(1); + if (!this.isUploaded){ + this.pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar()); + this.isUploaded = true; + } + + LocalResource updatedGlobalJarResource = this.uploader.makeLocalResourceForJarFile(this.pathToGlobalJar); + + if (this.globalJarResource != null + && this.globalJarResource.getTimestamp() != updatedGlobalJarResource.getTimestamp()) { + LOG.log(Level.WARNING, + "The global JAR LocalResource timestamp has been changed from " + + this.globalJarResource.getTimestamp() + " to " + updatedGlobalJarResource.getTimestamp()); } - return this.globalResources; + + this.globalJarResource = updatedGlobalJarResource; + + // For now, always rewrite the information due to REEF-348 + globalResources.put(this.fileNames.getGlobalFolderPath(), updatedGlobalJarResource); + + return globalResources; } /**
