Repository: incubator-reef Updated Branches: refs/heads/master 23ed475e9 -> b9b8dca4a
[REEF-794] Enable reporting progress to YARN This adds a `ProgressProvider` interface to allow Driver implementations to report their progress. JIRA: [REEF-794](https://issues.apache.org/jira/browse/REEF-794) Pull Request: This closes #533 Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/b9b8dca4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/b9b8dca4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/b9b8dca4 Branch: refs/heads/master Commit: b9b8dca4aabed02554d631a33e8b1d3be4e04b61 Parents: 23ed475 Author: Andrew Chung <[email protected]> Authored: Thu Oct 1 16:55:48 2015 -0700 Committer: Markus Weimer <[email protected]> Committed: Fri Oct 2 10:31:06 2015 -0700 ---------------------------------------------------------------------- .../apache/reef/client/DriverConfiguration.java | 7 +++ .../reef/driver/DefaultProgressProvider.java | 45 ++++++++++++++++++++ .../apache/reef/driver/ProgressProvider.java | 38 +++++++++++++++++ .../yarn/driver/YarnContainerManager.java | 8 +++- .../webserver/HttpServerReefEventHandler.java | 2 + 5 files changed, 98 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/b9b8dca4/lang/java/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java b/lang/java/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java index 49e55b9..6dc9c55 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java @@ -21,6 +21,7 @@ package org.apache.reef.client; import org.apache.reef.annotations.Provided; import org.apache.reef.annotations.audience.ClientSide; import org.apache.reef.annotations.audience.Public; +import org.apache.reef.driver.ProgressProvider; import org.apache.reef.driver.context.ActiveContext; import org.apache.reef.driver.context.ClosedContext; import org.apache.reef.driver.context.ContextMessage; @@ -177,6 +178,11 @@ public final class DriverConfiguration extends ConfigurationModuleBuilder { public static final OptionalImpl<EventHandler<ContextMessage>> ON_CONTEXT_MESSAGE = new OptionalImpl<>(); /** + * @see {@link ProgressProvider} + */ + public static final OptionalImpl<ProgressProvider> PROGRESS_PROVIDER = new OptionalImpl<>(); + + /** * Number of threads allocated per evaluator to dispatch events from this Evaluator. */ public static final OptionalParameter<Integer> EVALUATOR_DISPATCHER_THREADS = new OptionalParameter<>(); @@ -229,5 +235,6 @@ public final class DriverConfiguration extends ConfigurationModuleBuilder { // Various parameters .bindNamedParameter(EvaluatorDispatcherThreads.class, EVALUATOR_DISPATCHER_THREADS) + .bindImplementation(ProgressProvider.class, PROGRESS_PROVIDER) .build(); } http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/b9b8dca4/lang/java/reef-common/src/main/java/org/apache/reef/driver/DefaultProgressProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/driver/DefaultProgressProvider.java b/lang/java/reef-common/src/main/java/org/apache/reef/driver/DefaultProgressProvider.java new file mode 100644 index 0000000..ee05681 --- /dev/null +++ b/lang/java/reef-common/src/main/java/org/apache/reef/driver/DefaultProgressProvider.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.driver; + +import org.apache.reef.annotations.audience.DriverSide; +import org.apache.reef.annotations.audience.Public; + +import javax.inject.Inject; + +/** + * The default implementation of {@link ProgressProvider}. Always returns a + * progress of 0. + */ +@Public +@DriverSide +public final class DefaultProgressProvider implements ProgressProvider { + + @Inject + private DefaultProgressProvider(){ + } + + /** + * @return 0 as a default. + */ + @Override + public float getProgress() { + return 0; + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/b9b8dca4/lang/java/reef-common/src/main/java/org/apache/reef/driver/ProgressProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/driver/ProgressProvider.java b/lang/java/reef-common/src/main/java/org/apache/reef/driver/ProgressProvider.java new file mode 100644 index 0000000..a2d5886 --- /dev/null +++ b/lang/java/reef-common/src/main/java/org/apache/reef/driver/ProgressProvider.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.driver; + +import org.apache.reef.annotations.audience.DriverSide; +import org.apache.reef.annotations.audience.Public; +import org.apache.reef.tang.annotations.DefaultImplementation; + +/** + * Used in the to report the progress of the REEF application. + * The default implementation always returns a progress of 0. + */ +@Public +@DriverSide +@DefaultImplementation(DefaultProgressProvider.class) +public interface ProgressProvider { + + /** + * @return The progress of the REEF application. + */ + float getProgress(); +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/b9b8dca4/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java index 028744f..a886eb1 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.reef.driver.ProgressProvider; import org.apache.reef.driver.parameters.JobSubmissionDirectory; import org.apache.reef.exception.DriverFatalRuntimeException; import org.apache.reef.proto.ReefServiceProtos; @@ -81,6 +82,7 @@ final class YarnContainerManager private final String jobSubmissionDirectory; private final REEFFileNames reefFileNames; private final RackNameFormatter rackNameFormatter; + private final ProgressProvider progressProvider; @Inject YarnContainerManager( @@ -94,7 +96,8 @@ final class YarnContainerManager final REEFFileNames reefFileNames, @Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory, final TrackingURLProvider trackingURLProvider, - final RackNameFormatter rackNameFormatter) throws IOException { + final RackNameFormatter rackNameFormatter, + final ProgressProvider progressProvider) throws IOException { this.reefEventHandlers = reefEventHandlers; this.driverStatusManager = driverStatusManager; @@ -112,6 +115,7 @@ final class YarnContainerManager this.nodeManager = new NMClientAsyncImpl(this); this.jobSubmissionDirectory = jobSubmissionDirectory; this.reefFileNames = reefFileNames; + this.progressProvider = progressProvider; LOG.log(Level.FINEST, "Instantiated YarnContainerManager"); } @@ -158,7 +162,7 @@ final class YarnContainerManager @Override public float getProgress() { - return 0; // TODO: return actual values for progress + return progressProvider.getProgress(); } @Override http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/b9b8dca4/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java index 9055130..ad6f9b0 100644 --- a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java +++ b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java @@ -185,6 +185,8 @@ public final class HttpServerReefEventHandler implements HttpHandler { response.getWriter().println(String.format("Cannot find the log file: [%s].", fileName)); } break; + // TODO[JIRA REEF-798] Use this provider in the HTTP + case "progress": default: response.getWriter().println(String.format("Unsupported query for entity: [%s].", target)); }
