Repository: incubator-reef
Updated Branches:
refs/heads/master 8587b42dc -> 0e47fff57
[REEF-675] EvaluatorRequest(or) related cleanups.
* Made fields in `IEvaluatorRequest` immutable and documented them.
* Removed capabilities-related APIs from `IEvaluatorRequest` as they
never work.
* Added `IEvaluatorRequestor.NewBuilder()` and used it in all drivers.
* Fixed a typo in `EvaluatorRequstor.cs`
* Deprecated `DefaultEvaluatorRequestorHandler`
* Deprecated `IFailedEvaluator.EvaluatorRequetor`
JIRAs:
[REEF-675](https://issues.apache.org/jira/browse/REEF-675)
[REEF-671](https://issues.apache.org/jira/browse/REEF-671)
Pull Request:
This closes #465
Author: Markus Weimer <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/0e47fff5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/0e47fff5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/0e47fff5
Branch: refs/heads/master
Commit: 0e47fff5716f6d2500fceb51a5895ae025241cd3
Parents: 8587b42
Author: Markus Weimer <[email protected]>
Authored: Thu Sep 3 17:53:49 2015 -0700
Committer: Julia Wang <[email protected]>
Committed: Fri Sep 4 11:35:24 2015 -0700
----------------------------------------------------------------------
.../Bridge/Events/EvaluatorRequestor.cs | 108 +++++++++++++++++++
.../Bridge/Events/EvaluatorRequstor.cs | 96 -----------------
.../DefaultEvaluatorRequestorHandler.cs | 1 +
.../Evaluator/EvaluatorRequest.cs | 47 ++++----
.../Evaluator/EvaluatorRequestBuilder.cs | 91 ++++++++++++----
.../Evaluator/IEvaluatorRequest.cs | 44 ++++----
.../Evaluator/IEvaluatorRequestor.cs | 22 ++--
.../Evaluator/IFailedEvaluator.cs | 2 +
.../Org.Apache.REEF.Driver.csproj | 2 +-
.../HelloDriverStartHandler.cs | 18 +++-
.../HelloFailedEvaluatorHandler.cs | 15 +--
.../HelloDriver.cs | 2 +-
.../DriverRestart/HelloRestartDriver.cs | 2 +-
.../KMeans/KMeansDriverHandlers.cs | 4 +-
.../OnREEF/Driver/IMRUDriver.cs | 16 ++-
.../BroadcastReduceDriver.cs | 9 +-
.../PipelinedBroadcastReduceDriver.cs | 8 +-
.../ScatterReduceDriver.cs | 8 +-
.../Bridge/HelloSimpleEventHandlers.cs | 10 +-
.../Functional/Messaging/MessageDriver.cs | 9 +-
20 files changed, 326 insertions(+), 188 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/EvaluatorRequestor.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/EvaluatorRequestor.cs
b/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/EvaluatorRequestor.cs
new file mode 100644
index 0000000..3fc4cb7
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/EvaluatorRequestor.cs
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Runtime.Serialization;
+using Org.Apache.REEF.Common.Catalog;
+using Org.Apache.REEF.Common.Evaluator;
+using Org.Apache.REEF.Driver.Bridge.Clr2java;
+using Org.Apache.REEF.Driver.Evaluator;
+using Org.Apache.REEF.Utilities.Diagnostics;
+using Org.Apache.REEF.Utilities.Logging;
+
+namespace Org.Apache.REEF.Driver.Bridge.Events
+{
+ [DataContract]
+ internal class EvaluatorRequestor : IEvaluatorRequestor
+ {
+ private static readonly Logger LOGGER =
Logger.GetLogger(typeof(EvaluatorRequestor));
+
+ private static Dictionary<string, IEvaluatorDescriptor> _evaluators;
+
+ internal EvaluatorRequestor(IEvaluatorRequestorClr2Java clr2Java)
+ {
+ InstanceId = Guid.NewGuid().ToString("N");
+ Clr2Java = clr2Java;
+ }
+
+ internal static Dictionary<string, IEvaluatorDescriptor> Evaluators
+ {
+ get
+ {
+ if (_evaluators == null)
+ {
+ _evaluators = new Dictionary<string,
IEvaluatorDescriptor>();
+ }
+ return _evaluators;
+ }
+ }
+
+ public IResourceCatalog ResourceCatalog { get; set; }
+
+ [DataMember]
+ public string InstanceId { get; set; }
+
+ [DataMember]
+ private IEvaluatorRequestorClr2Java Clr2Java { get; set; }
+
+ public void Submit(IEvaluatorRequest request)
+ {
+ LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture,
"Submitting request for {0} evaluators and {1} MB memory and {2} core to rack
{3}.", request.Number, request.MemoryMegaBytes, request.VirtualCore,
request.Rack));
+
+ lock (Evaluators)
+ {
+ for (int i = 0; i < request.Number; i++)
+ {
+ EvaluatorDescriptorImpl descriptor = new
EvaluatorDescriptorImpl(new NodeDescriptorImpl(), EvaluatorType.CLR,
request.MemoryMegaBytes, request.VirtualCore);
+ descriptor.Rack = request.Rack;
+ string key = string.Format(CultureInfo.InvariantCulture,
"{0}_{1}", request.EvaluatorBatchId, i);
+ try
+ {
+ _evaluators.Add(key, descriptor);
+ }
+ catch (ArgumentException e)
+ {
+ Exceptions.Caught(e, Level.Error,
string.Format(CultureInfo.InvariantCulture, "EvaluatorBatchId [{0}] already
exists.", key), LOGGER);
+ Exceptions.Throw(new InvalidOperationException("Cannot
use evaluator id " + key, e), LOGGER);
+ }
+ }
+ }
+
+ Clr2Java.Submit(request);
+ }
+
+ public EvaluatorRequestBuilder NewBuilder()
+ {
+ return new EvaluatorRequestBuilder();
+ }
+
+ public EvaluatorRequestBuilder NewBuilder(IEvaluatorRequest request)
+ {
+#pragma warning disable 618
+ return new EvaluatorRequestBuilder(request);
+#pragma warning restore 618
+ }
+
+ public void Dispose()
+ {
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/EvaluatorRequstor.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/EvaluatorRequstor.cs
b/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/EvaluatorRequstor.cs
deleted file mode 100644
index 0be5c9b..0000000
--- a/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/EvaluatorRequstor.cs
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Runtime.Serialization;
-using Org.Apache.REEF.Common.Catalog;
-using Org.Apache.REEF.Common.Evaluator;
-using Org.Apache.REEF.Driver.Bridge.Clr2java;
-using Org.Apache.REEF.Driver.Evaluator;
-using Org.Apache.REEF.Utilities.Diagnostics;
-using Org.Apache.REEF.Utilities.Logging;
-
-namespace Org.Apache.REEF.Driver.Bridge.Events
-{
- [DataContract]
- internal class EvaluatorRequestor : IEvaluatorRequestor
- {
- private static readonly Logger LOGGER =
Logger.GetLogger(typeof(EvaluatorRequestor));
-
- private static Dictionary<string, IEvaluatorDescriptor> _evaluators;
-
- public EvaluatorRequestor(IEvaluatorRequestorClr2Java clr2Java)
- {
- InstanceId = Guid.NewGuid().ToString("N");
- Clr2Java = clr2Java;
- }
-
- public static Dictionary<string, IEvaluatorDescriptor> Evaluators
- {
- get
- {
- if (_evaluators == null)
- {
- _evaluators = new Dictionary<string,
IEvaluatorDescriptor>();
- }
- return _evaluators;
- }
- }
-
- public IResourceCatalog ResourceCatalog { get; set; }
-
- [DataMember]
- public string InstanceId { get; set; }
-
- [DataMember]
- private IEvaluatorRequestorClr2Java Clr2Java { get; set; }
-
- public void Submit(IEvaluatorRequest request)
- {
- LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture,
"Submitting request for {0} evaluators and {1} MB memory and {2} core to rack
{3}.", request.Number, request.MemoryMegaBytes, request.VirtualCore,
request.Rack));
-
- lock (Evaluators)
- {
- for (int i = 0; i < request.Number; i++)
- {
- EvaluatorDescriptorImpl descriptor = new
EvaluatorDescriptorImpl(new NodeDescriptorImpl(), EvaluatorType.CLR,
request.MemoryMegaBytes, request.VirtualCore);
- descriptor.Rack = request.Rack;
- string key = string.Format(CultureInfo.InvariantCulture,
"{0}_{1}", request.EvaluatorBatchId, i);
- try
- {
- _evaluators.Add(key, descriptor);
- }
- catch (ArgumentException e)
- {
- Exceptions.Caught(e, Level.Error,
string.Format(CultureInfo.InvariantCulture, "EvaluatorBatchId [{0}] already
exists.", key), LOGGER);
- Exceptions.Throw(new InvalidOperationException("Cannot
use evaluator id " + key, e), LOGGER);
- }
- }
- }
-
- Clr2Java.Submit(request);
- }
-
- public void Dispose()
- {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultEvaluatorRequestorHandler.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultEvaluatorRequestorHandler.cs
b/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultEvaluatorRequestorHandler.cs
index 068b3cf..0617e29 100644
---
a/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultEvaluatorRequestorHandler.cs
+++
b/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultEvaluatorRequestorHandler.cs
@@ -27,6 +27,7 @@ namespace Org.Apache.REEF.Driver.Defaults
/// <summary>
/// Default handler for close messages from the client: logging it
/// </summary>
+ [Obsolete("In Version 0.13. Have an instance of IEvaluatorRequestor
injected instead.")]
public class DefaultEvaluatorRequestorHandler :
IObserver<IEvaluatorRequestor>
{
private static readonly Logger LOGGER =
Logger.GetLogger(typeof(DefaultClientCloseHandler));
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
index 3132bc7..c0d01dc 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
@@ -18,41 +18,48 @@
*/
using System;
-using System.Collections.Generic;
using System.Runtime.Serialization;
-using Org.Apache.REEF.Common.Catalog;
-using Org.Apache.REEF.Common.Catalog.Capabilities;
namespace Org.Apache.REEF.Driver.Evaluator
{
+ /// <summary>
+ /// Default implementation of IEvaluatorRequest.
+ /// </summary>
+ [Obsolete("This class will be internal after 0.13")]
[DataContract]
public class EvaluatorRequest : IEvaluatorRequest
{
- public EvaluatorRequest()
+ [Obsolete("This constructor will be internal after 0.13")]
+ public EvaluatorRequest()
: this(0, 0, 1, string.Empty, Guid.NewGuid().ToString("N"))
{
}
- public EvaluatorRequest(int number, int megaBytes)
+ [Obsolete("This constructor will be internal after 0.13")]
+ public EvaluatorRequest(int number, int megaBytes)
: this(number, megaBytes, 1, string.Empty,
Guid.NewGuid().ToString("N"))
{
}
+ [Obsolete("This constructor will be internal after 0.13")]
public EvaluatorRequest(int number, int megaBytes, int core)
: this(number, megaBytes, core, string.Empty,
Guid.NewGuid().ToString("N"))
{
}
+ [Obsolete("This constructor will be internal after 0.13")]
public EvaluatorRequest(int number, int megaBytes, string rack)
: this(number, megaBytes, 1, rack, Guid.NewGuid().ToString("N"))
{
}
+ [Obsolete("This constructor will be internal after 0.13")]
public EvaluatorRequest(int number, int megaBytes, int core, string
rack)
: this(number, megaBytes, core, rack, Guid.NewGuid().ToString("N"))
{
}
+ [Obsolete("This constructor will be internal after 0.13")]
public EvaluatorRequest(int number, int megaBytes, int core, string
rack, string evaluatorBatchId)
{
Number = number;
@@ -62,46 +69,34 @@ namespace Org.Apache.REEF.Driver.Evaluator
EvaluatorBatchId = evaluatorBatchId;
}
- public EvaluatorRequest(int number, int megaBytes, int core,
List<ICapability> capabilitieses, IResourceCatalog catalog)
- {
- Number = number;
- MemoryMegaBytes = megaBytes;
- Capabilities = capabilitieses;
- VirtualCore = core;
- Catalog = catalog;
- EvaluatorBatchId = Guid.NewGuid().ToString("N");
- }
-
[DataMember]
public string InstanceId { get; set; }
[DataMember]
- public int MemoryMegaBytes { get; set; }
+ public int MemoryMegaBytes { get; private set; }
[DataMember]
- public int Number { get; set; }
-
- [DataMember]
- public int VirtualCore { get; set; }
+ public int Number { get; private set; }
[DataMember]
- public string Rack { get; set; }
+ public int VirtualCore { get; private set; }
[DataMember]
- public string EvaluatorBatchId { get; set; }
+ public string Rack { get; private set; }
- public List<ICapability> Capabilities { get; set; }
-
- public IResourceCatalog Catalog { get; set; }
+ [DataMember]
+ public string EvaluatorBatchId { get; private set; }
+ [Obsolete("Use IEvaluatorRequestor.NewBuilder() instead.")]
public static EvaluatorRequestBuilder NewBuilder()
{
return new EvaluatorRequestBuilder();
}
+ [Obsolete("Use IEvaluatorRequestor.NewBuilder() instead.")]
public static EvaluatorRequestBuilder NewBuilder(EvaluatorRequest
request)
{
return new EvaluatorRequestBuilder(request);
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
index d1b4864..8d018b1 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
@@ -17,43 +17,98 @@
* under the License.
*/
-using System.Collections.Generic;
-using Org.Apache.REEF.Common.Catalog;
-using Org.Apache.REEF.Common.Catalog.Capabilities;
+using System;
namespace Org.Apache.REEF.Driver.Evaluator
{
- public class EvaluatorRequestBuilder
+ public sealed class EvaluatorRequestBuilder
{
- public EvaluatorRequestBuilder(EvaluatorRequest request)
+ private string _evaluatorBatchId;
+ private string _rackName;
+
+ [Obsolete("This constructor will be internal after 0.13.")]
+ public EvaluatorRequestBuilder(IEvaluatorRequest request)
{
- foreach (ICapability capability in request.Capabilities)
- {
- Capabilities.Add(capability);
- }
Number = request.Number;
- Catalog = request.Catalog;
MegaBytes = request.MemoryMegaBytes;
VirtualCore = request.VirtualCore;
+ _evaluatorBatchId = request.EvaluatorBatchId;
+ _rackName = request.Rack;
}
internal EvaluatorRequestBuilder()
{
+ Number = 1;
+ VirtualCore = 1;
+ MegaBytes = 64;
+ _rackName = String.Empty;
+ _evaluatorBatchId = Guid.NewGuid().ToString("N");
}
- public int Number { get; set; }
+ public int Number { get; private set; }
+ public int MegaBytes { get; private set; }
+ public int VirtualCore { get; private set; }
+
+ /// <summary>
+ /// Set the number of evaluators to request.
+ /// </summary>
+ /// <param name="number"></param>
+ /// <returns>this</returns>
+ public EvaluatorRequestBuilder SetNumber(int number)
+ {
+ Number = number;
+ return this;
+ }
- public List<ICapability> Capabilities { get; set; }
+ /// <summary>
+ /// Set the amount of memory (in MB) for the evaluator.
+ /// </summary>
+ /// <param name="megabytes"></param>
+ /// <returns>this</returns>
+ public EvaluatorRequestBuilder SetMegabytes(int megabytes)
+ {
+ MegaBytes = megabytes;
+ return this;
+ }
- public IResourceCatalog Catalog { get; set; }
+ /// <summary>
+ /// Set the number of CPU cores for the evaluator.
+ /// </summary>
+ /// <param name="numberOfCores"></param>
+ /// <returns>this</returns>
+ public EvaluatorRequestBuilder SetCores(int numberOfCores)
+ {
+ VirtualCore = numberOfCores;
+ return this;
+ }
- public int MegaBytes { get; set; }
+ /// <summary>
+ /// Set the rack name to do the request for.
+ /// </summary>
+ /// <param name="rackName"></param>
+ /// <returns>this</returns>
+ public EvaluatorRequestBuilder SetRackName(string rackName)
+ {
+ _rackName = rackName;
+ return this;
+ }
- public int VirtualCore { get; set; }
+ // TODO[REEF-718]: Document.
+ public EvaluatorRequestBuilder SetEvaluatorBatchId(string
evaluatorBatchId)
+ {
+ _evaluatorBatchId = evaluatorBatchId;
+ return this;
+ }
- public EvaluatorRequest Build()
+ /// <summary>
+ /// Build the EvaluatorRequest.
+ /// </summary>
+ /// <returns></returns>
+ public IEvaluatorRequest Build()
{
- return new EvaluatorRequest(Number, MegaBytes, VirtualCore,
Capabilities, Catalog);
+#pragma warning disable 618
+ return new EvaluatorRequest(Number, MegaBytes, VirtualCore,
rack:_rackName, evaluatorBatchId:_evaluatorBatchId);
+#pragma warning restore 618
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
b/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
index dcfd85b..a1cadff 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
@@ -17,26 +17,34 @@
* under the License.
*/
-using System.Collections.Generic;
-using Org.Apache.REEF.Common.Catalog;
-using Org.Apache.REEF.Common.Catalog.Capabilities;
-
namespace Org.Apache.REEF.Driver.Evaluator
{
+ /// <summary>
+ /// A request for an Evaluator allocation.
+ /// </summary>
public interface IEvaluatorRequest
{
- int MemoryMegaBytes { get; set; }
-
- int Number { get; set; }
-
- int VirtualCore { get; set; }
-
- string Rack { get; set; }
-
- string EvaluatorBatchId { get; set; }
-
- List<ICapability> Capabilities { get; set; }
-
- IResourceCatalog Catalog { get; set; }
+ /// <summary>
+ /// Memory for the Evaluator in megabytes.
+ /// </summary>
+ int MemoryMegaBytes { get; }
+
+ /// <summary>
+ /// Number of Evaluators to allocate.
+ /// </summary>
+ int Number { get; }
+
+ /// <summary>
+ /// Number of cores in the Evaluator.
+ /// </summary>
+ int VirtualCore { get; }
+
+ /// <summary>
+ /// The desired rack name for the Evaluator to be allocated in.
+ /// </summary>
+ string Rack { get; }
+
+ // TODO[REEF-718] Document
+ string EvaluatorBatchId { get; }
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequestor.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequestor.cs
b/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequestor.cs
index 63a1dda..f423af5 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequestor.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequestor.cs
@@ -29,17 +29,25 @@ namespace Org.Apache.REEF.Driver.Evaluator
/// <summary>
/// Access to the {@link ResourceCatalog} for the cluster this Factory
has access to
/// </summary>
- IResourceCatalog ResourceCatalog { get; set; }
-
- /// <summary>
- /// Map between user evaluator id and evaluator information
- /// </summary>
- //IDictionary<string, IEvaluatorDescriptor> Evaluators { get; }
+ IResourceCatalog ResourceCatalog { get; }
/// <summary>
/// Submit the request for new evaluator. The response will surface in
the AllocatedEvaluator message handler.
/// </summary>
/// <param name="request"></param>
void Submit(IEvaluatorRequest request);
+
+ /// <summary>
+ /// Returns a builder for new Evaluator requests.
+ /// </summary>
+ /// <returns></returns>
+ EvaluatorRequestBuilder NewBuilder();
+
+ /// <summary>
+ /// Returns a builder for new Evaluator requests.
+ /// </summary>
+ /// <param name="request">The request to clone</param>
+ /// <returns></returns>
+ EvaluatorRequestBuilder NewBuilder(IEvaluatorRequest request);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Evaluator/IFailedEvaluator.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Evaluator/IFailedEvaluator.cs
b/lang/cs/Org.Apache.REEF.Driver/Evaluator/IFailedEvaluator.cs
index bce2937..a87d01c 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/IFailedEvaluator.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/IFailedEvaluator.cs
@@ -17,6 +17,7 @@
* under the License.
*/
+using System;
using System.Collections.Generic;
using Org.Apache.REEF.Common.Exceptions;
using Org.Apache.REEF.Driver.Bridge.Events;
@@ -36,6 +37,7 @@ namespace Org.Apache.REEF.Driver.Evaluator
Optional<IFailedTask> FailedTask { get; set; }
+ [Obsolete("Will be removed after 0.13. Have an instance injected
instead.")]
IEvaluatorRequestor GetEvaluatorRequetor();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
b/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
index 285275a..b06feb1 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
+++ b/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
@@ -73,7 +73,7 @@ under the License.
<Compile Include="Bridge\Events\DriverRestartCompleted.cs" />
<Compile Include="Bridge\Events\DriverRestarted.cs" />
<Compile Include="Bridge\Events\DriverStarted.cs" />
- <Compile Include="Bridge\Events\EvaluatorRequstor.cs" />
+ <Compile Include="Bridge\Events\EvaluatorRequestor.cs" />
<Compile Include="Bridge\Events\FailedContext.cs" />
<Compile Include="Bridge\Events\FailedEvaluator.cs" />
<Compile Include="Bridge\Events\FailedTask.cs" />
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloDriverStartHandler.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloDriverStartHandler.cs
b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloDriverStartHandler.cs
index fd29e9d..dbba419 100644
--- a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloDriverStartHandler.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloDriverStartHandler.cs
@@ -63,7 +63,14 @@ namespace Org.Apache.REEF.Examples.AllHandlers
int core = 2;
string rack = "WonderlandRack";
string evaluatorBatchId = "evaluatorThatRequires512MBofMemory";
- EvaluatorRequest request = new EvaluatorRequest(evaluatorsNumber,
memory, core, rack, evaluatorBatchId);
+ var request =
+ _evaluatorRequestor.NewBuilder()
+ .SetNumber(evaluatorsNumber)
+ .SetMegabytes(memory)
+ .SetCores(core)
+ .SetRackName(rack)
+ .SetEvaluatorBatchId(evaluatorBatchId)
+ .Build();
_evaluatorRequestor.Submit(request);
@@ -72,7 +79,14 @@ namespace Org.Apache.REEF.Examples.AllHandlers
core = 2;
rack = "WonderlandRack";
evaluatorBatchId = "evaluatorThatRequires1999MBofMemory";
- request = new EvaluatorRequest(evaluatorsNumber, memory, core,
rack, evaluatorBatchId);
+ request =
+ _evaluatorRequestor.NewBuilder()
+ .SetNumber(evaluatorsNumber)
+ .SetMegabytes(memory)
+ .SetCores(core)
+ .SetRackName(rack)
+ .SetEvaluatorBatchId(evaluatorBatchId)
+ .Build();
_evaluatorRequestor.Submit(request);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloFailedEvaluatorHandler.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloFailedEvaluatorHandler.cs
b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloFailedEvaluatorHandler.cs
index 542fd3e..d0a781e 100644
---
a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloFailedEvaluatorHandler.cs
+++
b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/HelloFailedEvaluatorHandler.cs
@@ -32,9 +32,12 @@ namespace Org.Apache.REEF.Examples.AllHandlers
private static readonly int _maxTrial = 2;
+ private readonly IEvaluatorRequestor _evaluatorRequestor;
+
[Inject]
- private HelloFailedEvaluatorHandler()
+ private HelloFailedEvaluatorHandler(IEvaluatorRequestor
evaluatorRequestor)
{
+ _evaluatorRequestor = evaluatorRequestor;
}
/// <summary>
@@ -47,12 +50,10 @@ namespace Org.Apache.REEF.Examples.AllHandlers
if (++_failureCount < _maxTrial)
{
Console.WriteLine("Requesting another evaluator");
- EvaluatorRequest newRequest = new EvaluatorRequest(1, 512,
"somerack");
- IEvaluatorRequestor requestor =
failedEvaluator.GetEvaluatorRequetor();
- if (failedEvaluator.GetEvaluatorRequetor() != null)
- {
- requestor.Submit(newRequest);
- }
+ var newRequest =
+
_evaluatorRequestor.NewBuilder().SetNumber(1).SetMegabytes(512).SetRackName("somerack").Build();
+ _evaluatorRequestor.Submit(newRequest);
+
}
else
{
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs
b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs
index e1dbde1..b724b23 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs
@@ -70,7 +70,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF
public void OnNext(IDriverStarted driverStarted)
{
_Logger.Log(Level.Info, string.Format("HelloDriver started at
{0}", driverStarted.StartTime));
- _evaluatorRequestor.Submit(new EvaluatorRequest(1, 64));
+
_evaluatorRequestor.Submit(_evaluatorRequestor.NewBuilder().SetMegabytes(64).Build());
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Examples/DriverRestart/HelloRestartDriver.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Examples/DriverRestart/HelloRestartDriver.cs
b/lang/cs/Org.Apache.REEF.Examples/DriverRestart/HelloRestartDriver.cs
index 63ada39..2b0485b 100644
--- a/lang/cs/Org.Apache.REEF.Examples/DriverRestart/HelloRestartDriver.cs
+++ b/lang/cs/Org.Apache.REEF.Examples/DriverRestart/HelloRestartDriver.cs
@@ -92,7 +92,7 @@ namespace Org.Apache.REEF.Examples.DriverRestart
public void OnNext(IDriverStarted driverStarted)
{
Logger.Log(Level.Info, "HelloRestartDriver started at {0}",
driverStarted.StartTime);
- _evaluatorRequestor.Submit(new
EvaluatorRequest(NumberOfTasksToSubmit, 64));
+
_evaluatorRequestor.Submit(_evaluatorRequestor.NewBuilder().SetNumber(NumberOfTasksToSubmit).SetMegabytes(64).Build());
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Examples/MachineLearning/KMeans/KMeansDriverHandlers.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Examples/MachineLearning/KMeans/KMeansDriverHandlers.cs
b/lang/cs/Org.Apache.REEF.Examples/MachineLearning/KMeans/KMeansDriverHandlers.cs
index 24e82dd..20f30a5 100644
---
a/lang/cs/Org.Apache.REEF.Examples/MachineLearning/KMeans/KMeansDriverHandlers.cs
+++
b/lang/cs/Org.Apache.REEF.Examples/MachineLearning/KMeans/KMeansDriverHandlers.cs
@@ -190,9 +190,7 @@ namespace Org.Apache.REEF.Examples.MachineLearning.KMeans
public void OnNext(IDriverStarted value)
{
- int memory = 2048;
- int core = 1;
- EvaluatorRequest request = new EvaluatorRequest(_totalEvaluators,
memory, core);
+ var request =
_evaluatorRequestor.NewBuilder().SetCores(1).SetMegabytes(2048).Build();
_evaluatorRequestor.Submit(request);
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Driver/IMRUDriver.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Driver/IMRUDriver.cs
b/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Driver/IMRUDriver.cs
index 2d8bad5..0655835 100644
--- a/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Driver/IMRUDriver.cs
+++ b/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Driver/IMRUDriver.cs
@@ -113,7 +113,13 @@ namespace Org.Apache.REEF.IMRU.OnREEF.Driver
/// <param name="value">Event fired when driver started</param>
public void OnNext(IDriverStarted value)
{
- _evaluatorRequestor.Submit(new EvaluatorRequest(1,
_memoryForUpdateTask, _coresForUpdateTask));
+ var request =
+ _evaluatorRequestor.NewBuilder()
+ .SetCores(_coresForUpdateTask)
+ .SetMegabytes(_memoryForUpdateTask)
+ .SetNumber(1)
+ .Build();
+ _evaluatorRequestor.Submit(request);
//TODO[REEF-598]: Set a timeout for this request to be satisfied.
If it is not within that time, exit the Driver.
}
@@ -145,7 +151,13 @@ namespace Org.Apache.REEF.IMRU.OnREEF.Driver
serviceConf = Configurations.Merge(serviceConf, codecConfig,
_tcpPortProviderConfig);
_allocatedUpdateTaskEvaluator = true;
- _evaluatorRequestor.Submit(new
EvaluatorRequest(_dataSet.Count, _memoryPerMapper, _coresPerMapper));
+ var request =
+ _evaluatorRequestor.NewBuilder()
+ .SetMegabytes(_memoryForUpdateTask)
+ .SetNumber(_dataSet.Count)
+ .SetCores(_coresPerMapper)
+ .Build();
+ _evaluatorRequestor.Submit(request);
//TODO[REEF-598]: Set a timeout for this request to be
satisfied. If it is not within that time, exit the Driver.
}
else
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/BroadcastReduceDriverAndTasks/BroadcastReduceDriver.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/BroadcastReduceDriverAndTasks/BroadcastReduceDriver.cs
b/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/BroadcastReduceDriverAndTasks/BroadcastReduceDriver.cs
index 84082af..6f1ad0f 100644
---
a/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/BroadcastReduceDriverAndTasks/BroadcastReduceDriver.cs
+++
b/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/BroadcastReduceDriverAndTasks/BroadcastReduceDriver.cs
@@ -177,7 +177,14 @@ namespace
Org.Apache.REEF.Network.Examples.GroupCommunication.BroadcastReduceDri
public void OnNext(IDriverStarted value)
{
- EvaluatorRequest request = new EvaluatorRequest(_numEvaluators,
512, 2, "WonderlandRack", "BroadcastEvaluator");
+ var request =
+ _evaluatorRequestor.NewBuilder()
+ .SetNumber(_numEvaluators)
+ .SetMegabytes(512)
+ .SetCores(2)
+ .SetRackName("WonderlandRack")
+ .SetEvaluatorBatchId("BroadcastEvaluator")
+ .Build();
_evaluatorRequestor.Submit(request);
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/PipelineBroadcastReduceDriverAndTasks/PipelinedBroadcastReduceDriver.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/PipelineBroadcastReduceDriverAndTasks/PipelinedBroadcastReduceDriver.cs
b/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/PipelineBroadcastReduceDriverAndTasks/PipelinedBroadcastReduceDriver.cs
index 32439bd..9fb9652 100644
---
a/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/PipelineBroadcastReduceDriverAndTasks/PipelinedBroadcastReduceDriver.cs
+++
b/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/PipelineBroadcastReduceDriverAndTasks/PipelinedBroadcastReduceDriver.cs
@@ -201,7 +201,13 @@ namespace
Org.Apache.REEF.Network.Examples.GroupCommunication.PipelineBroadcastR
public void OnNext(IDriverStarted value)
{
- EvaluatorRequest request = new EvaluatorRequest(_numEvaluators,
512, 2, "WonderlandRack", "BroadcastEvaluator");
+ var request =
+ _evaluatorRequestor.NewBuilder()
+ .SetNumber(_numEvaluators)
+ .SetMegabytes(512)
+ .SetRackName("WonderlandRack")
+ .SetEvaluatorBatchId("BroadcastEvaluator")
+ .Build();
_evaluatorRequestor.Submit(request);
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/ScatterReduceDriverAndTasks/ScatterReduceDriver.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/ScatterReduceDriverAndTasks/ScatterReduceDriver.cs
b/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/ScatterReduceDriverAndTasks/ScatterReduceDriver.cs
index d2a7ea5..eb4cf86 100644
---
a/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/ScatterReduceDriverAndTasks/ScatterReduceDriver.cs
+++
b/lang/cs/Org.Apache.REEF.Network.Examples/GroupCommunication/ScatterReduceDriverAndTasks/ScatterReduceDriver.cs
@@ -148,7 +148,13 @@ namespace
Org.Apache.REEF.Network.Examples.GroupCommunication.ScatterReduceDrive
public void OnNext(IDriverStarted value)
{
- EvaluatorRequest request = new EvaluatorRequest(_numEvaluators,
512, 2, "WonderlandRack", "BroadcastEvaluator");
+ IEvaluatorRequest request =
+ _evaluatorRequestor.NewBuilder()
+ .SetNumber(_numEvaluators)
+ .SetMegabytes(512)
+ .SetCores(2)
+ .SetRackName("WonderlandRack")
+ .SetEvaluatorBatchId("BroadcastEvaluator").Build();
_evaluatorRequestor.Submit(request);
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/HelloSimpleEventHandlers.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/HelloSimpleEventHandlers.cs
b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/HelloSimpleEventHandlers.cs
index 545c6fb..7b3b27c 100644
---
a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/HelloSimpleEventHandlers.cs
+++
b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/HelloSimpleEventHandlers.cs
@@ -134,8 +134,14 @@ namespace Org.Apache.REEF.Tests.Functional.Bridge
int cpuCoreCount = 1;
string rack = "WonderlandRack";
string evaluatorBatchId = "evaluatorThatRequires3GBofMemory";
- EvaluatorRequest request = new
EvaluatorRequest(evaluatorsNumber, memory, cpuCoreCount, rack,
evaluatorBatchId);
-
+ var request =
+ _evaluatorRequestor.NewBuilder()
+ .SetNumber(evaluatorsNumber)
+ .SetMegabytes(memory)
+ .SetCores(cpuCoreCount)
+ .SetRackName(rack)
+ .SetEvaluatorBatchId(evaluatorBatchId)
+ .Build();
_evaluatorRequestor.Submit(request);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0e47fff5/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageDriver.cs
----------------------------------------------------------------------
diff --git
a/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageDriver.cs
b/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageDriver.cs
index 2b9e532..e9d39f9 100644
--- a/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageDriver.cs
+++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageDriver.cs
@@ -93,7 +93,14 @@ namespace Org.Apache.REEF.Tests.Functional.Messaging
public void OnNext(IDriverStarted value)
{
- EvaluatorRequest request = new EvaluatorRequest(NumerOfEvaluator,
512, 2, "WonderlandRack", "TaskMessagingEvaluator");
+ var request =
+ _evaluatorRequestor.NewBuilder()
+ .SetNumber(NumerOfEvaluator)
+ .SetMegabytes(512)
+ .SetCores(2)
+ .SetRackName("WonderlandRack")
+ .SetEvaluatorBatchId("TaskMessagingEvaluator")
+ .Build();
_evaluatorRequestor.Submit(request);
}
