[REEF-1948] Add IREEFClient.NewJobRequestBuilder This adds the new API to `IREEFClient`.
To ease implementation and foster re-use, this also adds the new class `JobRequestBuilderFactory`. It is used on all implementations of `IREEFClient` to implement the new method. Lastly, this changes `HelloREEF` to use the new API to demonstrate it to work. JIRA: [REEF-1948](https://issues.apache.org/jira/browse/REEF-1948) Pull Request: This closes #1411 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/7a2aa5cc Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/7a2aa5cc Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/7a2aa5cc Branch: refs/heads/REEF-335 Commit: 7a2aa5cc6e2f6133976c60f00b6146c383c3d1ab Parents: fece629 Author: Markus Weimer <[email protected]> Authored: Fri Nov 3 09:46:18 2017 -0700 Committer: Sergiy Matusevych <[email protected]> Committed: Fri Nov 3 11:07:24 2017 -0700 ---------------------------------------------------------------------- .../Org.Apache.REEF.Client/API/IREEFClient.cs | 6 +++ .../API/JobRequestBuilder.cs | 7 +-- .../API/JobRequestBuilderFactory.cs | 48 ++++++++++++++++++++ .../Org.Apache.REEF.Client/Local/LocalClient.cs | 24 +++++++--- .../Org.Apache.REEF.Client.csproj | 1 + .../YARN/YARNREEFClient.cs | 10 +++- .../YARN/YarnREEFDotNetClient.cs | 10 +++- .../HelloREEF.cs | 6 +-- 8 files changed, 94 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs index a4734f5..bc04186 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs @@ -35,6 +35,12 @@ namespace Org.Apache.REEF.Client.API void Submit(JobRequest jobRequest); /// <summary> + /// Instantiate a new JobRequestBuilder. + /// </summary> + /// <returns>A new JobRequestBuilder</returns> + JobRequestBuilder NewJobRequestBuilder(); + + /// <summary> /// Submit the job described in jobRequest to the cluster. /// Expect IJobSubmissionResult returned after the call. /// </summary> http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs index 234cd7a..7760b7f 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs @@ -34,16 +34,11 @@ namespace Org.Apache.REEF.Client.API } [Inject] - private JobRequestBuilder([Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders) + internal JobRequestBuilder([Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders) { AddDriverConfigurationProviders(configurationProviders); } - public static JobRequestBuilder NewBuilder() - { - return new JobRequestBuilder(); - } - /// <summary> /// Bake the information provided so far and return a IJobSubmission /// </summary> http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilderFactory.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilderFactory.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilderFactory.cs new file mode 100644 index 0000000..6986c34 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilderFactory.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Generic; +using Org.Apache.REEF.Common.Client.Parameters; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.Client.API +{ + /// <summary> + /// Helper class to create instances of JobRequestBuilder. + /// </summary> + internal sealed class JobRequestBuilderFactory + { + private readonly ISet<IConfigurationProvider> _configurationProviders; + + [Inject] + private JobRequestBuilderFactory( + [Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders) + { + _configurationProviders = configurationProviders; + } + + /// <summary> + /// Instantiates a new JobRequestBuilder. + /// </summary> + /// <returns>JobRequestBuilder</returns> + public JobRequestBuilder NewInstance() + { + return new JobRequestBuilder(_configurationProviders); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs index 0f1ef1d..722bf9b 100644 --- a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs @@ -31,7 +31,6 @@ using Org.Apache.REEF.Common.Files; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Tang.Interface; -using Org.Apache.REEF.Tang.Util; using Org.Apache.REEF.Utilities.AsyncUtils; using Org.Apache.REEF.Utilities.Attributes; using Org.Apache.REEF.Utilities.Logging; @@ -63,20 +62,25 @@ namespace Org.Apache.REEF.Client.Local private readonly string _runtimeFolder; private readonly REEFFileNames _fileNames; private readonly IConfiguration _localConfigurationOnDriver; + private readonly JobRequestBuilderFactory _jobRequestBuilderFactory; [Inject] private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper, [Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder, [Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators, IJavaClientLauncher javaClientLauncher, - REEFFileNames fileNames) + REEFFileNames fileNames, + JobRequestBuilderFactory jobRequestBuilderFactory) { _driverFolderPreparationHelper = driverFolderPreparationHelper; _runtimeFolder = runtimeFolder; _maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators; _javaClientLauncher = javaClientLauncher; _fileNames = fileNames; - _localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder().BindImplementation(GenericType<ILocalAddressProvider>.Class, GenericType<LoopbackLocalAddressProvider>.Class).Build(); + _jobRequestBuilderFactory = jobRequestBuilderFactory; + _localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder() + .BindImplementation<ILocalAddressProvider, LoopbackLocalAddressProvider>() + .Build(); } /// <summary> @@ -86,13 +90,16 @@ namespace Org.Apache.REEF.Client.Local /// <param name="numberOfEvaluators"></param> /// <param name="javaClientLauncher"></param> /// <param name="fileNames"></param> + /// <param name="jobRequestBuilderFactory">The helper used to instantiate JobRequestBuilder instances.</param> [Inject] private LocalClient( DriverFolderPreparationHelper driverFolderPreparationHelper, [Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators, IJavaClientLauncher javaClientLauncher, - REEFFileNames fileNames) - : this(driverFolderPreparationHelper, Path.GetTempPath(), numberOfEvaluators, javaClientLauncher, fileNames) + REEFFileNames fileNames, + JobRequestBuilderFactory jobRequestBuilderFactory) + : this(driverFolderPreparationHelper, Path.GetTempPath(), + numberOfEvaluators, javaClientLauncher, fileNames, jobRequestBuilderFactory) { // Intentionally left blank. } @@ -171,6 +178,11 @@ namespace Org.Apache.REEF.Client.Local Logger.Log(Level.Info, "Submitted the Driver for execution."); } + public JobRequestBuilder NewJobRequestBuilder() + { + return _jobRequestBuilderFactory.NewInstance(); + } + public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest) { var driverFolder = PrepareDriverFolder(jobRequest); @@ -211,4 +223,4 @@ namespace Org.Apache.REEF.Client.Local return Path.Combine(_runtimeFolder, string.Join("-", "reef", jobId, timestamp)); } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index 00c6d87..80b8d90 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -73,6 +73,7 @@ under the License. <Compile Include="API\JobParametersBuilder.cs" /> <Compile Include="API\JobRequest.cs" /> <Compile Include="API\JobRequestBuilder.cs" /> + <Compile Include="API\JobRequestBuilderFactory.cs" /> <Compile Include="API\TcpPortConfigurationModule.cs" /> <Compile Include="Avro\AvroAppSubmissionParameters.cs" /> <Compile Include="Avro\AvroJobSubmissionParameters.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs index b4e5ad7..09434f0 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs @@ -47,6 +47,7 @@ namespace Org.Apache.REEF.Client.Yarn private readonly REEFFileNames _fileNames; private readonly IYarnRMClient _yarnClient; private readonly YarnREEFParamSerializer _paramSerializer; + private readonly JobRequestBuilderFactory _jobRequestBuilderFactory; [Inject] internal YarnREEFClient(IJavaClientLauncher javaClientLauncher, @@ -54,7 +55,8 @@ namespace Org.Apache.REEF.Client.Yarn REEFFileNames fileNames, YarnCommandLineEnvironment yarn, IYarnRMClient yarnClient, - YarnREEFParamSerializer paramSerializer) + YarnREEFParamSerializer paramSerializer, + JobRequestBuilderFactory jobRequestBuilderFactory) { _javaClientLauncher = javaClientLauncher; _javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList()); @@ -62,6 +64,7 @@ namespace Org.Apache.REEF.Client.Yarn _fileNames = fileNames; _yarnClient = yarnClient; _paramSerializer = paramSerializer; + _jobRequestBuilderFactory = jobRequestBuilderFactory; } public void Submit(JobRequest jobRequest) @@ -73,6 +76,11 @@ namespace Org.Apache.REEF.Client.Yarn Launch(jobRequest, driverFolderPath); } + public JobRequestBuilder NewJobRequestBuilder() + { + return _jobRequestBuilderFactory.NewInstance(); + } + public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest) { // Prepare the job submission folder http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs index babb613..65c6c87 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs @@ -51,6 +51,7 @@ namespace Org.Apache.REEF.Client.YARN private readonly REEFFileNames _fileNames; private readonly IJobSubmissionDirectoryProvider _jobSubmissionDirectoryProvider; private readonly YarnREEFDotNetParamSerializer _paramSerializer; + private readonly JobRequestBuilderFactory _jobRequestBuilderFactory; [Inject] private YarnREEFDotNetClient( @@ -60,7 +61,8 @@ namespace Org.Apache.REEF.Client.YARN IJobResourceUploader jobResourceUploader, REEFFileNames fileNames, IJobSubmissionDirectoryProvider jobSubmissionDirectoryProvider, - YarnREEFDotNetParamSerializer paramSerializer) + YarnREEFDotNetParamSerializer paramSerializer, + JobRequestBuilderFactory jobRequestBuilderFactory) { _injector = injector; _jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider; @@ -69,6 +71,7 @@ namespace Org.Apache.REEF.Client.YARN _driverFolderPreparationHelper = driverFolderPreparationHelper; _yarnRMClient = yarnRMClient; _paramSerializer = paramSerializer; + _jobRequestBuilderFactory = jobRequestBuilderFactory; } public void Submit(JobRequest jobRequest) @@ -135,6 +138,11 @@ namespace Org.Apache.REEF.Client.YARN } } + public JobRequestBuilder NewJobRequestBuilder() + { + return _jobRequestBuilderFactory.NewInstance(); + } + public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest) { throw new NotSupportedException(); http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs index b1d0cd3..e29ea0c 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs @@ -41,13 +41,11 @@ namespace Org.Apache.REEF.Examples.HelloREEF private const string YARNRest = "yarnrest"; private const string HDInsight = "hdi"; private readonly IREEFClient _reefClient; - private readonly JobRequestBuilder _jobRequestBuilder; [Inject] - private HelloREEF(IREEFClient reefClient, JobRequestBuilder jobRequestBuilder) + private HelloREEF(IREEFClient reefClient) { _reefClient = reefClient; - _jobRequestBuilder = jobRequestBuilder; } /// <summary> @@ -62,7 +60,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF .Build(); // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. - var helloJobRequest = _jobRequestBuilder + var helloJobRequest = _reefClient.NewJobRequestBuilder() .AddDriverConfiguration(helloDriverConfiguration) .AddGlobalAssemblyForType(typeof(HelloDriver)) .SetJobIdentifier("HelloREEF")
