Repository: incubator-reef Updated Branches: refs/heads/master 7849ee670 -> c09150067
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs new file mode 100644 index 0000000..ff17af8 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.IO; +using System.Linq; +using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.Common; +using Org.Apache.REEF.Common.Files; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Utilities.Logging; + +namespace Org.Apache.REEF.Client.Yarn +{ + internal sealed class YarnREEFClient : IREEFClient + { + /// <summary> + /// The class name that contains the Java counterpart for this client. + /// </summary> + private const string JavaClassName = "org.apache.reef.bridge.client.YarnJobSubmissionClient"; + + private static readonly Logger Logger = Logger.GetLogger(typeof(YarnREEFClient)); + private readonly DriverFolderPreparationHelper _driverFolderPreparationHelper; + private readonly JavaClientLauncher _javaClientLauncher; + private String _driverUrl; + private REEFFileNames _fileNames; + + [Inject] + internal YarnREEFClient(JavaClientLauncher javaClientLauncher, + DriverFolderPreparationHelper driverFolderPreparationHelper, + REEFFileNames fileNames, + YarnCommandLineEnvironment yarn) + { + _javaClientLauncher = javaClientLauncher; + _javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList()); + _driverFolderPreparationHelper = driverFolderPreparationHelper; + _fileNames = fileNames; + } + + public void Submit(IJobSubmission jobSubmission) + { + // Prepare the job submission folder + var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier); + Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath); + + Launch(jobSubmission, driverFolderPath); + } + + public IDriverHttpEndpoint SubmitAndGetDriverUrl(IJobSubmission jobSubmission) + { + // Prepare the job submission folder + var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier); + Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath); + + Launch(jobSubmission, driverFolderPath); + + var pointerFileName = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint); + + var httpClient = new HttpClientHelper(); + _driverUrl = httpClient.GetDriverUrlForYarn(pointerFileName); + + return httpClient; + } + + private void Launch(IJobSubmission jobSubmission, string driverFolderPath) + { + _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath); + + //TODO: Remove this when we have a generalized way to pass config to java + var javaParams = TangFactory.GetTang() + .NewInjector(jobSubmission.DriverConfigurations.ToArray()) + .GetInstance<ClrClient2JavaClientCuratedParameters>(); + + // Submit the driver + _javaClientLauncher.Launch(JavaClassName, driverFolderPath, jobSubmission.JobIdentifier, + jobSubmission.DriverMemory.ToString(), + javaParams.TcpPortRangeStart.ToString(), + javaParams.TcpPortRangeCount.ToString(), + javaParams.TcpPortRangeTryCount.ToString(), + javaParams.MaxApplicationSubmissions.ToString(), + javaParams.DriverRestartEvaluatorRecoverySeconds.ToString() + ); + Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier); + } + + public string DriverUrl + { + get { return _driverUrl; } + } + + /// <summary> + /// Creates the temporary directory to hold the job submission. + /// </summary> + /// <returns>The path to the folder created.</returns> + private string CreateDriverFolder(string jobId) + { + var timestamp = DateTime.Now.ToString("yyyyMMddHHmmssfff"); + return Path.GetFullPath(Path.Combine(Path.GetTempPath(), string.Join("-", "reef", jobId, timestamp))); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs index 753c53c..4f590d7 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs @@ -27,7 +27,7 @@ using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Utilities.Diagnostics; using Org.Apache.REEF.Utilities.Logging; -namespace Org.Apache.REEF.Client.YARN +namespace Org.Apache.REEF.Client.Yarn { /// <summary> /// Helper class to interact with the YARN command line. http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Client/packages.config ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/packages.config b/lang/cs/Org.Apache.REEF.Client/packages.config new file mode 100644 index 0000000..763dbe1 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/packages.config @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<packages> + <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net45" /> + <package id="RestSharpSigned" version="105.2.3" targetFramework="net45" /> +</packages> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs index 0eeb7d5..3f7a444 100644 --- a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs +++ b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs @@ -22,7 +22,7 @@ using System.IO; using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.Common; using Org.Apache.REEF.Client.Local; -using Org.Apache.REEF.Client.YARN; +using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Common.Evaluator; using Org.Apache.REEF.Driver; using Org.Apache.REEF.Driver.Bridge; http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs b/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs index 6924c44..d5df938 100644 --- a/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs +++ b/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs @@ -19,7 +19,7 @@ using System; using Org.Apache.REEF.Client.API; -using Org.Apache.REEF.Client.YARN; +using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Common.Evaluator; using Org.Apache.REEF.Driver; using Org.Apache.REEF.Tang.Annotations; http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs index e71356b..685ed23 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs @@ -20,7 +20,7 @@ using System; using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.Local; -using Org.Apache.REEF.Client.YARN; +using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Driver; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Implementations.Tang; http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs b/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs index 870e010..4502bfa 100644 --- a/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs @@ -19,7 +19,7 @@ using System.Globalization; using Org.Apache.REEF.Client.Local; -using Org.Apache.REEF.Client.YARN; +using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.IMRU.OnREEF.Client; using Org.Apache.REEF.Tang.Implementations.Configuration; using Org.Apache.REEF.Tang.Interface; http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs b/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs index 1c02ecd..d2a532e 100644 --- a/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs +++ b/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs @@ -22,7 +22,7 @@ using System.Globalization; using System.IO; using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.Local; -using Org.Apache.REEF.Client.YARN; +using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Driver; using Org.Apache.REEF.Driver.Bridge; using Org.Apache.REEF.Network.Examples.GroupCommunication; http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs index 0a8935d..dba3b26 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs @@ -29,7 +29,7 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.Local; -using Org.Apache.REEF.Client.YARN; +using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Tang.Interface; using Org.Apache.REEF.Utilities; http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Utilities/AsyncUtils/RemoveSynchronizationContextAwaiter.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Utilities/AsyncUtils/RemoveSynchronizationContextAwaiter.cs b/lang/cs/Org.Apache.REEF.Utilities/AsyncUtils/RemoveSynchronizationContextAwaiter.cs new file mode 100644 index 0000000..c9b4da9 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Utilities/AsyncUtils/RemoveSynchronizationContextAwaiter.cs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Org.Apache.REEF.Utilities.AsyncUtils +{ + /// <summary> + /// This is to avoid deadlocks in clients that use synchronization context (WinForms/ASP.NET) + /// Use at asynchronous entry points of all async REEF client APIs. + /// + /// There is no need to use this in evaluator/driver. + /// </summary> + public struct RemoveSynchronizationContextAwaiter : INotifyCompletion + { + public bool IsCompleted + { + get { return SynchronizationContext.Current == null; } + } + + public void OnCompleted(Action continuation) + { + SynchronizationContext previousSynchronizationContext = SynchronizationContext.Current; + try + { + SynchronizationContext.SetSynchronizationContext(null); + continuation(); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousSynchronizationContext); + } + } + + public RemoveSynchronizationContextAwaiter GetAwaiter() + { + return this; + } + + public void GetResult() + { + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj b/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj index 78789ec..45c1408 100644 --- a/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj +++ b/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj @@ -37,6 +37,7 @@ under the License. <Reference Include="System.Core" /> </ItemGroup> <ItemGroup> + <Compile Include="AsyncUtils\RemoveSynchronizationContextAwaiter.cs" /> <Compile Include="AvroUtils.cs" /> <Compile Include="ByteUtilities.cs" /> <Compile Include="Diagnostics\DiagnosticsMessages.cs" /> @@ -55,6 +56,7 @@ under the License. <None Include="Org.Apache.Reef.Utilities.nuspec" /> <None Include="packages.config" /> </ItemGroup> + <ItemGroup /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" /> <!-- To modify your build process, add your task inside one of the targets below and uncomment it. http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c0915006/lang/cs/Org.Apache.REEF.sln ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.sln b/lang/cs/Org.Apache.REEF.sln index 2691796..c8720d9 100644 Binary files a/lang/cs/Org.Apache.REEF.sln and b/lang/cs/Org.Apache.REEF.sln differ
