Repository: incubator-reef Updated Branches: refs/heads/master 3686317b3 -> c85d45a27
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c85d45a2/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/ScatterReduceDriver.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/ScatterReduceDriver.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/ScatterReduceDriver.cs deleted file mode 100644 index a71e886..0000000 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/ScatterReduceDriver.cs +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using Org.Apache.REEF.Common.Io; -using Org.Apache.REEF.Common.Tasks; -using Org.Apache.REEF.Driver; -using Org.Apache.REEF.Driver.Bridge; -using Org.Apache.REEF.Driver.Context; -using Org.Apache.REEF.Driver.Evaluator; -using Org.Apache.REEF.Network.Group.Config; -using Org.Apache.REEF.Network.Group.Driver; -using Org.Apache.REEF.Network.Group.Driver.Impl; -using Org.Apache.REEF.Network.Group.Operators; -using Org.Apache.REEF.Network.Group.Operators.Impl; -using Org.Apache.REEF.Network.Group.Topology; -using Org.Apache.REEF.Network.NetworkService; -using Org.Apache.REEF.Tang.Annotations; -using Org.Apache.REEF.Tang.Formats; -using Org.Apache.REEF.Tang.Implementations.Tang; -using Org.Apache.REEF.Tang.Interface; -using Org.Apache.REEF.Tang.Util; -using Org.Apache.REEF.Utilities.Logging; -using Org.Apache.REEF.Wake.Remote.Impl; - -namespace Org.Apache.REEF.Tests.Functional.MPI.ScatterReduceTest -{ - public class ScatterReduceDriver : IStartHandler, IObserver<IEvaluatorRequestor>, IObserver<IAllocatedEvaluator>, IObserver<IActiveContext>, IObserver<IFailedEvaluator> - { - private static readonly Logger LOGGER = Logger.GetLogger(typeof(ScatterReduceDriver)); - - private readonly int _numEvaluators; - - private readonly IMpiDriver _mpiDriver; - private readonly ICommunicationGroupDriver _commGroup; - private readonly TaskStarter _mpiTaskStarter; - - [Inject] - public ScatterReduceDriver( - [Parameter(typeof(MpiTestConfig.NumEvaluators))] int numEvaluators, - MpiDriver mpiDriver) - { - Identifier = "BroadcastStartHandler"; - _numEvaluators = numEvaluators; - _mpiDriver = mpiDriver; - _commGroup = _mpiDriver.DefaultGroup - .AddScatter<int, IntCodec>( - MpiTestConstants.ScatterOperatorName, - MpiTestConstants.MasterTaskId, - TopologyTypes.Tree) - .AddReduce<int, IntCodec>( - MpiTestConstants.ReduceOperatorName, - MpiTestConstants.MasterTaskId, - new SumFunction()) - .Build(); - - _mpiTaskStarter = new TaskStarter(_mpiDriver, numEvaluators); - - CreateClassHierarchy(); - } - - public string Identifier { get; set; } - - public void OnNext(IEvaluatorRequestor evaluatorRequestor) - { - EvaluatorRequest request = new EvaluatorRequest(_numEvaluators, 512, 2, "WonderlandRack", "BroadcastEvaluator"); - evaluatorRequestor.Submit(request); - } - - public void OnNext(IAllocatedEvaluator allocatedEvaluator) - { - IConfiguration contextConf = _mpiDriver.GetContextConfiguration(); - IConfiguration serviceConf = _mpiDriver.GetServiceConfiguration(); - allocatedEvaluator.SubmitContextAndService(contextConf, serviceConf); - } - - public void OnNext(IActiveContext activeContext) - { - if (_mpiDriver.IsMasterTaskContext(activeContext)) - { - // Configure Master Task - IConfiguration partialTaskConf = TaskConfiguration.ConfigurationModule - .Set(TaskConfiguration.Identifier, MpiTestConstants.MasterTaskId) - .Set(TaskConfiguration.Task, GenericType<MasterTask>.Class) - .Build(); - - _commGroup.AddTask(MpiTestConstants.MasterTaskId); - _mpiTaskStarter.QueueTask(partialTaskConf, activeContext); - } - else - { - // Configure Slave Task - string slaveTaskId = MpiTestConstants.SlaveTaskId + - _mpiDriver.GetContextNum(activeContext); - - IConfiguration partialTaskConf = TaskConfiguration.ConfigurationModule - .Set(TaskConfiguration.Identifier, slaveTaskId) - .Set(TaskConfiguration.Task, GenericType<SlaveTask>.Class) - .Build(); - - _commGroup.AddTask(slaveTaskId); - _mpiTaskStarter.QueueTask(partialTaskConf, activeContext); - } - } - - public void OnNext(IFailedEvaluator value) - { - } - - public void OnError(Exception error) - { - } - - public void OnCompleted() - { - } - - private void CreateClassHierarchy() - { - HashSet<string> clrDlls = new HashSet<string>(); - clrDlls.Add(typeof(IDriver).Assembly.GetName().Name); - clrDlls.Add(typeof(ITask).Assembly.GetName().Name); - clrDlls.Add(typeof(ScatterReduceDriver).Assembly.GetName().Name); - clrDlls.Add(typeof(INameClient).Assembly.GetName().Name); - clrDlls.Add(typeof(INetworkService<>).Assembly.GetName().Name); - - ClrHandlerHelper.GenerateClassHierarchy(clrDlls); - } - - private class SumFunction : IReduceFunction<int> - { - [Inject] - public SumFunction() - { - } - - public int Reduce(IEnumerable<int> elements) - { - return elements.Sum(); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c85d45a2/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/ScatterReduceTest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/ScatterReduceTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/ScatterReduceTest.cs deleted file mode 100644 index d3afddc..0000000 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/ScatterReduceTest.cs +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -using System.Collections.Generic; -using System.Globalization; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Org.Apache.REEF.Common.Io; -using Org.Apache.REEF.Common.Tasks; -using Org.Apache.REEF.Driver; -using Org.Apache.REEF.Driver.Bridge; -using Org.Apache.REEF.Network.Group.Config; -using Org.Apache.REEF.Network.NetworkService; -using Org.Apache.REEF.Tang.Implementations.Configuration; -using Org.Apache.REEF.Tang.Implementations.Tang; -using Org.Apache.REEF.Tang.Interface; -using Org.Apache.REEF.Tang.Util; -using Org.Apache.REEF.Utilities.Logging; - -namespace Org.Apache.REEF.Tests.Functional.MPI.ScatterReduceTest -{ - [TestClass] - public class ScatterReduceTest : ReefFunctionalTest - { - [TestInitialize] - public void TestSetup() - { - CleanUp(); - } - - [TestCleanup] - public void TestCleanup() - { - CleanUp(); - } - - [TestMethod] - public void TestScatterAndReduceOnLocalRuntime() - { - int numTasks = 5; - TestScatterAndReduce(false, numTasks); - ValidateSuccessForLocalRuntime(numTasks); - } - - [TestMethod] - public void TestScatterAndReduceOnYarn() - { - int numTasks = 5; - TestScatterAndReduce(true, numTasks); - } - - [TestMethod] - public void TestScatterAndReduce(bool runOnYarn, int numTasks) - { - IConfiguration driverConfig = TangFactory.GetTang().NewConfigurationBuilder( - DriverBridgeConfiguration.ConfigurationModule - .Set(DriverBridgeConfiguration.OnDriverStarted, GenericType<ScatterReduceDriver>.Class) - .Set(DriverBridgeConfiguration.OnEvaluatorAllocated, GenericType<ScatterReduceDriver>.Class) - .Set(DriverBridgeConfiguration.OnEvaluatorRequested, GenericType<ScatterReduceDriver>.Class) - .Set(DriverBridgeConfiguration.OnEvaluatorFailed, GenericType<ScatterReduceDriver>.Class) - .Set(DriverBridgeConfiguration.OnContextActive, GenericType<ScatterReduceDriver>.Class) - .Set(DriverBridgeConfiguration.CustomTraceLevel, Level.Info.ToString()) - .Build()) - .BindNamedParameter<MpiTestConfig.NumEvaluators, int>( - GenericType<MpiTestConfig.NumEvaluators>.Class, - numTasks.ToString(CultureInfo.InvariantCulture)) - .Build(); - - IConfiguration mpiDriverConfig = TangFactory.GetTang().NewConfigurationBuilder() - .BindStringNamedParam<MpiConfigurationOptions.DriverId>(MpiTestConstants.DriverId) - .BindStringNamedParam<MpiConfigurationOptions.MasterTaskId>(MpiTestConstants.MasterTaskId) - .BindStringNamedParam<MpiConfigurationOptions.GroupName>(MpiTestConstants.GroupName) - .BindIntNamedParam<MpiConfigurationOptions.FanOut>(MpiTestConstants.FanOut.ToString(CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture)) - .BindIntNamedParam<MpiConfigurationOptions.NumberOfTasks>(numTasks.ToString()) - .Build(); - - IConfiguration merged = Configurations.Merge(driverConfig, mpiDriverConfig); - - HashSet<string> appDlls = new HashSet<string>(); - appDlls.Add(typeof(IDriver).Assembly.GetName().Name); - appDlls.Add(typeof(ITask).Assembly.GetName().Name); - appDlls.Add(typeof(ScatterReduceDriver).Assembly.GetName().Name); - appDlls.Add(typeof(INameClient).Assembly.GetName().Name); - appDlls.Add(typeof(INetworkService<>).Assembly.GetName().Name); - - TestRun(appDlls, merged, runOnYarn, JavaLoggingSetting.VERBOSE); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c85d45a2/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/SlaveTask.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/SlaveTask.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/SlaveTask.cs deleted file mode 100644 index 6c43280..0000000 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/MPI/ScatterReduceTest/SlaveTask.cs +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -using System.Collections.Generic; -using System.Linq; -using Org.Apache.REEF.Common.Tasks; -using Org.Apache.REEF.Network.Group.Operators; -using Org.Apache.REEF.Network.Group.Task; -using Org.Apache.REEF.Tang.Annotations; -using Org.Apache.REEF.Utilities.Logging; - -namespace Org.Apache.REEF.Tests.Functional.MPI.ScatterReduceTest -{ - public class SlaveTask : ITask - { - private static readonly Logger _logger = Logger.GetLogger(typeof(SlaveTask)); - - private readonly IMpiClient _mpiClient; - private readonly ICommunicationGroupClient _commGroup; - private readonly IScatterReceiver<int> _scatterReceiver; - private readonly IReduceSender<int> _sumSender; - - [Inject] - public SlaveTask(IMpiClient mpiClient) - { - _logger.Log(Level.Info, "Hello from slave task"); - - _mpiClient = mpiClient; - _commGroup = _mpiClient.GetCommunicationGroup(MpiTestConstants.GroupName); - _scatterReceiver = _commGroup.GetScatterReceiver<int>(MpiTestConstants.ScatterOperatorName); - _sumSender = _commGroup.GetReduceSender<int>(MpiTestConstants.ReduceOperatorName); - } - - public byte[] Call(byte[] memento) - { - List<int> data = _scatterReceiver.Receive(); - _logger.Log(Level.Info, "Received data: {0}", string.Join(" ", data)); - - int sum = data.Sum(); - _logger.Log(Level.Info, "Sending back sum: {0}", sum); - _sumSender.Send(sum); - - return null; - } - - public void Dispose() - { - _mpiClient.Dispose(); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c85d45a2/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj index 8cbcf61..3827941 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj +++ b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj @@ -58,20 +58,9 @@ under the License. <Compile Include="Functional\Messaging\MessageTask.cs" /> <Compile Include="Functional\Messaging\TestTaskMessage.cs" /> <Compile Include="Functional\ML\KMeans\TestKMeans.cs" /> - <Compile Include="Functional\MPI\BroadcastReduceTest\BroadcastReduceDriver.cs" /> - <Compile Include="Functional\MPI\BroadcastReduceTest\BroadcastReduceTest.cs" /> - <Compile Include="Functional\MPI\BroadcastReduceTest\MasterTask.cs" /> - <Compile Include="Functional\MPI\BroadcastReduceTest\SlaveTask.cs" /> - <Compile Include="Functional\MPI\MpiTestConfig.cs" /> - <Compile Include="Functional\MPI\MpiTestConstants.cs" /> - <Compile Include="Functional\MPI\PipelinedBroadcastReduceTest\PipelinedBroadcastReduceDriver.cs" /> - <Compile Include="Functional\MPI\PipelinedBroadcastReduceTest\PipelinedBroadcastReduceTest.cs" /> - <Compile Include="Functional\MPI\PipelinedBroadcastReduceTest\PipelinedMasterTask.cs" /> - <Compile Include="Functional\MPI\PipelinedBroadcastReduceTest\PipelinedSlaveTask.cs" /> - <Compile Include="Functional\MPI\ScatterReduceTest\MasterTask.cs" /> - <Compile Include="Functional\MPI\ScatterReduceTest\ScatterReduceDriver.cs" /> - <Compile Include="Functional\MPI\ScatterReduceTest\ScatterReduceTest.cs" /> - <Compile Include="Functional\MPI\ScatterReduceTest\SlaveTask.cs" /> + <Compile Include="Functional\Group\BroadcastReduceTest.cs" /> + <Compile Include="Functional\Group\PipelinedBroadcastReduceTest.cs" /> + <Compile Include="Functional\Group\ScatterReduceTest.cs" /> <Compile Include="Functional\ReefFunctionalTest.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Utility\TestDriverConfigGenerator.cs" /> @@ -81,9 +70,6 @@ under the License. <None Include="run.cmd"> <CopyToOutputDirectory>Always</CopyToOutputDirectory> </None> - <None Include="ConfigFiles\evaluator.conf"> - <CopyToOutputDirectory>Always</CopyToOutputDirectory> - </None> <None Include="packages.config" /> </ItemGroup> <ItemGroup> @@ -127,6 +113,10 @@ under the License. <Project>{4e69d40a-26d6-4d4a-b96d-729946c07fe1}</Project> <Name>Org.Apache.REEF.Bridge</Name> </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Network.Examples\Org.Apache.REEF.Network.Examples.csproj"> + <Project>{b1b43b60-ddd0-4805-a9b4-ba84a0ccb7c7}</Project> + <Name>Org.Apache.REEF.Network.Examples</Name> + </ProjectReference> </ItemGroup> <ItemGroup> <Folder Include="bin\Debug\" /> http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c85d45a2/lang/cs/Org.Apache.REEF.Tests/run.cmd ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/run.cmd b/lang/cs/Org.Apache.REEF.Tests/run.cmd index c9420e6..bfdd44e 100644 --- a/lang/cs/Org.Apache.REEF.Tests/run.cmd +++ b/lang/cs/Org.Apache.REEF.Tests/run.cmd @@ -33,7 +33,7 @@ :: RUNTIME -set SHADED_JAR=.\reef-bridge-0.11.0-incubating-SNAPSHOT-shaded.jar +set SHADED_JAR=.\reef-bridge-java-0.11.0-incubating-SNAPSHOT-shaded.jar set LOGGING_CONFIG=-Djava.util.logging.config.class=org.apache.reef.util.logging.Config http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/c85d45a2/lang/cs/Org.Apache.REEF.sln ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.sln b/lang/cs/Org.Apache.REEF.sln index 69fc800..760b0f3 100644 Binary files a/lang/cs/Org.Apache.REEF.sln and b/lang/cs/Org.Apache.REEF.sln differ
