Repository: incubator-reef Updated Branches: refs/heads/master 85bf1da4d -> 561e7a0ad
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs new file mode 100644 index 0000000..8da1cad --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloDriver.cs @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Org.Apache.REEF.Common.Files; +using Org.Apache.REEF.Driver; +using Org.Apache.REEF.Driver.Bridge; +using Org.Apache.REEF.Driver.Evaluator; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.Examples.HelloREEF +{ + /// <summary> + /// The Driver for HelloREEF: It requests a single Evaluator and then submits the HelloTask to it. + /// </summary> + public sealed class HelloDriver : IObserver<IAllocatedEvaluator>, IObserver<IEvaluatorRequestor>, IStartHandler + { + /// <summary> + /// Contexts contain configuration data used beyond a single task. + /// </summary> + private static readonly IConfiguration ContextConfiguration = + Driver.Context.ContextConfiguration.ConfigurationModule + .Set(Driver.Context.ContextConfiguration.Identifier, "HelloContext") + .Build(); + + /// <summary> + /// The TaskConfiguration contains the type of Task to run as well as the identifier of that task + /// </summary> + private static readonly IConfiguration TaskConfiguration = Common.Tasks.TaskConfiguration.ConfigurationModule + .Set(Common.Tasks.TaskConfiguration.Identifier, "HelloTask") + .Set(Common.Tasks.TaskConfiguration.Task, GenericType<HelloTask>.Class) + .Build(); + + private readonly REEFFileNames _fileNames; + + [Inject] + private HelloDriver(REEFFileNames fileNames) + { + _fileNames = fileNames; + ClrHandlerHelper.GenerateClassHierarchy(GetGlobalAssemblies()); + } + + /// <summary> + /// Submits the HelloTask to the Evaluator. + /// </summary> + /// <param name="allocatedEvaluator"></param> + public void OnNext(IAllocatedEvaluator allocatedEvaluator) + { + allocatedEvaluator.SubmitContextAndTask(ContextConfiguration, TaskConfiguration); + } + + public void OnError(Exception error) + { + throw error; + } + + public void OnCompleted() + { + } + + /// <summary> + /// Ask for one Evaluator with 64MB of memory. + /// </summary> + /// <param name="evaluatorRequestor"></param> + public void OnNext(IEvaluatorRequestor evaluatorRequestor) + { + evaluatorRequestor.Submit(new EvaluatorRequest(number:1, megaBytes:64)); + } + + public string Identifier { get; set; } + + /// <summary> + /// </summary> + /// <returns>All DLLs in the global folder</returns> + private ISet<string> GetGlobalAssemblies() + { + return new HashSet<string>(Directory.GetFiles(_fileNames.GetGlobalFolderPath()) + .Where(e => !(string.IsNullOrWhiteSpace(e))) + .Select(Path.GetFullPath) + .Where(File.Exists) + .Where(IsBinary) + .Select(Path.GetFileNameWithoutExtension)); + } + + /// <summary> + /// </summary> + /// <param name="path"></param> + /// <returns>True, if the path refers to an EXE or DLL</returns> + private static Boolean IsBinary(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return false; + } + var extension = Path.GetExtension(path).ToLower(); + return extension.EndsWith("dll") || extension.EndsWith("exe"); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs new file mode 100644 index 0000000..5962d38 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.Local; +using Org.Apache.REEF.Driver.Bridge; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.Examples.HelloREEF +{ + /// <summary> + /// A Tool that submits HelloREEFDriver for execution. + /// </summary> + public sealed class HelloREEF + { + private const string Local = "local"; + private readonly IREEFClient _reefClient; + + [Inject] + private HelloREEF(IREEFClient reefClient) + { + _reefClient = reefClient; + } + + /// <summary> + /// Runs HelloREEF using the IREEFClient passed into the constructor. + /// </summary> + private void Run() + { + // The driver configuration contains all the needed bindings. + var helloDriverConfiguration = DriverBridgeConfiguration.ConfigurationModule + .Set(DriverBridgeConfiguration.OnEvaluatorRequested, GenericType<HelloDriver>.Class) + .Set(DriverBridgeConfiguration.OnEvaluatorAllocated, GenericType<HelloDriver>.Class) + .Set(DriverBridgeConfiguration.OnDriverStarted, GenericType<HelloDriver>.Class) + .Build(); + // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. + var helloJobSubmission = new JobSubmission() + .AddDriverConfiguration(helloDriverConfiguration) + .AddGlobalAssemblyForType(typeof(HelloDriver)) + .SetJobIdentifier("HelloREEF"); + + _reefClient.Submit(helloJobSubmission); + } + + /// <summary> + /// </summary> + /// <param name="name"></param> + /// <returns></returns> + private static IConfiguration GetRuntimeConfiguration(string name) + { + switch (name) + { + case Local: + return LocalRuntimeClientConfiguration.ConfigurationModule + .Set(LocalRuntimeClientConfiguration.NumberOfEvaluators, "2") + .Build(); + default: + throw new Exception("Unknown runtime: " + name); + } + } + + public static void Main(string[] args) + { + TangFactory.GetTang().NewInjector(GetRuntimeConfiguration(args[0])).GetInstance<HelloREEF>().Run(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloTask.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloTask.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloTask.cs new file mode 100644 index 0000000..bd8d9ac --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloTask.cs @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Org.Apache.REEF.Common.Tasks; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Examples.HelloREEF +{ + /// <summary> + /// A Task that merely prints a greeting and exits. + /// </summary> + public sealed class HelloTask : ITask + { + [Inject] + private HelloTask() + { + } + + public void Dispose() + { + Console.WriteLine("Disposed."); + } + + public byte[] Call(byte[] memento) + { + Console.WriteLine("Hello, REEF!"); + return null; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj new file mode 100644 index 0000000..46330a9 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj @@ -0,0 +1,77 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003";> + <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> + <PropertyGroup> + <ProjectGuid>{0FF8CEE9-B0B6-4A14-9A52-44441BE048FE}</ProjectGuid> + <OutputType>Exe</OutputType> + <AppDesignerFolder>Properties</AppDesignerFolder> + <RootNamespace>Org.Apache.REEF.Examples.HelloREEF</RootNamespace> + <AssemblyName>Org.Apache.REEF.Examples.HelloREEF</AssemblyName> + <TargetFrameworkVersion>v4.5</TargetFrameworkVersion> + <FileAlignment>512</FileAlignment> + </PropertyGroup> + <Import Project="$(SolutionDir)\build.props" /> + <ItemGroup> + <Reference Include="System" /> + <Reference Include="System.Core" /> + <Reference Include="System.Xml.Linq" /> + <Reference Include="System.Data.DataSetExtensions" /> + <Reference Include="Microsoft.CSharp" /> + <Reference Include="System.Data" /> + <Reference Include="System.Xml" /> + </ItemGroup> + <ItemGroup> + <Compile Include="HelloDriver.cs" /> + <Compile Include="HelloREEF.cs" /> + <Compile Include="HelloTask.cs" /> + <Compile Include="Properties\AssemblyInfo.cs" /> + </ItemGroup> + <ItemGroup> + <None Include="App.config" /> + <None Include="Readme.md" /> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Bridge\Org.Apache.REEF.Bridge.vcxproj"> + <Project>{4e69d40a-26d6-4d4a-b96d-729946c07fe1}</Project> + <Name>Org.Apache.REEF.Bridge</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Client\Org.Apache.REEF.Client.csproj"> + <Project>{5094c35b-4fdb-4322-ac05-45d684501cbf}</Project> + <Name>Org.Apache.REEF.Client</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Common\Org.Apache.REEF.Common.csproj"> + <Project>{545a0582-4105-44ce-b99c-b1379514a630}</Project> + <Name>Org.Apache.REEF.Common</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Driver\Org.Apache.REEF.Driver.csproj"> + <Project>{a6baa2a7-f52f-4329-884e-1bcf711d6805}</Project> + <Name>Org.Apache.REEF.Driver</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Evaluator\Org.Apache.REEF.Evaluator.csproj"> + <Project>{1b983182-9c30-464c-948d-f87eb93a8240}</Project> + <Name>Org.Apache.REEF.Evaluator</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.csproj"> + <Project>{97dbb573-3994-417a-9f69-ffa25f00d2a6}</Project> + <Name>Org.Apache.REEF.Tang</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.csproj"> + <Project>{79e7f89a-1dfb-45e1-8d43-d71a954aeb98}</Project> + <Name>Org.Apache.REEF.Utilities</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Wake\Org.Apache.REEF.Wake.csproj"> + <Project>{cdfb3464-4041-42b1-9271-83af24cd5008}</Project> + <Name>Org.Apache.REEF.Wake</Name> + </ProjectReference> + </ItemGroup> + <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> + <!--begin jar reference--> + <PropertyGroup> + <AfterBuildDependsOn> + $(AfterBuildDependsOn); + CopyJarFiles; + </AfterBuildDependsOn> + </PropertyGroup> + <Target Name="AfterBuild" DependsOnTargets="$(AfterBuildDependsOn);" /> + <!--end jar reference--> +</Project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Properties/AssemblyInfo.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Properties/AssemblyInfo.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..7f8fb7c --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Properties/AssemblyInfo.cs @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Org.Apache.REEF.Examples.HelloREEF")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Org.Apache.REEF.Examples.HelloREEF")] +[assembly: AssemblyCopyright("Copyright © 2015")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("1aaf7ad7-445c-49f3-b519-f77cd8cd58d7")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Readme.md ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Readme.md b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Readme.md new file mode 100644 index 0000000..0ea427e --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Readme.md @@ -0,0 +1,9 @@ +# HelloREEF +This project contains a simple example of a REEF Program. It contains the following classes: + + * `HelloREEF`: This is the program that submits the driver to the local runtime. + * `HelloDriver`: The Driver requests a single Evaluator and submits the `HelloTask` to it. + * `HelloTask`: This Task prints a greeting to STDOUT of the Evaluator. + +## Running it +Just run the main class, `HelloREEF`, followed by the runtime you want, e.g. `local`. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestBridgeClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestBridgeClient.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestBridgeClient.cs index 3ef579b..03e7e1f 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestBridgeClient.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestBridgeClient.cs @@ -85,7 +85,7 @@ namespace Org.Apache.REEF.Tests.Functional.Bridge CreateNoWindow = false }; - LOGGER.Log(Level.Info, "executing\r\n" + startInfo.FileName + "\r\n" + startInfo.Arguments); + LOGGER.Log(Level.Info, "Executing '" + startInfo.FileName + " " + startInfo.Arguments +"' in working directory '" + Directory.GetCurrentDirectory() +"'"); using (Process process = Process.Start(startInfo)) { process.WaitForExit(); http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.Tests/app.config ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/app.config b/lang/cs/Org.Apache.REEF.Tests/app.config new file mode 100644 index 0000000..226a6a8 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Tests/app.config @@ -0,0 +1,29 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<?xml version="1.0" encoding="utf-8"?> +<configuration> + <runtime> + <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> + <dependentAssembly> + <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> + <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" /> + </dependentAssembly> + </assemblyBinding> + </runtime> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/cs/Org.Apache.REEF.sln ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.sln b/lang/cs/Org.Apache.REEF.sln index cfaace9..e0f89e9 100644 Binary files a/lang/cs/Org.Apache.REEF.sln and b/lang/cs/Org.Apache.REEF.sln differ http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/java/reef-bridge-client/pom.xml ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/pom.xml b/lang/java/reef-bridge-client/pom.xml new file mode 100644 index 0000000..7864953 --- /dev/null +++ b/lang/java/reef-bridge-client/pom.xml @@ -0,0 +1,136 @@ +<?xml version="1.0"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns="http://maven.apache.org/POM/4.0.0"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <artifactId>reef-bridge-client</artifactId> + <name>REEF Bridge (Client)</name> + <description>The Client side of the REEF Bridge</description> + + <parent> + <groupId>org.apache.reef</groupId> + <artifactId>reef-project</artifactId> + <version>0.11.0-incubating-SNAPSHOT</version> + <relativePath>../../..</relativePath> + </parent> + + <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>reef-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>reef-runtime-local</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>reef-runtime-yarn</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>reef-runtime-hdinsight</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>reef-io</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>reef-checkpoint</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>reef-webserver</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.reef</groupId> + <artifactId>reef-bridge-java</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + </dependency> + + <!-- Adding Hadoop to make sure we have it in the shaded jar --> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>runtime</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + <configuration> + <outputFile> + ${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar + </outputFile> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>yarn-default.xml</exclude> + <exclude>yarn-version-info.properties</exclude> + <exclude>core-default.xml</exclude> + <exclude>LICENSE</exclude> + <exclude>META-INF/*</exclude> + </excludes> + </filter> + </filters> + </configuration> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java new file mode 100644 index 0000000..a931251 --- /dev/null +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.bridge.client; + +import org.apache.reef.client.DriverConfiguration; +import org.apache.reef.client.DriverServiceConfiguration; +import org.apache.reef.io.network.naming.NameServerConfiguration; +import org.apache.reef.javabridge.generic.JobDriver; +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.Configurations; +import org.apache.reef.webserver.HttpHandlerConfiguration; +import org.apache.reef.webserver.HttpServerReefEventHandler; +import org.apache.reef.webserver.ReefEventStateManager; + +/** + * Constant Configuration instances used by the bridge. + */ +public final class Constants { + + /** + * Contains all bindings of event handlers to the bridge. + */ + public static final Configuration DRIVER_CONFIGURATION = DriverConfiguration.CONF + .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, JobDriver.AllocatedEvaluatorHandler.class) + .set(DriverConfiguration.ON_EVALUATOR_FAILED, JobDriver.FailedEvaluatorHandler.class) + .set(DriverConfiguration.ON_CONTEXT_ACTIVE, JobDriver.ActiveContextHandler.class) + .set(DriverConfiguration.ON_DRIVER_RESTART_CONTEXT_ACTIVE, JobDriver.DriverRestartActiveContextHandler.class) + .set(DriverConfiguration.ON_CONTEXT_CLOSED, JobDriver.ClosedContextHandler.class) + .set(DriverConfiguration.ON_CONTEXT_FAILED, JobDriver.FailedContextHandler.class) + .set(DriverConfiguration.ON_CONTEXT_MESSAGE, JobDriver.ContextMessageHandler.class) + .set(DriverConfiguration.ON_TASK_MESSAGE, JobDriver.TaskMessageHandler.class) + .set(DriverConfiguration.ON_TASK_FAILED, JobDriver.FailedTaskHandler.class) + .set(DriverConfiguration.ON_TASK_RUNNING, JobDriver.RunningTaskHandler.class) + .set(DriverConfiguration.ON_DRIVER_RESTART_TASK_RUNNING, JobDriver.DriverRestartRunningTaskHandler.class) + .set(DriverConfiguration.ON_DRIVER_RESTART_COMPLETED, JobDriver.DriverRestartCompletedHandler.class) + .set(DriverConfiguration.ON_TASK_COMPLETED, JobDriver.CompletedTaskHandler.class) + .set(DriverConfiguration.ON_DRIVER_STARTED, JobDriver.StartHandler.class) + .set(DriverConfiguration.ON_DRIVER_RESTARTED, JobDriver.RestartHandler.class) + .set(DriverConfiguration.ON_TASK_SUSPENDED, JobDriver.SuspendedTaskHandler.class) + .set(DriverConfiguration.ON_EVALUATOR_COMPLETED, JobDriver.CompletedEvaluatorHandler.class) + .build(); + + /** + * The HTTP Server configuration assumed by the bridge. + */ + public static final Configuration HTTP_SERVER_CONFIGURATION = Configurations.merge( + HttpHandlerConfiguration.CONF + .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class) + .build(), + DriverServiceConfiguration.CONF + .set(DriverServiceConfiguration.ON_EVALUATOR_ALLOCATED, ReefEventStateManager.AllocatedEvaluatorStateHandler.class) + .set(DriverServiceConfiguration.ON_CONTEXT_ACTIVE, ReefEventStateManager.ActiveContextStateHandler.class) + .set(DriverServiceConfiguration.ON_DRIVER_RESTART_CONTEXT_ACTIVE, ReefEventStateManager.DrivrRestartActiveContextStateHandler.class) + .set(DriverServiceConfiguration.ON_TASK_RUNNING, ReefEventStateManager.TaskRunningStateHandler.class) + .set(DriverServiceConfiguration.ON_DRIVER_RESTART_TASK_RUNNING, ReefEventStateManager.DriverRestartTaskRunningStateHandler.class) + .set(DriverServiceConfiguration.ON_DRIVER_STARTED, ReefEventStateManager.StartStateHandler.class) + .set(DriverServiceConfiguration.ON_DRIVER_STOP, ReefEventStateManager.StopStateHandler.class) + .build() + ); + + /** + * The name server configuration assumed by the bridge. + */ + public static final Configuration NAME_SERVER_CONFIGURATION = NameServerConfiguration.CONF + .set(NameServerConfiguration.NAME_SERVICE_PORT, 0) + .build(); + + /** + * The driver configuration assumed by the the bridge. + */ + public static final Configuration DRIVER_CONFIGURATION_WITH_HTTP_AND_NAMESERVER = Configurations.merge( + DRIVER_CONFIGURATION, + HTTP_SERVER_CONFIGURATION, + NAME_SERVER_CONFIGURATION + ); +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalClient.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalClient.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalClient.java new file mode 100644 index 0000000..e8d4fdf --- /dev/null +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/LocalClient.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.bridge.client; + +import org.apache.reef.runtime.common.driver.api.AbstractDriverRuntimeConfiguration; +import org.apache.reef.runtime.common.files.REEFFileNames; +import org.apache.reef.runtime.local.client.DriverConfigurationProvider; +import org.apache.reef.runtime.local.client.LocalRuntimeConfiguration; +import org.apache.reef.runtime.local.client.PreparedDriverFolderLauncher; +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.tang.formats.AvroConfigurationSerializer; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; + +/** + * Submits a folder containing a Driver to the local runtime. + */ +public class LocalClient { + + private static final String CLIENT_REMOTE_ID = AbstractDriverRuntimeConfiguration.ClientRemoteIdentifier.NONE; + private final AvroConfigurationSerializer configurationSerializer; + private final PreparedDriverFolderLauncher launcher; + private final REEFFileNames fileNames; + private final DriverConfigurationProvider driverConfigurationProvider; + + @Inject + public LocalClient(final AvroConfigurationSerializer configurationSerializer, + final PreparedDriverFolderLauncher launcher, + final REEFFileNames fileNames, + final DriverConfigurationProvider driverConfigurationProvider) { + this.configurationSerializer = configurationSerializer; + this.launcher = launcher; + this.fileNames = fileNames; + this.driverConfigurationProvider = driverConfigurationProvider; + } + + public void submit(final File jobFolder, final String jobId) throws IOException { + if (!jobFolder.exists()) { + throw new IOException("The Job folder" + jobFolder.getAbsolutePath() + "doesn't exist."); + } + + final File driverFolder = new File(jobFolder, PreparedDriverFolderLauncher.DRIVER_FOLDER_NAME); + if (!driverFolder.exists()) { + throw new IOException("The Driver folder " + driverFolder.getAbsolutePath() + " doesn't exist."); + } + + final Configuration driverConfiguration = driverConfigurationProvider + .getDriverConfiguration(jobFolder, CLIENT_REMOTE_ID, jobId, Constants.DRIVER_CONFIGURATION_WITH_HTTP_AND_NAMESERVER); + final File driverConfigurationFile = new File(driverFolder, fileNames.getDriverConfigurationPath()); + configurationSerializer.toFile(driverConfiguration, driverConfigurationFile); + launcher.launch(driverFolder, jobId, CLIENT_REMOTE_ID); + } + + + public static void main(final String[] args) throws InjectionException, IOException { + // TODO: Make the parameters of the local runtime command line arguments of this tool. + + // We assume the given path to be the one of the driver. The job folder is one level up from there. + final File jobFolder = new File(args[0]).getParentFile(); + // The job identifier + final String jobId = args[1]; + // The number of evaluators the local runtime can create + final int numberOfEvaluators = Integer.valueOf(args[2]); + + final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF + .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, numberOfEvaluators) + .set(LocalRuntimeConfiguration.RUNTIME_ROOT_FOLDER, jobFolder.getParentFile().getAbsolutePath()) + .build(); + + final LocalClient client = Tang.Factory.getTang() + .newInjector(runtimeConfiguration) + .getInstance(LocalClient.class); + + client.submit(jobFolder, jobId); + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/package-info.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/package-info.java new file mode 100644 index 0000000..e64ad71 --- /dev/null +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Contains the client of the bridge. + */ +package org.apache.reef.bridge.client; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java index 228743b..f7adb53 100644 --- a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java +++ b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java @@ -268,7 +268,7 @@ public final class JobDriver { /** * Handles AllocatedEvaluator: Submit an empty context */ - final class AllocatedEvaluatorHandler implements EventHandler<AllocatedEvaluator> { + public final class AllocatedEvaluatorHandler implements EventHandler<AllocatedEvaluator> { @Override public void onNext(final AllocatedEvaluator allocatedEvaluator) { try (final LoggingScope ls = loggingScopeFactory.evaluatorAllocated(allocatedEvaluator.getId())) { @@ -283,7 +283,7 @@ public final class JobDriver { /** * Receive notification that a new Context is available. */ - final class ActiveContextHandler implements EventHandler<ActiveContext> { + public final class ActiveContextHandler implements EventHandler<ActiveContext> { @Override public void onNext(final ActiveContext context) { try (final LoggingScope ls = loggingScopeFactory.activeContextReceived(context.getId())) { @@ -300,7 +300,7 @@ public final class JobDriver { /** * Receive notification that the Task has completed successfully. */ - final class CompletedTaskHandler implements EventHandler<CompletedTask> { + public final class CompletedTaskHandler implements EventHandler<CompletedTask> { @Override public void onNext(final CompletedTask task) { LOG.log(Level.INFO, "Completed task: {0}", task.getId()); @@ -328,7 +328,7 @@ public final class JobDriver { /** * Receive notification that the entire Evaluator had failed. */ - final class FailedEvaluatorHandler implements EventHandler<FailedEvaluator> { + public final class FailedEvaluatorHandler implements EventHandler<FailedEvaluator> { @Override public void onNext(final FailedEvaluator eval) { try (final LoggingScope ls = loggingScopeFactory.evaluatorFailed(eval.getId())) { @@ -428,7 +428,7 @@ public final class JobDriver { /** * Handle failed task. */ - final class FailedTaskHandler implements EventHandler<FailedTask> { + public final class FailedTaskHandler implements EventHandler<FailedTask> { @Override public void onNext(final FailedTask task) throws RuntimeException { LOG.log(Level.SEVERE, "FailedTask received, will be handle in CLR handler, if set."); @@ -449,7 +449,7 @@ public final class JobDriver { /** * Receive notification that the Task is running. */ - final class RunningTaskHandler implements EventHandler<RunningTask> { + public final class RunningTaskHandler implements EventHandler<RunningTask> { @Override public void onNext(final RunningTask task) { try (final LoggingScope ls = loggingScopeFactory.taskRunning(task.getId())) { @@ -472,7 +472,7 @@ public final class JobDriver { /** * Receive notification that the Task is running when driver restarted. */ - final class DriverRestartRunningTaskHandler implements EventHandler<RunningTask> { + public final class DriverRestartRunningTaskHandler implements EventHandler<RunningTask> { @Override public void onNext(final RunningTask task) { try (final LoggingScope ls = loggingScopeFactory.driverRestartRunningTask(task.getId())) { @@ -499,7 +499,7 @@ public final class JobDriver { /** * Receive notification that an context is active on Evaluator when the driver restarted */ - final class DriverRestartActiveContextHandler implements EventHandler<ActiveContext> { + public final class DriverRestartActiveContextHandler implements EventHandler<ActiveContext> { @Override public void onNext(final ActiveContext context) { try (final LoggingScope ls = loggingScopeFactory.driverRestartActiveContextReceived(context.getId())) { @@ -528,7 +528,7 @@ public final class JobDriver { /** * Job Driver is ready and the clock is set up: request the evaluators. */ - final class StartHandler implements EventHandler<StartTime> { + public final class StartHandler implements EventHandler<StartTime> { @Override public void onNext(final StartTime startTime) { try (final LoggingScope ls = loggingScopeFactory.driverStart(startTime)) { @@ -554,7 +554,7 @@ public final class JobDriver { /** * Job driver is restarted after previous crash */ - final class RestartHandler implements EventHandler<StartTime> { + public final class RestartHandler implements EventHandler<StartTime> { @Override public void onNext(final StartTime startTime) { try (final LoggingScope ls = loggingScopeFactory.driverRestart(startTime)) { @@ -573,7 +573,7 @@ public final class JobDriver { /** * Receive notification that driver restart has completed. */ - final class DriverRestartCompletedHandler implements EventHandler<DriverRestartCompleted> { + public final class DriverRestartCompletedHandler implements EventHandler<DriverRestartCompleted> { @Override public void onNext(final DriverRestartCompleted driverRestartCompleted) { LOG.log(Level.INFO, "Java DriverRestartCompleted event received at time [{0}]. ", driverRestartCompleted.getTimeStamp()); @@ -604,7 +604,7 @@ public final class JobDriver { } } - final class TaskMessageHandler implements EventHandler<TaskMessage> { + public final class TaskMessageHandler implements EventHandler<TaskMessage> { @Override public void onNext(final TaskMessage taskMessage) { String msg = new String(taskMessage.get()); @@ -622,7 +622,7 @@ public final class JobDriver { /** * Receive notification that the Task has been suspended. */ - final class SuspendedTaskHandler implements EventHandler<SuspendedTask> { + public final class SuspendedTaskHandler implements EventHandler<SuspendedTask> { @Override public final void onNext(final SuspendedTask task) { final String message = "Received notification that task [" + task.getId() + "] has been suspended."; @@ -642,7 +642,7 @@ public final class JobDriver { /** * Receive notification that the Evaluator has been shut down. */ - final class CompletedEvaluatorHandler implements EventHandler<CompletedEvaluator> { + public final class CompletedEvaluatorHandler implements EventHandler<CompletedEvaluator> { @Override public void onNext(final CompletedEvaluator evaluator) { LOG.log(Level.INFO, " Completed Evaluator {0}", evaluator.getId()); @@ -662,7 +662,7 @@ public final class JobDriver { * Receive notification that the Context had completed. * Remove context from the list of active context. */ - final class ClosedContextHandler implements EventHandler<ClosedContext> { + public final class ClosedContextHandler implements EventHandler<ClosedContext> { @Override public void onNext(final ClosedContext context) { LOG.log(Level.INFO, "Completed Context: {0}", context.getId()); @@ -685,7 +685,7 @@ public final class JobDriver { * Receive notification that the Context had failed. * Remove context from the list of active context and notify the client. */ - final class FailedContextHandler implements EventHandler<FailedContext> { + public final class FailedContextHandler implements EventHandler<FailedContext> { @Override public void onNext(final FailedContext context) { LOG.log(Level.SEVERE, "FailedContext", context); @@ -710,7 +710,7 @@ public final class JobDriver { /** * Receive notification that a ContextMessage has been received */ - final class ContextMessageHandler implements EventHandler<ContextMessage> { + public final class ContextMessageHandler implements EventHandler<ContextMessage> { @Override public void onNext(final ContextMessage message) { LOG.log(Level.SEVERE, "Received ContextMessage:", message.get()); http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalJobSubmissionHandler.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalJobSubmissionHandler.java b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalJobSubmissionHandler.java index 75fc86e..d417fce 100644 --- a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalJobSubmissionHandler.java +++ b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalJobSubmissionHandler.java @@ -18,7 +18,6 @@ */ package org.apache.reef.runtime.local.client; -import org.apache.commons.lang.StringUtils; import org.apache.reef.annotations.audience.ClientSide; import org.apache.reef.annotations.audience.Private; import org.apache.reef.proto.ClientRuntimeProtocol; @@ -37,7 +36,6 @@ import org.apache.reef.util.logging.LoggingScopeFactory; import javax.inject.Inject; import java.io.File; -import java.util.List; import java.util.concurrent.ExecutorService; import java.util.logging.Level; import java.util.logging.Logger; @@ -49,20 +47,14 @@ import java.util.logging.Logger; @ClientSide final class LocalJobSubmissionHandler implements JobSubmissionHandler { - /** - * The name of the folder for the driver within the Job folder. - */ - public static final String DRIVER_FOLDER_NAME = "driver"; - /** - * The (hard-coded) amount of memory to be used for the driver. - */ - public static final int DRIVER_MEMORY = 512; + private static final Logger LOG = Logger.getLogger(LocalJobSubmissionHandler.class.getName()); private final ExecutorService executor; private final String rootFolderName; private final ConfigurationSerializer configurationSerializer; private final REEFFileNames fileNames; private final ClasspathProvider classpath; + private final PreparedDriverFolderLauncher driverLauncher; private final LoggingScopeFactory loggingScopeFactory; private final DriverConfigurationProvider driverConfigurationProvider; @@ -73,6 +65,8 @@ final class LocalJobSubmissionHandler implements JobSubmissionHandler { final ConfigurationSerializer configurationSerializer, final REEFFileNames fileNames, final ClasspathProvider classpath, + + final PreparedDriverFolderLauncher driverLauncher, final LoggingScopeFactory loggingScopeFactory, final DriverConfigurationProvider driverConfigurationProvider) { @@ -80,6 +74,8 @@ final class LocalJobSubmissionHandler implements JobSubmissionHandler { this.configurationSerializer = configurationSerializer; this.fileNames = fileNames; this.classpath = classpath; + + this.driverLauncher = driverLauncher; this.driverConfigurationProvider = driverConfigurationProvider; this.rootFolderName = new File(rootFolderName).getAbsolutePath(); this.loggingScopeFactory = loggingScopeFactory; @@ -101,7 +97,7 @@ final class LocalJobSubmissionHandler implements JobSubmissionHandler { final File jobFolder = new File(new File(rootFolderName), "/" + t.getIdentifier() + "-" + System.currentTimeMillis() + "/"); - final File driverFolder = new File(jobFolder, DRIVER_FOLDER_NAME); + final File driverFolder = new File(jobFolder, PreparedDriverFolderLauncher.DRIVER_FOLDER_NAME); driverFolder.mkdirs(); final DriverFiles driverFiles = DriverFiles.fromJobSubmission(t, this.fileNames); @@ -113,28 +109,7 @@ final class LocalJobSubmissionHandler implements JobSubmissionHandler { this.configurationSerializer.toFile(driverConfiguration, new File(driverFolder, this.fileNames.getDriverConfigurationPath())); - - final List<String> command = new JavaLaunchCommandBuilder() - .setErrorHandlerRID(t.getRemoteId()) - .setLaunchID(t.getIdentifier()) - .setConfigurationFileName(this.fileNames.getDriverConfigurationPath()) - .setClassPath(this.classpath.getDriverClasspath()) - .setMemory(DRIVER_MEMORY) - .build(); - - if (LOG.isLoggable(Level.FINEST)) { - LOG.log(Level.FINEST, "REEF app command: {0}", StringUtils.join(command, ' ')); - } - - final RunnableProcess process = new RunnableProcess(command, - "driver", - driverFolder, - new LoggingRunnableProcessObserver(), - this.fileNames.getDriverStdoutFileName(), - this.fileNames.getDriverStderrFileName()); - this.executor.submit(process); - this.executor.shutdown(); - + this.driverLauncher.launch(driverFolder, t.getIdentifier(), t.getRemoteId()); } catch (final Exception e) { LOG.log(Level.SEVERE, "Unable to setup driver.", e); throw new RuntimeException("Unable to setup driver.", e); http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/PreparedDriverFolderLauncher.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/PreparedDriverFolderLauncher.java b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/PreparedDriverFolderLauncher.java new file mode 100644 index 0000000..602c04d --- /dev/null +++ b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/PreparedDriverFolderLauncher.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.local.client; + +import org.apache.commons.lang.StringUtils; +import org.apache.reef.runtime.common.files.ClasspathProvider; +import org.apache.reef.runtime.common.files.REEFFileNames; +import org.apache.reef.runtime.common.launch.JavaLaunchCommandBuilder; +import org.apache.reef.runtime.local.process.LoggingRunnableProcessObserver; +import org.apache.reef.runtime.local.process.RunnableProcess; + +import javax.inject.Inject; +import java.io.File; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Launcher for a already prepared driver folder. + */ +public class PreparedDriverFolderLauncher { + + /** + * The name of the folder for the driver within the Job folder. + */ + public static final String DRIVER_FOLDER_NAME = "driver"; + + private final ExecutorService executor; + private final REEFFileNames fileNames; + private final ClasspathProvider classpath; + /** + * The (hard-coded) amount of memory to be used for the driver. + */ + public static final int DRIVER_MEMORY = 512; + + private static final Logger LOG = Logger.getLogger(PreparedDriverFolderLauncher.class.getName()); + + @Inject + PreparedDriverFolderLauncher(final ExecutorService executor, REEFFileNames fileNames, ClasspathProvider classpath) { + this.executor = executor; + this.fileNames = fileNames; + this.classpath = classpath; + } + + /** + * Launches the driver prepared in driverFolder + * + * @param driverFolder + * @param jobId + * @param clientRemoteId + */ + public void launch(final File driverFolder, final String jobId, final String clientRemoteId) { + assert (driverFolder.isDirectory()); + + final List<String> command = makeLaunchCommand(jobId, clientRemoteId); + + final RunnableProcess process = new RunnableProcess(command, + "driver", + driverFolder, + new LoggingRunnableProcessObserver(), + this.fileNames.getDriverStdoutFileName(), + this.fileNames.getDriverStderrFileName()); + this.executor.submit(process); + this.executor.shutdown(); + } + + private List<String> makeLaunchCommand(final String jobId, final String clientRemoteId) { + + final List<String> command = new JavaLaunchCommandBuilder() + .setErrorHandlerRID(clientRemoteId) + .setLaunchID(jobId) + .setConfigurationFileName(this.fileNames.getDriverConfigurationPath()) + .setClassPath(this.classpath.getDriverClasspath()) + .setMemory(DRIVER_MEMORY) + .build(); + + if (LOG.isLoggable(Level.FINEST)) { + LOG.log(Level.FINEST, "REEF app command: {0}", StringUtils.join(command, ' ')); + } + return command; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/561e7a0a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f913080..dc84e3b 100644 --- a/pom.xml +++ b/pom.xml @@ -240,7 +240,9 @@ under the License. <exclude>lang/java/.idea/**</exclude> <exclude>**/*.iml</exclude> <exclude>**/target/**</exclude> + <!-- ReadMe files --> <exclude>**/README.*</exclude> + <exclude>**/*.md</exclude> <!-- The below are sometimes created during tests --> <exclude>REEF_LOCAL_RUNTIME/**</exclude> <!-- The Visual Studio build files --> @@ -625,7 +627,8 @@ under the License. <modules> <module>lang/cs</module> <module>lang/java/reef-annotations</module> - <module>lang/java/reef-bridge-java</module> + <module>lang/java/reef-bridge-client</module> + <module>lang/java/reef-bridge-java</module> <module>lang/java/reef-checkpoint</module> <module>lang/java/reef-common</module> <module>lang/java/reef-examples</module>
