http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/packages.config ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/packages.config b/lang/cs/Org.Apache.REEF.Client/packages.config index d6832d2..094733d 100644 --- a/lang/cs/Org.Apache.REEF.Client/packages.config +++ b/lang/cs/Org.Apache.REEF.Client/packages.config @@ -18,7 +18,20 @@ specific language governing permissions and limitations under the License. --> <packages> + <package id="Azure.Batch" version="8.0.1" targetFramework="net452" /> + <package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net452" /> + <package id="Microsoft.Data.Edm" version="5.8.2" targetFramework="net452" /> + <package id="Microsoft.Data.OData" version="5.8.2" targetFramework="net452" /> + <package id="Microsoft.Data.Services.Client" version="5.8.2" targetFramework="net452" /> + <package id="Microsoft.Rest.ClientRuntime" version="2.3.10" targetFramework="net452" /> + <package id="Microsoft.Rest.ClientRuntime.Azure" version="3.3.10" targetFramework="net452" /> <package id="Newtonsoft.Json" version="10.0.3" targetFramework="net451" /> <package id="StyleCop.MSBuild" version="5.0.0" targetFramework="net45" developmentDependency="true" /> + <package id="System.ComponentModel.EventBasedAsync" version="4.0.11" targetFramework="net452" /> + <package id="System.Dynamic.Runtime" version="4.0.0" targetFramework="net452" /> + <package id="System.Linq.Queryable" version="4.0.0" targetFramework="net452" /> + <package id="System.Net.Requests" version="4.0.11" targetFramework="net452" /> + <package id="System.Spatial" version="5.8.2" targetFramework="net452" /> <package id="TransientFaultHandling.Core" version="5.1.1209.1" targetFramework="net45" /> + <package id="WindowsAzure.Storage" version="8.1.3" targetFramework="net452" /> </packages> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs b/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs index 59ea062..2328289 100644 --- a/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs +++ b/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs @@ -38,6 +38,11 @@ namespace Org.Apache.REEF.Common.Runtime Mesos, /// <summary> + /// Same value as org.apache.reef.runtime.mesos.driver.RuntimeIdentifier.RUNTIME_NAME + /// </summary> + AzBatch, + + /// <summary> /// Default value for the enum /// </summary> Default http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs index 33f5aa7..406fb36 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs @@ -16,7 +16,9 @@ // under the License. using System; +using System.Globalization; using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.AzureBatch; using Org.Apache.REEF.Client.Common; using Org.Apache.REEF.Client.Local; using Org.Apache.REEF.Client.Yarn; @@ -40,6 +42,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF private const string YARN = "yarn"; private const string YARNRest = "yarnrest"; private const string HDInsight = "hdi"; + private const string AzureBatch = "azurebatch"; private readonly IREEFClient _reefClient; [Inject] @@ -60,19 +63,26 @@ namespace Org.Apache.REEF.Examples.HelloREEF .Set(DriverConfiguration.CustomTraceLevel, Level.Verbose.ToString()) .Build(); + string applicationId = GetApplicationId(); + // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. var helloJobRequest = _reefClient.NewJobRequestBuilder() .AddDriverConfiguration(helloDriverConfiguration) .AddGlobalAssemblyForType(typeof(HelloDriver)) .AddGlobalAssembliesInDirectoryOfExecutingAssembly() - .SetJobIdentifier("HelloREEF") + .SetJobIdentifier(applicationId) .SetJavaLogLevel(JavaLoggingSetting.Verbose) .Build(); IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobRequest); // Wait for the Driver to complete. - jobSubmissionResult.WaitForDriverToFinish(); + jobSubmissionResult?.WaitForDriverToFinish(); + } + + private string GetApplicationId() + { + return "HelloWorldJob-" + DateTime.Now.ToString("ddd-MMM-d-HH-mm-ss-yyyy", CultureInfo.CreateSpecificCulture("en-US")); } /// <summary> @@ -104,6 +114,17 @@ namespace Org.Apache.REEF.Examples.HelloREEF .Set(HDInsightClientConfiguration.JobSubmissionDirectoryPrefix, string.Format(@"/{0}/tmp", continerName)) .Set(AzureBlockBlobFileSystemConfiguration.ConnectionString, connectionString) .Build(); + case AzureBatch: + return AzureBatchRuntimeClientConfiguration.ConfigurationModule + .Set(AzureBatchRuntimeClientConfiguration.AzureBatchAccountKey, @"##########################################") + .Set(AzureBatchRuntimeClientConfiguration.AzureBatchAccountName, @"######") + .Set(AzureBatchRuntimeClientConfiguration.AzureBatchAccountUri, @"######################") + .Set(AzureBatchRuntimeClientConfiguration.AzureBatchPoolId, @"######") + .Set(AzureBatchRuntimeClientConfiguration.AzureStorageAccountKey, @"##########################################") + .Set(AzureBatchRuntimeClientConfiguration.AzureStorageAccountName, @"############") + .Set(AzureBatchRuntimeClientConfiguration.AzureStorageContainerName, @"###########") + .Build(); + default: throw new Exception("Unknown runtime: " + name); } http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs index f8aa715..e88892e 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs @@ -25,14 +25,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF /// <param name="args"></param> public static void Main(string[] args) { - if (args.Length < 2) - { - HelloREEF.MainSimple(args); - } - else - { - HelloREEFYarn.MainYarn(args); - } + HelloREEF.MainSimple(args); } } } http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.sln ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.sln b/lang/cs/Org.Apache.REEF.sln index b4cbccd..95fb5d0 100644 Binary files a/lang/cs/Org.Apache.REEF.sln and b/lang/cs/Org.Apache.REEF.sln differ http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-client/pom.xml ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/pom.xml b/lang/java/reef-bridge-client/pom.xml index 7bc6a28..4c1b279 100644 --- a/lang/java/reef-bridge-client/pom.xml +++ b/lang/java/reef-bridge-client/pom.xml @@ -58,6 +58,11 @@ under the License. </dependency> <dependency> <groupId>${project.groupId}</groupId> + <artifactId>reef-runtime-azbatch</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> <artifactId>reef-io</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc index ecf0043..e2a30f3 100644 --- a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc +++ b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc @@ -65,5 +65,22 @@ { "name": "driverStdoutFilePath", "type": "string" }, { "name": "driverStderrFilePath", "type": "string" } ] + }, + { + "namespace": "org.apache.reef.reef.bridge.client.avro", + "type": "record", + "name": "AvroAzureBatchJobSubmissionParameters", + "doc": "Cross-language submission parameters to the Azure Batch runtime", + "fields": [ + { "name": "sharedJobSubmissionParameters", "type": "AvroJobSubmissionParameters" }, + { "name": "AzureBatchAccountKey", "type": "string" }, + { "name": "AzureBatchAccountName", "type": "string" }, + { "name": "AzureBatchAccountUri", "type": "string" }, + { "name": "AzureBatchPoolId", "type": "string" }, + { "name": "AzureStorageAccountKey", "type": "string" }, + { "name": "AzureStorageAccountName", "type": "string" }, + { "name": "AzureStorageContainerName", "type": "string" }, + { "name": "AzureBatchIsWindows", "type": "boolean" } + ] } ] http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java new file mode 100644 index 0000000..c054722 --- /dev/null +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.bridge.client; + +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.JsonDecoder; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters; +import org.apache.reef.runtime.common.client.DriverConfigurationProvider; +import org.apache.reef.runtime.common.driver.parameters.ClientRemoteIdentifier; +import org.apache.reef.tang.Configuration; + +import javax.inject.Inject; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This is the Java Driver configuration generator for .NET Drivers that generates + * the Driver configuration at runtime. Called by {@link AzureBatchBootstrapREEFLauncher}. + */ +final class AzureBatchBootstrapDriverConfigGenerator { + + private static final Logger LOG = Logger.getLogger(AzureBatchBootstrapDriverConfigGenerator.class.getName()); + + private final DriverConfigurationProvider driverConfigurationProvider; + + @Inject + private AzureBatchBootstrapDriverConfigGenerator(final DriverConfigurationProvider driverConfigurationProvider) { + this.driverConfigurationProvider = driverConfigurationProvider; + } + + Configuration getDriverConfigurationFromParams(final String bootstrapJobArgsLocation) throws IOException { + + final File bootstrapJobArgsFile = new File(bootstrapJobArgsLocation).getCanonicalFile(); + + final AvroAzureBatchJobSubmissionParameters azureBatchBootstrapJobArgs = + readAzureBatchJobSubmissionParametersFromFile(bootstrapJobArgsFile); + + final String jobId = azureBatchBootstrapJobArgs.getSharedJobSubmissionParameters().getJobId().toString(); + final File jobFolder = new File(azureBatchBootstrapJobArgs + .getSharedJobSubmissionParameters().getJobSubmissionFolder().toString()); + + LOG.log(Level.INFO, "jobFolder {0} jobId {1}.", new Object[]{jobFolder.toURI(), jobId}); + + return this.driverConfigurationProvider.getDriverConfiguration( + jobFolder.toURI(), ClientRemoteIdentifier.NONE, jobId, + Constants.DRIVER_CONFIGURATION_WITH_HTTP_AND_NAMESERVER); + } + + private AvroAzureBatchJobSubmissionParameters readAzureBatchJobSubmissionParametersFromFile(final File file) + throws IOException { + try (final FileInputStream fileInputStream = new FileInputStream(file)) { + return readAzureBatchSubmissionParametersFromInputStream(fileInputStream); + } + } + + private static AvroAzureBatchJobSubmissionParameters readAzureBatchSubmissionParametersFromInputStream( + final InputStream inputStream) throws IOException { + final JsonDecoder decoder = DecoderFactory.get().jsonDecoder( + AvroAzureBatchJobSubmissionParameters.getClassSchema(), inputStream); + final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader = new SpecificDatumReader<>( + AvroAzureBatchJobSubmissionParameters.class); + return reader.read(null, decoder); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java new file mode 100644 index 0000000..481ae53 --- /dev/null +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.bridge.client; + +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.JsonDecoder; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.reef.annotations.audience.Interop; +import org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters; +import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfiguration; +import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfigurationCreator; +import org.apache.reef.runtime.common.REEFEnvironment; +import org.apache.reef.runtime.common.evaluator.PIDStoreStartHandler; +import org.apache.reef.runtime.common.launch.REEFErrorHandler; +import org.apache.reef.runtime.common.launch.REEFMessageCodec; +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.wake.remote.RemoteConfiguration; +import org.apache.reef.wake.time.Clock; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This is a bootstrap launcher for Azure Batch for submission from C#. It allows for Java Driver + * configuration generation directly on the Driver without need of Java dependency if REST + * submission is used. Note that the name of the class must contain "REEFLauncher" for the time + * being in order for the Interop code to discover the class. + */ +@Interop(CppFiles = "DriverLauncher.cpp") +public final class AzureBatchBootstrapREEFLauncher { + + private static final Logger LOG = Logger.getLogger(AzureBatchBootstrapREEFLauncher.class.getName()); + private static final Tang TANG = Tang.Factory.getTang(); + + public static void main(final String[] args) throws IOException, InjectionException { + + LOG.log(Level.INFO, "Entering BootstrapLauncher.main(). {0}", args[0]); + + if (args.length != 1) { + + final StringBuilder sb = new StringBuilder( + "Bootstrap launcher should have one configuration file input," + + " specifying the job submission parameters to be deserialized" + + " to create the Azure Batch DriverConfiguration on the fly." + + " Current args are [ "); + for (String arg : args) { + sb.append(arg).append(" "); + } + sb.append("]"); + + final String message = sb.toString(); + throw fatal(message, new IllegalArgumentException(message)); + } + + final File partialConfigFile = new File(args[0]); + final AzureBatchBootstrapDriverConfigGenerator azureBatchBootstrapDriverConfigGenerator = + TANG.newInjector(generateConfigurationFromJobSubmissionParameters(partialConfigFile)) + .getInstance(AzureBatchBootstrapDriverConfigGenerator.class); + + final Configuration launcherConfig = + TANG.newConfigurationBuilder() + .bindNamedParameter(RemoteConfiguration.ManagerName.class, "AzureBatchBootstrapREEFLauncher") + .bindNamedParameter(RemoteConfiguration.ErrorHandler.class, REEFErrorHandler.class) + .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class) + .bindSetEntry(Clock.RuntimeStartHandler.class, PIDStoreStartHandler.class) + .build(); + + try (final REEFEnvironment reef = REEFEnvironment.fromConfiguration( + azureBatchBootstrapDriverConfigGenerator.getDriverConfigurationFromParams(args[0]), launcherConfig)) { + reef.run(); + } catch (final InjectionException ex) { + throw fatal("Unable to configure and start REEFEnvironment.", ex); + } + } + + private static Configuration generateConfigurationFromJobSubmissionParameters(final File params) throws IOException { + + final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters; + + try (final FileInputStream fileInputStream = new FileInputStream(params)) { + final JsonDecoder decoder = DecoderFactory.get().jsonDecoder( + AvroAzureBatchJobSubmissionParameters.getClassSchema(), fileInputStream); + final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader = + new SpecificDatumReader<>(AvroAzureBatchJobSubmissionParameters.class); + avroAzureBatchJobSubmissionParameters = reader.read(null, decoder); + } + + return AzureBatchRuntimeConfigurationCreator + .getOrCreateAzureBatchRuntimeConfiguration(avroAzureBatchJobSubmissionParameters.getAzureBatchIsWindows()) + .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_NAME, + avroAzureBatchJobSubmissionParameters.getAzureBatchAccountName().toString()) + .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_KEY, + avroAzureBatchJobSubmissionParameters.getAzureBatchAccountKey().toString()) + .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_URI, + avroAzureBatchJobSubmissionParameters.getAzureBatchAccountUri().toString()) + .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_POOL_ID, + avroAzureBatchJobSubmissionParameters.getAzureBatchPoolId().toString()) + .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_ACCOUNT_NAME, + avroAzureBatchJobSubmissionParameters.getAzureStorageAccountName().toString()) + .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_ACCOUNT_KEY, + avroAzureBatchJobSubmissionParameters.getAzureStorageAccountKey().toString()) + .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_CONTAINER_NAME, + avroAzureBatchJobSubmissionParameters.getAzureStorageContainerName().toString()) + .build(); + } + + private static RuntimeException fatal(final String msg, final Throwable t) { + LOG.log(Level.SEVERE, msg, t); + return new RuntimeException(msg, t); + } + + private AzureBatchBootstrapREEFLauncher() { + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-java/pom.xml ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-java/pom.xml b/lang/java/reef-bridge-java/pom.xml index 2728e44..3b4ff86 100644 --- a/lang/java/reef-bridge-java/pom.xml +++ b/lang/java/reef-bridge-java/pom.xml @@ -53,6 +53,11 @@ under the License. </dependency> <dependency> <groupId>${project.groupId}</groupId> + <artifactId>reef-runtime-azbatch</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> <artifactId>reef-io</artifactId> <version>${project.version}</version> </dependency>
