Repository: reef Updated Branches: refs/heads/master 7924188ca -> 561a336f2
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/CommandBuilderTests.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/CommandBuilderTests.java b/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/CommandBuilderTests.java new file mode 100644 index 0000000..230cca3 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/CommandBuilderTests.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch; + +import org.apache.reef.runtime.azbatch.util.command.LinuxCommandBuilder; +import org.apache.reef.runtime.azbatch.util.command.WindowsCommandBuilder; +import org.apache.reef.runtime.common.client.api.JobSubmissionEvent; +import org.apache.reef.runtime.common.files.RuntimeClasspathProvider; +import org.apache.reef.tang.Injector; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.util.Optional; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for the CommandBuilder functions. + */ +public final class CommandBuilderTests { + + private Injector injector; + private LinuxCommandBuilder linuxCommandBuilder; + private WindowsCommandBuilder windowsCommandBuilder; + + @SuppressWarnings("unchecked") + @Before + public void setUp() throws InjectionException { + this.injector = Tang.Factory.getTang().newInjector(); + RuntimeClasspathProvider classpathProvider = mock(RuntimeClasspathProvider.class); + when(classpathProvider.getDriverClasspathPrefix()).thenReturn(Arrays.asList("c:\\driverpath1", "c:\\driverpath2")); + when(classpathProvider.getEvaluatorClasspathPrefix()) + .thenReturn(Arrays.asList("c:\\evaluatorpath1", "c:\\evaluatorpath2")); + when(classpathProvider.getDriverClasspathSuffix()).thenReturn(Arrays.asList("driverclasspathsuffix")); + when(classpathProvider.getEvaluatorClasspathSuffix()).thenReturn(Arrays.asList("evaluatorclasspathsuffix")); + this.injector + .bindVolatileInstance(RuntimeClasspathProvider.class, classpathProvider); + this.linuxCommandBuilder = this.injector.getInstance(LinuxCommandBuilder.class); + this.windowsCommandBuilder = this.injector.getInstance(WindowsCommandBuilder.class); + + } + + @Test + public void linuxCommandBuilderDriverTest() { + JobSubmissionEvent event = mock(JobSubmissionEvent.class); + + Optional<Integer> memory = Optional.of(100); + when(event.getDriverMemory()).thenReturn(memory); + + String actual = this.linuxCommandBuilder.buildDriverCommand(event); + String expected = + "/bin/sh -c \"unzip local.jar -d 'reef/'; {{JAVA_HOME}}/bin/java -Xmx100m -XX:PermSize=128m " + + "-XX:MaxPermSize=128m -ea -classpath " + + "c:\\driverpath1:c:\\driverpath2:reef/local/*:reef/global/*:driverclasspathsuffix " + + "-Dproc_reef org.apache.reef.runtime.common.REEFLauncher reef/local/driver.conf\""; + Assert.assertEquals(expected, actual); + } + + @Test + public void windowsCommandBuilderDriverTest() { + JobSubmissionEvent event = mock(JobSubmissionEvent.class); + + Optional<Integer> memory = Optional.of(100); + when(event.getDriverMemory()).thenReturn(memory); + + String actual = this.windowsCommandBuilder.buildDriverCommand(event); + String expected = "powershell.exe /c \"Add-Type -AssemblyName System.IO.Compression.FileSystem; " + + "[System.IO.Compression.ZipFile]::ExtractToDirectory(\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\local.jar\\\", " + + "\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\reef\\\"); {{JAVA_HOME}}/bin/java -Xmx100m -XX:PermSize=128m " + + "-XX:MaxPermSize=128m -ea -classpath " + + "'c:\\driverpath1;c:\\driverpath2;reef/local/*;reef/global/*;driverclasspathsuffix' " + + "-Dproc_reef org.apache.reef.runtime.common.REEFLauncher reef/local/driver.conf\";"; + Assert.assertEquals(expected, actual); + } + + @Test + public void linuxCommandBuilderShimEvaluatorTest() { + String actual = this.linuxCommandBuilder.buildEvaluatorShimCommand(1, "conf"); + String expected = "/bin/sh -c \"unzip local.jar -d 'reef/'; {{JAVA_HOME}}/bin/java -Xmx1m " + + "-XX:PermSize=128m -XX:MaxPermSize=128m -ea " + + "-classpath c:\\evaluatorpath1:c:\\evaluatorpath2:reef/local/*:reef/global/*:evaluatorclasspathsuffix " + + "-Dproc_reef org.apache.reef.runtime.azbatch.evaluator.EvaluatorShimLauncher conf\""; + Assert.assertEquals(expected, actual); + } + + @Test + public void windowsCommandBuilderShimEvaluatorTest() { + String actual = this.windowsCommandBuilder.buildEvaluatorShimCommand(1, "conf"); + String expected = "powershell.exe /c \"Add-Type -AssemblyName System.IO.Compression.FileSystem; " + + "[System.IO.Compression.ZipFile]::ExtractToDirectory(\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\local.jar\\\", " + + "\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\reef\\\"); {{JAVA_HOME}}/bin/java -Xmx1m -XX:PermSize=128m " + + "-XX:MaxPermSize=128m -ea -classpath " + + "'c:\\evaluatorpath1;c:\\evaluatorpath2;reef/local/*;reef/global/*;evaluatorclasspathsuffix' -Dproc_reef " + + "org.apache.reef.runtime.azbatch.evaluator.EvaluatorShimLauncher conf\";"; + Assert.assertEquals(expected, actual); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/package-info.java b/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/package-info.java new file mode 100644 index 0000000..13a5caa --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Unit tests for REEF Driver running under Azure Batch runtime. + */ +package org.apache.reef.runtime.azbatch; http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-tests/pom.xml ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/pom.xml b/lang/java/reef-tests/pom.xml index beefd5f..b096e39 100644 --- a/lang/java/reef-tests/pom.xml +++ b/lang/java/reef-tests/pom.xml @@ -44,6 +44,11 @@ under the License. </dependency> <dependency> <groupId>${project.groupId}</groupId> + <artifactId>reef-runtime-azbatch</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> <artifactId>reef-runtime-local</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AzureBatchTestEnvironment.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AzureBatchTestEnvironment.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AzureBatchTestEnvironment.java new file mode 100644 index 0000000..7076dc9 --- /dev/null +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AzureBatchTestEnvironment.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.tests; + +import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfiguration; +import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfigurationProvider; +import org.apache.reef.runtime.azbatch.driver.RuntimeIdentifier; +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.Injector; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.InjectionException; + +import java.io.IOException; + +/** + * A TestEnvironment for the Azure Batch resourcemanager. + */ +public final class AzureBatchTestEnvironment extends TestEnvironmentBase implements TestEnvironment { + + // Used to make sure the tests call the methods in the right order. + private boolean ready = false; + + @Override + public synchronized void setUp() { + this.ready = true; + } + + @Override + public synchronized Configuration getRuntimeConfiguration() { + assert this.ready; + try { + Configuration userConfiguration = AzureBatchRuntimeConfiguration.fromEnvironment(); + final Injector injector = Tang.Factory.getTang().newInjector(userConfiguration); + final AzureBatchRuntimeConfigurationProvider runtimeConfigurationProvider = + injector.getInstance(AzureBatchRuntimeConfigurationProvider.class); + return runtimeConfigurationProvider.getAzureBatchRuntimeConfiguration(); + } catch (IOException | InjectionException e) { + throw new RuntimeException(e); + } + } + + @Override + public synchronized void tearDown() { + assert this.ready; + this.ready = false; + } + + @Override + public int getTestTimeout() { + return 5 * 60000; // 5 min. + } + + @Override + public String getRuntimeName() { + return RuntimeIdentifier.RUNTIME_NAME; + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java index ab1454b..f716bec 100644 --- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java @@ -37,8 +37,9 @@ public final class TestEnvironmentFactory { public static TestEnvironment getNewTestEnvironment() { final boolean isYarn = Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")); final boolean isMesos = Boolean.parseBoolean(System.getenv("REEF_TEST_MESOS")); + final boolean isAzBatch = Boolean.parseBoolean(System.getenv("REEF_TEST_AZBATCH")); - if (isYarn && isMesos) { + if (isYarn ? (isMesos || isAzBatch) : (isMesos && isAzBatch)) { throw new RuntimeException("Cannot test on two runtimes at once"); } else if (isYarn) { LOG.log(Level.INFO, "Running tests on YARN"); @@ -46,6 +47,9 @@ public final class TestEnvironmentFactory { } else if (isMesos) { LOG.log(Level.INFO, "Running tests on Mesos"); return new MesosTestEnvironment(); + } else if (isAzBatch) { + LOG.log(Level.INFO, "Running tests on Azure Batch"); + return new AzureBatchTestEnvironment(); } else { LOG.log(Level.INFO, "Running tests on Local"); return new LocalTestEnvironment(); http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7d4431e..8d58a93 100644 --- a/pom.xml +++ b/pom.xml @@ -753,6 +753,13 @@ under the License. <artifactId>mesos</artifactId> <version>0.25.0</version> </dependency> + + <!-- Microsoft Azure Batch APIs --> + <dependency> + <groupId>com.microsoft.azure</groupId> + <artifactId>azure-batch</artifactId> + <version>3.0.0</version> + </dependency> </dependencies> </dependencyManagement> @@ -770,6 +777,7 @@ under the License. <module>lang/java/reef-experimental</module> <module>lang/java/reef-io</module> <module>lang/java/reef-poison</module> + <module>lang/java/reef-runtime-azbatch</module> <module>lang/java/reef-runtime-hdinsight</module> <module>lang/java/reef-runtime-local</module> <module>lang/java/reef-runtime-yarn</module>
