http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureBatchTaskStatusCheckPeriod.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureBatchTaskStatusCheckPeriod.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureBatchTaskStatusCheckPeriod.java new file mode 100644 index 0000000..b4dc7e4 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureBatchTaskStatusCheckPeriod.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.parameters; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * The period to check for task status from Azure Batch in ms. + */ +@NamedParameter(doc = "The period to check for task status from Azure Batch in ms.", default_value = "5000") +public final class AzureBatchTaskStatusCheckPeriod implements Name<Integer> { +}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountKey.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountKey.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountKey.java new file mode 100644 index 0000000..f82cf50 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountKey.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.parameters; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * The Azure Storage account key. + */ +@NamedParameter(doc = "The Azure Storage account key.") +public final class AzureStorageAccountKey implements Name<String> { +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountName.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountName.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountName.java new file mode 100644 index 0000000..8e0af3e --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountName.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.parameters; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * The Azure Storage account name. + */ +@NamedParameter(doc = "The Azure Storage account name.") +public final class AzureStorageAccountName implements Name<String> { +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageBlobSASTokenValidityHours.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageBlobSASTokenValidityHours.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageBlobSASTokenValidityHours.java new file mode 100644 index 0000000..4bd8644 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageBlobSASTokenValidityHours.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.parameters; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * The Azure Blob Shared Access Signature token validity in hours. + */ +@NamedParameter(doc = "The Azure Blob Shared Access Signature token validity in hours.", default_values = "1") +public final class AzureStorageBlobSASTokenValidityHours implements Name<Integer> { +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageContainerName.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageContainerName.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageContainerName.java new file mode 100644 index 0000000..ef32d9e --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageContainerName.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.parameters; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * The Azure Storage container name. + */ +@NamedParameter(doc = "The Azure Storage container name.") +public final class AzureStorageContainerName implements Name<String> { +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/ContainerIdentifier.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/ContainerIdentifier.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/ContainerIdentifier.java new file mode 100644 index 0000000..00139db --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/ContainerIdentifier.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.parameters; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * The container identifier for the evaluator shim. + */ +@NamedParameter(doc = "The container identifier.") +public final class ContainerIdentifier implements Name<String> { +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/EvaluatorShimConfigFilePath.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/EvaluatorShimConfigFilePath.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/EvaluatorShimConfigFilePath.java new file mode 100644 index 0000000..70ca902 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/EvaluatorShimConfigFilePath.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.parameters; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * Evaluator shim config file path parameter. + */ +@NamedParameter(doc = "The path to the configuration file for the evaluator shim.") +public final class EvaluatorShimConfigFilePath implements Name<String> { +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/IsWindows.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/IsWindows.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/IsWindows.java new file mode 100644 index 0000000..8b26db5 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/IsWindows.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.parameters; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * Whether or not the Azure Batch VMs are Windows based. + */ +@NamedParameter(doc = "Whether or not the Azure Batch VMs are Windows based.") +public final class IsWindows implements Name<Boolean> { +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/package-info.java new file mode 100644 index 0000000..e00140d --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Client for the REEF runtime for Azure Batch. + */ +package org.apache.reef.runtime.azbatch.parameters; http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/AzureBatchFileNames.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/AzureBatchFileNames.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/AzureBatchFileNames.java new file mode 100644 index 0000000..1eb4e05 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/AzureBatchFileNames.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util; + +import org.apache.reef.annotations.audience.Private; +import org.apache.reef.runtime.common.files.REEFFileNames; + +import javax.inject.Inject; + +/** + * Access to the various places things go according to the REEF Azure Batch runtime. + */ +@Private +public final class AzureBatchFileNames { + + private static final String STORAGE_JOB_FOLDER_PATH = "apps/reef/jobs/"; + private static final String TASK_JAR_FILE_NAME = "local.jar"; + private static final String EVALUATOR_RESOURCE_FILES_JAR_NAME = "resources.jar"; + + private static final String EVALUATOR_SHIM_CONFIGURATION_NAME = "shim.conf"; + private static final String TEXTFILE_EXTENSION = ".txt"; + + private final REEFFileNames reefFileNames; + + @Inject + private AzureBatchFileNames(final REEFFileNames reefFileNames) { + this.reefFileNames = reefFileNames; + } + + /** + * @return The relative path to the folder storing the job assets. + */ + public String getStorageJobFolder(final String jobId) { + return STORAGE_JOB_FOLDER_PATH + jobId; + } + + /** + * @return The name used within the current working directory of the driver to redirect standard error to. + */ + public String getEvaluatorStdErrFilename() { + return this.reefFileNames.getEvaluatorStderrFileName() + TEXTFILE_EXTENSION; + } + + /** + * @return The name used within the current working directory of the driver to redirect standard out to. + */ + public String getEvaluatorStdOutFilename() { + return this.reefFileNames.getEvaluatorStdoutFileName() + TEXTFILE_EXTENSION; + } + + /** + * @return The path to the evaluator shim configuration: + * REEF_BASE_FOLDER/LOCAL_FOLDER/EVALUATOR_SHIM_CONFIGURATION_NAME. + */ + public String getEvaluatorShimConfigurationPath() { + return this.reefFileNames.getLocalFolderPath() + "/" + EVALUATOR_SHIM_CONFIGURATION_NAME; + } + + /** + * @return The name of the evaluator resource JAR file. + */ + public String getEvaluatorResourceFilesJarName() { + return EVALUATOR_RESOURCE_FILES_JAR_NAME; + } + + /** + * @return The name of the evaluator configuration file. + */ + public String getEvaluatorShimConfigurationName() { + return EVALUATOR_SHIM_CONFIGURATION_NAME; + } + + /** + * @return The name under which the task jar will be stored. + */ + public static String getTaskJarFileName() { + return TASK_JAR_FILE_NAME; + } +} + http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/RemoteIdentifierParser.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/RemoteIdentifierParser.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/RemoteIdentifierParser.java new file mode 100644 index 0000000..3275648 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/RemoteIdentifierParser.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util; + +/** + * Utility class to parse IP and port information from Azure Batch Node ID. + */ +public final class RemoteIdentifierParser { + + private static final String PROTOCOL = "socket://"; + + private RemoteIdentifierParser() {} + + /** + * Get the IP address from the remote identifier. + * + * @param remoteIdentifier the remote identifier. + * @return the IP address. + */ + public static String parseIp(final String remoteIdentifier) { + return remoteIdentifier.substring(PROTOCOL.length(), remoteIdentifier.lastIndexOf(':')); + } + + /** + * Get the port from the remote identifier. + * + * @param remoteIdentifier the remote identifier. + * @return the port. + */ + public static int parsePort(final String remoteIdentifier) { + return Integer.parseInt(remoteIdentifier.substring(remoteIdentifier.lastIndexOf(':') + 1)); + } + + /** + * Get the node ID from the remote identifier. + * + * @param remoteIdentifier the remote identifier. + * @return the node ID. + */ + public static String parseNodeId(final String remoteIdentifier) { + return remoteIdentifier.substring(PROTOCOL.length()); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/TaskStatusMapper.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/TaskStatusMapper.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/TaskStatusMapper.java new file mode 100644 index 0000000..69d44e9 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/TaskStatusMapper.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util; + +import com.microsoft.azure.batch.protocol.models.CloudTask; +import com.microsoft.azure.batch.protocol.models.TaskExecutionResult; +import org.apache.reef.runtime.common.driver.evaluator.pojos.State; + +/** + * Class that maps status of Azure Batch task to a REEF task. + */ +public final class TaskStatusMapper { + + private TaskStatusMapper() { + } + + /** + * Get the {@link State} from a {@link CloudTask}. + * + * @param task the task. + * @return the state of the task. + */ + public static State getReefTaskState(final CloudTask task) { + switch (task.state()) { + case ACTIVE: + return State.INIT; + case RUNNING: + return State.RUNNING; + case COMPLETED: + if (task.executionInfo().result() == TaskExecutionResult.SUCCESS) { + return State.DONE; + } else { + return State.FAILED; + } + case PREPARING: + return State.RUNNING; + default: + throw new IllegalArgumentException("Azure batch cloud task has unknown state: " + task.state()); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/AzureBatchHelper.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/AzureBatchHelper.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/AzureBatchHelper.java new file mode 100644 index 0000000..e0753df --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/AzureBatchHelper.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.batch; + +import com.microsoft.azure.batch.BatchClient; +import com.microsoft.azure.batch.protocol.models.*; + +import org.apache.reef.runtime.azbatch.client.AzureBatchJobSubmissionHandler; +import org.apache.reef.runtime.azbatch.parameters.AzureBatchPoolId; +import org.apache.reef.runtime.azbatch.util.AzureBatchFileNames; +import org.apache.reef.runtime.azbatch.util.storage.SharedAccessSignatureCloudBlobClientProvider; +import org.apache.reef.tang.annotations.Parameter; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A helper class for Azure Batch. + */ +public final class AzureBatchHelper { + + private static final Logger LOG = Logger.getLogger(AzureBatchJobSubmissionHandler.class.getName()); + + /* + * Environment variable that contains the Azure Batch jobId. + */ + private static final String AZ_BATCH_JOB_ID_ENV = "AZ_BATCH_JOB_ID"; + + private final AzureBatchFileNames azureBatchFileNames; + + private final BatchClient client; + private final PoolInformation poolInfo; + + @Inject + public AzureBatchHelper( + final AzureBatchFileNames azureBatchFileNames, + final IAzureBatchCredentialProvider credentialProvider, + @Parameter(AzureBatchPoolId.class) final String azureBatchPoolId) { + this.azureBatchFileNames = azureBatchFileNames; + + this.client = BatchClient.open(credentialProvider.getCredentials()); + this.poolInfo = new PoolInformation().withPoolId(azureBatchPoolId); + } + + /** + * Create a job on Azure Batch. + * + * @param applicationId the ID of the application. + * @param storageContainerSAS the publicly accessible uri to the job container. + * @param jobJarUri the publicly accessible uri to the job jar directory. + * @param command the commandline argument to execute the job. + * @throws IOException + */ + public void submitJob(final String applicationId, final String storageContainerSAS, final URI jobJarUri, + final String command) throws IOException { + ResourceFile jarResourceFile = new ResourceFile() + .withBlobSource(jobJarUri.toString()) + .withFilePath(AzureBatchFileNames.getTaskJarFileName()); + + // This setting will signal Batch to generate an access token and pass it to the Job Manager Task (aka the Driver) + // as an environment variable. + // See https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.batch.cloudtask.authenticationtokensettings + // for more info. + AuthenticationTokenSettings authenticationTokenSettings = new AuthenticationTokenSettings(); + authenticationTokenSettings.withAccess(Collections.singletonList(AccessScope.JOB)); + + EnvironmentSetting environmentSetting = new EnvironmentSetting() + .withName(SharedAccessSignatureCloudBlobClientProvider.AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV) + .withValue(storageContainerSAS); + + JobManagerTask jobManagerTask = new JobManagerTask() + .withRunExclusive(false) + .withId(applicationId) + .withResourceFiles(Collections.singletonList(jarResourceFile)) + .withEnvironmentSettings(Collections.singletonList(environmentSetting)) + .withAuthenticationTokenSettings(authenticationTokenSettings) + .withCommandLine(command); + + LOG.log(Level.INFO, "Job Manager (aka driver) task command: " + command); + + JobAddParameter jobAddParameter = new JobAddParameter() + .withId(applicationId) + .withJobManagerTask(jobManagerTask) + .withPoolInfo(poolInfo); + + client.jobOperations().createJob(jobAddParameter); + } + + /** + * Adds a single task to a job on Azure Batch. + * + * @param jobId the ID of the job. + * @param taskId the ID of the task. + * @param jobJarUri the publicly accessible uri list to the job jar directory. + * @param confUri the publicly accessible uri list to the job configuration directory. + * @param command the commandline argument to execute the job. + * @throws IOException + */ + public void submitTask(final String jobId, final String taskId, final URI jobJarUri, + final URI confUri, final String command) + throws IOException { + + final List<ResourceFile> resources = new ArrayList<>(); + + final ResourceFile jarSourceFile = new ResourceFile() + .withBlobSource(jobJarUri.toString()) + .withFilePath(AzureBatchFileNames.getTaskJarFileName()); + resources.add(jarSourceFile); + + final ResourceFile confSourceFile = new ResourceFile() + .withBlobSource(confUri.toString()) + .withFilePath(this.azureBatchFileNames.getEvaluatorShimConfigurationPath()); + resources.add(confSourceFile); + + LOG.log(Level.INFO, "Evaluator task command: " + command); + + final TaskAddParameter taskAddParameter = new TaskAddParameter() + .withId(taskId) + .withResourceFiles(resources) + .withCommandLine(command); + + this.client.taskOperations().createTask(jobId, taskAddParameter); + } + + /** + * List the tasks of the specified job. + * + * @param jobId the ID of the job. + * @return A list of CloudTask objects. + */ + public List<CloudTask> getTaskStatusForJob(final String jobId) { + List<CloudTask> tasks = null; + try { + tasks = client.taskOperations().listTasks(jobId); + LOG.log(Level.INFO, "Task status for job: {0} returned {1} tasks", new Object[]{jobId, tasks.size()}); + } catch (IOException | BatchErrorException ex) { + LOG.log(Level.SEVERE, "Exception when fetching Task status for job: {0}. Exception [{1}]:[2]", + new Object[]{jobId, ex.getMessage(), ex.getStackTrace()}); + } + + return tasks; + } + + /** + * @return the job ID specified in the current system environment. + */ + public String getAzureBatchJobId() { + return System.getenv(AZ_BATCH_JOB_ID_ENV); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/IAzureBatchCredentialProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/IAzureBatchCredentialProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/IAzureBatchCredentialProvider.java new file mode 100644 index 0000000..681337d --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/IAzureBatchCredentialProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.batch; + +import com.microsoft.azure.batch.auth.BatchCredentials; + +/** + * An interface for Azure Batch credential providers. + */ +public interface IAzureBatchCredentialProvider { + + /** + * Returns {@link BatchCredentials} for Azure Batch account. + * @return an implementation of {@link BatchCredentials}. + */ + BatchCredentials getCredentials(); +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/SharedKeyBatchCredentialProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/SharedKeyBatchCredentialProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/SharedKeyBatchCredentialProvider.java new file mode 100644 index 0000000..3e40319 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/SharedKeyBatchCredentialProvider.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.batch; + +import com.microsoft.azure.batch.auth.BatchCredentials; +import com.microsoft.azure.batch.auth.BatchSharedKeyCredentials; +import org.apache.reef.annotations.audience.Private; +import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountKey; +import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountName; +import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountUri; +import org.apache.reef.tang.annotations.Parameter; + +import javax.inject.Inject; + +/** + * An implementation of {@link IAzureBatchCredentialProvider} which returns {@link BatchSharedKeyCredentials} + * for Azure Batch account. + */ +@Private +public final class SharedKeyBatchCredentialProvider implements IAzureBatchCredentialProvider { + + private final String azureBatchAccountUri; + private final String azureBatchAccountName; + private final String azureBatchAccountKey; + + @Inject + private SharedKeyBatchCredentialProvider(@Parameter(AzureBatchAccountUri.class) final String azureBatchAccountUri, + @Parameter(AzureBatchAccountName.class) final String azureBatchAccountName, + @Parameter(AzureBatchAccountKey.class) final String azureBatchAccountKey) { + this.azureBatchAccountUri = azureBatchAccountUri; + this.azureBatchAccountName = azureBatchAccountName; + this.azureBatchAccountKey = azureBatchAccountKey; + } + + /** + * Returns credentials for Azure Batch account. + * @return an instance of {@link BatchSharedKeyCredentials} based on {@link AzureBatchAccountKey} parameter. + */ + @Override + public BatchCredentials getCredentials() { + return new BatchSharedKeyCredentials(this.azureBatchAccountUri, this.azureBatchAccountName, azureBatchAccountKey); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/TokenBatchCredentialProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/TokenBatchCredentialProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/TokenBatchCredentialProvider.java new file mode 100644 index 0000000..a3b8caa --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/TokenBatchCredentialProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.batch; + +import com.microsoft.azure.batch.auth.BatchCredentials; +import com.microsoft.rest.credentials.TokenCredentials; +import okhttp3.OkHttpClient; +import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountUri; +import org.apache.reef.tang.annotations.Parameter; + +import javax.inject.Inject; + +/** + * An implementation of {@link IAzureBatchCredentialProvider} which returns credentials based on + * the authentication token provided in AZ_BATCH_AUTHENTICATION_TOKEN environment variable. This environment variable + * is populated by Azure Batch and allows the task running in Azure Batch perform certain operations on the job. + * + * See: https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.batch.cloudtask.authenticationtokensettings + * for more info. + */ +public class TokenBatchCredentialProvider implements IAzureBatchCredentialProvider { + + private static final String AZ_BATCH_AUTH_TOKEN_ENV = "AZ_BATCH_AUTHENTICATION_TOKEN"; + + private final String azureBatchAccountUri; + + @Inject + TokenBatchCredentialProvider(@Parameter(AzureBatchAccountUri.class) final String azureBatchAccountUri) { + this.azureBatchAccountUri = azureBatchAccountUri; + } + + /** + * Returns credentials for Azure Batch account. + * @return an implementation of {@link BatchCredentials} which is based on the token provided by Azure Batch. + */ + @Override + public BatchCredentials getCredentials() { + + final TokenCredentials tokenCredentials = new TokenCredentials(null, System.getenv(AZ_BATCH_AUTH_TOKEN_ENV)); + + return new BatchCredentials() { + @Override + public String baseUrl() { + return azureBatchAccountUri; + } + + @Override + public void applyCredentialsFilter(final OkHttpClient.Builder builder) { + tokenCredentials.applyCredentialsFilter(builder); + } + }; + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/package-info.java new file mode 100644 index 0000000..830749e --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Azure Batch utilities for REEF Azure Batch runtime. + */ +package org.apache.reef.runtime.azbatch.util.batch; http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/AbstractCommandBuilder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/AbstractCommandBuilder.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/AbstractCommandBuilder.java new file mode 100644 index 0000000..9a6cc15 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/AbstractCommandBuilder.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.command; + +import org.apache.commons.lang.StringUtils; +import org.apache.reef.driver.evaluator.EvaluatorProcess; +import org.apache.reef.runtime.azbatch.util.AzureBatchFileNames; +import org.apache.reef.runtime.common.client.api.JobSubmissionEvent; +import org.apache.reef.runtime.common.driver.api.ResourceLaunchEvent; +import org.apache.reef.runtime.common.files.ClasspathProvider; +import org.apache.reef.runtime.common.files.REEFFileNames; +import org.apache.reef.runtime.common.files.RuntimePathProvider; +import org.apache.reef.runtime.common.launch.JavaLaunchCommandBuilder; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Abstract implementation of the OS command builder. + */ +public abstract class AbstractCommandBuilder implements CommandBuilder { + + private final Class launcherClass; + private final Class shimLauncherClass; + private final List<String> commandListPrefix; + private final String osCommandFormat; + private final RuntimePathProvider runtimePathProvider; + + protected final ClasspathProvider classpathProvider; + protected final REEFFileNames reefFileNames; + protected final AzureBatchFileNames azureBatchFileNames; + + AbstractCommandBuilder( + final Class launcherClass, + final Class shimLauncherClass, + final List<String> commandListPrefix, + final String osCommandFormat, + final ClasspathProvider classpathProvider, + final RuntimePathProvider runtimePathProvider, + final REEFFileNames reefFileNames, + final AzureBatchFileNames azureBatchFileNames) { + this.launcherClass = launcherClass; + this.shimLauncherClass = shimLauncherClass; + this.commandListPrefix = commandListPrefix; + this.osCommandFormat = osCommandFormat; + + this.classpathProvider = classpathProvider; + this.reefFileNames = reefFileNames; + this.azureBatchFileNames = azureBatchFileNames; + this.runtimePathProvider = runtimePathProvider; + } + + public String buildDriverCommand(final JobSubmissionEvent jobSubmissionEvent) { + List<String> commandList = new JavaLaunchCommandBuilder(this.launcherClass, this.commandListPrefix) + .setJavaPath(runtimePathProvider.getPath()) + .setConfigurationFilePaths(Collections.singletonList(this.reefFileNames.getDriverConfigurationPath())) + .setClassPath(getDriverClasspath()) + .setMemory(jobSubmissionEvent.getDriverMemory().get()) + .build(); + return String.format(this.osCommandFormat, StringUtils.join(commandList, ' ')); + } + + public String buildEvaluatorShimCommand(final int evaluatorShimMemory, final String configurationPath) { + List<String> commandList = new JavaLaunchCommandBuilder(this.shimLauncherClass, this.commandListPrefix) + .setJavaPath(runtimePathProvider.getPath()) + .setConfigurationFilePaths(Collections.singletonList(configurationPath)) + .setClassPath(getEvaluatorShimClasspath()) + .setMemory(evaluatorShimMemory) + .build(); + return String.format(this.osCommandFormat, StringUtils.join(commandList, ' ')); + } + + public String buildEvaluatorCommand(final ResourceLaunchEvent resourceLaunchEvent, + final int containerMemory, final double jvmHeapFactor) { + List<String> commandList = new ArrayList<>(); + + // Use EvaluatorProcess to be compatible with JVMProcess and CLRProcess + final EvaluatorProcess process = resourceLaunchEvent.getProcess() + .setConfigurationFileName(this.reefFileNames.getEvaluatorConfigurationPath()); + + if (process.isOptionSet()) { + commandList.addAll(process.getCommandLine()); + } else { + commandList.addAll(process.setMemory((int) (jvmHeapFactor * containerMemory)).getCommandLine()); + } + + return StringUtils.join(commandList, ' '); + } + + /** + * Returns the driver classpath string which is compatible with the intricacies of the OS. + * + * @return classpath parameter string. + */ + protected abstract String getDriverClasspath(); + + /** + * Returns the evaluator shim classpath string which is compatible with the intricacies of the OS. + * + * @return classpath parameter string. + */ + protected abstract String getEvaluatorShimClasspath(); +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/CommandBuilder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/CommandBuilder.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/CommandBuilder.java new file mode 100644 index 0000000..96cd716 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/CommandBuilder.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.command; + +import org.apache.reef.runtime.common.client.api.JobSubmissionEvent; +import org.apache.reef.runtime.common.driver.api.ResourceLaunchEvent; + +/** + * Build the launch command for Java REEF processes for Azure Batch. + */ +public interface CommandBuilder { + + /** + * Assembles the command to execute the Driver. + * + * @param jobSubmissionEvent the submission event for which to build the command. + * @return the command string. + */ + String buildDriverCommand(JobSubmissionEvent jobSubmissionEvent); + + /** + * Assembles the command to execute the Evaluator Shim. + * + * @param evaluatorShimMemory the maximum size in, in bytes, of the memory allocation pool. + * @param configurationPath the relative path to the configuration file. + * @return the command string. + */ + String buildEvaluatorShimCommand(int evaluatorShimMemory, String configurationPath); + + /** + * Assembles the command to execute the Evaluator. + * + * @param resourceLaunchEvent the launch event for which to build the command. + * @param containerMemory the maximum size in, in bytes, of the memory allocation pool. + * @param jvmHeapFactor a multiplicative factor to increase the container memory. + * @return the command string. + */ + String buildEvaluatorCommand(final ResourceLaunchEvent resourceLaunchEvent, + final int containerMemory, final double jvmHeapFactor); +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/LinuxCommandBuilder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/LinuxCommandBuilder.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/LinuxCommandBuilder.java new file mode 100644 index 0000000..adb58f5 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/LinuxCommandBuilder.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.command; + +import org.apache.commons.lang.StringUtils; +import org.apache.reef.annotations.audience.Private; +import org.apache.reef.runtime.azbatch.evaluator.EvaluatorShimLauncher; +import org.apache.reef.runtime.azbatch.util.AzureBatchFileNames; +import org.apache.reef.runtime.common.REEFLauncher; +import org.apache.reef.runtime.common.files.ClasspathProvider; +import org.apache.reef.runtime.common.files.REEFFileNames; +import org.apache.reef.runtime.common.files.RuntimePathProvider; + +import javax.inject.Inject; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Build the launch command for Java REEF processes for Azure Batch Linux pools. + */ +@Private +public class LinuxCommandBuilder extends AbstractCommandBuilder { + + private static final Class LAUNCHER_CLASS = REEFLauncher.class; + private static final Class SHIM_LAUNCHER_CLASS = EvaluatorShimLauncher.class; + private static final List<String> COMMAND_LIST_PREFIX = + Collections.unmodifiableList(Arrays.asList( + "unzip " + AzureBatchFileNames.getTaskJarFileName() + " -d 'reef/'" + ";")); + private static final char CLASSPATH_SEPARATOR_CHAR = ':'; + private static final String OS_COMMAND_FORMAT = "/bin/sh -c \"%s\""; + + @Inject + LinuxCommandBuilder( + final ClasspathProvider classpathProvider, + final RuntimePathProvider runtimePathProvider, + final REEFFileNames reefFileNames, + final AzureBatchFileNames azureBatchFileNames) { + super(LAUNCHER_CLASS, SHIM_LAUNCHER_CLASS, COMMAND_LIST_PREFIX, OS_COMMAND_FORMAT, + classpathProvider, runtimePathProvider, reefFileNames, azureBatchFileNames); + } + + @Override + protected String getDriverClasspath() { + return StringUtils.join(super.classpathProvider.getDriverClasspath(), CLASSPATH_SEPARATOR_CHAR); + } + + @Override + protected String getEvaluatorShimClasspath() { + return StringUtils.join(super.classpathProvider.getEvaluatorClasspath(), CLASSPATH_SEPARATOR_CHAR); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/WindowsCommandBuilder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/WindowsCommandBuilder.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/WindowsCommandBuilder.java new file mode 100644 index 0000000..f382bef --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/WindowsCommandBuilder.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.command; + +import org.apache.commons.lang.StringUtils; +import org.apache.reef.annotations.audience.Private; +import org.apache.reef.runtime.azbatch.evaluator.EvaluatorShimLauncher; +import org.apache.reef.runtime.azbatch.util.AzureBatchFileNames; +import org.apache.reef.runtime.common.REEFLauncher; +import org.apache.reef.runtime.common.files.ClasspathProvider; +import org.apache.reef.runtime.common.files.REEFFileNames; +import org.apache.reef.runtime.common.files.RuntimePathProvider; + +import javax.inject.Inject; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Build the launch command for Java REEF processes for Azure Batch Windows pools. + */ +@Private +public class WindowsCommandBuilder extends AbstractCommandBuilder { + + private static final Class LAUNCHER_CLASS = REEFLauncher.class; + private static final Class SHIM_LAUNCHER_CLASS = EvaluatorShimLauncher.class; + private static final List<String> COMMAND_LIST_PREFIX = Collections.unmodifiableList( + Arrays.asList( + "Add-Type -AssemblyName System.IO.Compression.FileSystem; ", + "[System.IO.Compression.ZipFile]::ExtractToDirectory(\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\" + + AzureBatchFileNames.getTaskJarFileName() + "\\\", " + + "\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\reef\\\"); ") + ); + private static final char CLASSPATH_SEPARATOR_CHAR = ';'; + private static final String OS_COMMAND_FORMAT = "powershell.exe /c \"%s\";"; + + @Inject + WindowsCommandBuilder( + final ClasspathProvider classpathProvider, + final RuntimePathProvider runtimePathProvider, + final REEFFileNames reefFileNames, + final AzureBatchFileNames azureBatchFileNames) { + super(LAUNCHER_CLASS, SHIM_LAUNCHER_CLASS, COMMAND_LIST_PREFIX, OS_COMMAND_FORMAT, + classpathProvider, runtimePathProvider, reefFileNames, azureBatchFileNames); + } + + @Override + protected String getDriverClasspath() { + return String.format("'%s'", StringUtils.join( + super.classpathProvider.getDriverClasspath(), CLASSPATH_SEPARATOR_CHAR)); + } + + @Override + protected String getEvaluatorShimClasspath() { + return String.format("'%s'", StringUtils.join( + super.classpathProvider.getEvaluatorClasspath(), CLASSPATH_SEPARATOR_CHAR)); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/package-info.java new file mode 100644 index 0000000..edd04b9 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Command line utilities for Azure Batch runtime. + */ +package org.apache.reef.runtime.azbatch.util.command; http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/package-info.java new file mode 100644 index 0000000..622e972 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * A REEF runtime for Azure Batch. + */ +package org.apache.reef.runtime.azbatch.util; http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/AzureStorageClient.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/AzureStorageClient.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/AzureStorageClient.java new file mode 100644 index 0000000..83b522b --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/AzureStorageClient.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.storage; + +import com.microsoft.windowsazure.storage.StorageException; +import com.microsoft.windowsazure.storage.blob.*; +import org.apache.reef.annotations.audience.Private; +import org.apache.reef.runtime.azbatch.parameters.AzureStorageBlobSASTokenValidityHours; +import org.apache.reef.runtime.azbatch.parameters.AzureStorageContainerName; +import org.apache.reef.tang.annotations.Parameter; + +import javax.inject.Inject; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.InvalidKeyException; +import java.util.Calendar; +import java.util.Date; +import java.util.EnumSet; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Azure storage utility to upload Driver and Evaluator jars to blob storage + * and generate SAS URIs. + */ +@Private +public final class AzureStorageClient { + private static final Logger LOG = Logger.getLogger(AzureStorageClient.class.getName()); + + private final ICloudBlobClientProvider cloudBlobClientProvider; + + private final String azureStorageContainerName; + private final int blobSASTokenValidityHours; + + @Inject + AzureStorageClient( + final ICloudBlobClientProvider cloudBlobClientProvider, + @Parameter(AzureStorageContainerName.class) final String azureStorageContainerName, + @Parameter(AzureStorageBlobSASTokenValidityHours.class) final int blobSASTokenValidityHours) { + this.cloudBlobClientProvider = cloudBlobClientProvider; + this.azureStorageContainerName = azureStorageContainerName; + this.blobSASTokenValidityHours = blobSASTokenValidityHours; + } + + public URI getJobSubmissionFolderUri(final String jobFolder) throws IOException { + final CloudBlobClient cloudBlobClient = this.cloudBlobClientProvider.getCloudBlobClient(); + try { + final CloudBlobContainer container = cloudBlobClient.getContainerReference(this.azureStorageContainerName); + return container.getDirectoryReference(jobFolder).getUri(); + } catch (URISyntaxException | StorageException e) { + throw new IOException("Failed to get the job submission folder URI", e); + } + } + + + public String createContainerSharedAccessSignature() throws IOException { + try { + CloudBlobClient cloudBlobClient = this.cloudBlobClientProvider.getCloudBlobClient(); + CloudBlobContainer cloudBlobContainer = cloudBlobClient.getContainerReference(this.azureStorageContainerName); + cloudBlobContainer.createIfNotExists(); + + return cloudBlobContainer.generateSharedAccessSignature(getSharedAccessContainerPolicy(), null); + + } catch (StorageException | URISyntaxException | InvalidKeyException e) { + throw new IOException("Failed to generate a shared access signature for storage container.", e); + } + } + + + /** + * Upload a file to the storage account. + * + * @param jobFolder the path to the destination folder within storage container. + * @param file the source file. + * @return the SAS URI to the uploaded file. + * @throws IOException + */ + public URI uploadFile(final String jobFolder, final File file) throws IOException { + + LOG.log(Level.INFO, "Uploading [{0}] to [{1}]", new Object[]{file, jobFolder}); + + try { + final CloudBlobClient cloudBlobClient = this.cloudBlobClientProvider.getCloudBlobClient(); + final CloudBlobContainer container = cloudBlobClient.getContainerReference(this.azureStorageContainerName); + + final String destination = String.format("%s/%s", jobFolder, file.getName()); + final CloudBlockBlob blob = container.getBlockBlobReference(destination); + + try (FileInputStream fis = new FileInputStream(file)) { + blob.upload(fis, file.length()); + } + + LOG.log(Level.FINE, "Uploaded to: {0}", blob.getStorageUri().getPrimaryUri()); + return this.cloudBlobClientProvider.generateSharedAccessSignature(blob, getSharedAccessBlobPolicy()); + + } catch (final URISyntaxException | StorageException e) { + throw new IOException(e); + } + } + + private SharedAccessBlobPolicy getSharedAccessBlobPolicy() { + return getSharedAccessBlobPolicy(EnumSet.of(SharedAccessBlobPermissions.READ)); + } + + private SharedAccessBlobPolicy getSharedAccessContainerPolicy() { + return getSharedAccessBlobPolicy(EnumSet.of(SharedAccessBlobPermissions.READ, SharedAccessBlobPermissions.WRITE)); + } + + private SharedAccessBlobPolicy getSharedAccessBlobPolicy( + final EnumSet<SharedAccessBlobPermissions> permissions) { + + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.HOUR, this.blobSASTokenValidityHours); + Date tokenExpirationDate = calendar.getTime(); + + final SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy(); + policy.setPermissions(permissions); + policy.setSharedAccessStartTime(Calendar.getInstance().getTime()); + policy.setSharedAccessExpiryTime(tokenExpirationDate); + + return policy; + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/ICloudBlobClientProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/ICloudBlobClientProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/ICloudBlobClientProvider.java new file mode 100644 index 0000000..d51dbc2 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/ICloudBlobClientProvider.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.storage; + +import com.microsoft.windowsazure.storage.blob.CloudBlob; +import com.microsoft.windowsazure.storage.blob.CloudBlobClient; +import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy; + +import java.io.IOException; +import java.net.URI; + +/** + * An interface for classes that provide an instance of {@link CloudBlobClient} based + * on available authentication mechanism. + */ +public interface ICloudBlobClientProvider { + + /** + * Returns an instance of {@link CloudBlobClient} based on available authentication mechanism. + * @return an instance of {@link CloudBlobClient}. + * @throws IOException + */ + CloudBlobClient getCloudBlobClient() throws IOException; + + /** + * Generates a Shared Access Key URI for the given {@link CloudBlob}. + * @param cloudBlob cloud blob to create a Shared Access Key URI for. + * @param policy an instance of {@link SharedAccessBlobPolicy} that specifies permissions and signature's + * validity time period. + * @return a Shared Access Key URI for the given {@link CloudBlob}. + * @throws IOException + */ + URI generateSharedAccessSignature(final CloudBlob cloudBlob, final SharedAccessBlobPolicy policy) + throws IOException; +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/SharedAccessSignatureCloudBlobClientProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/SharedAccessSignatureCloudBlobClientProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/SharedAccessSignatureCloudBlobClientProvider.java new file mode 100644 index 0000000..e678445 --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/SharedAccessSignatureCloudBlobClientProvider.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.storage; + +import com.microsoft.windowsazure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.windowsazure.storage.StorageException; +import com.microsoft.windowsazure.storage.blob.CloudBlob; +import com.microsoft.windowsazure.storage.blob.CloudBlobClient; +import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy; +import com.microsoft.windowsazure.storage.core.PathUtility; +import com.microsoft.windowsazure.storage.core.UriQueryBuilder; +import org.apache.reef.runtime.azbatch.parameters.AzureStorageAccountName; +import org.apache.reef.tang.annotations.Parameter; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Cloud Blob client provider that uses Azure Storage Shared Access Signature authorization. + */ +public final class SharedAccessSignatureCloudBlobClientProvider implements ICloudBlobClientProvider { + + private static final Logger LOG = Logger.getLogger(AzureStorageClient.class.getName()); + + public static final String AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV = "AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV"; + + private static final String AZURE_STORAGE_ACCOUNT_URI_FORMAT = "https://%s.blob.core.windows.net";; + + private final String azureStorageAccountName; + private final String azureStorageContainerSASToken; + + @Inject + SharedAccessSignatureCloudBlobClientProvider( + @Parameter(AzureStorageAccountName.class) final String azureStorageAccountName) { + this.azureStorageAccountName = azureStorageAccountName; + this.azureStorageContainerSASToken = System.getenv(AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV); + } + + /** + * Returns an instance of {@link CloudBlobClient} based on available authentication mechanism. + * @return an instance of {@link CloudBlobClient}. + * @throws IOException + */ + @Override + public CloudBlobClient getCloudBlobClient() throws IOException { + StorageCredentialsSharedAccessSignature signature = + new StorageCredentialsSharedAccessSignature(this.azureStorageContainerSASToken); + URI storageAccountUri; + try { + storageAccountUri = new URI(String.format(AZURE_STORAGE_ACCOUNT_URI_FORMAT, this.azureStorageAccountName)); + } catch (URISyntaxException e) { + throw new IOException("Failed to generate Storage Account URI", e); + } + + return new CloudBlobClient(storageAccountUri, signature); + } + + /** + * Generates a Shared Access Key URI for the given {@link CloudBlob}. + * @param cloudBlob cloud blob to create a Shared Access Key URI for. + * @param policy an instance of {@link SharedAccessBlobPolicy} that specifies permissions and signature's + * validity time period. + * @return a Shared Access Key URI for the given {@link CloudBlob}. + * @throws IOException + */ + @Override + public URI generateSharedAccessSignature(final CloudBlob cloudBlob, final SharedAccessBlobPolicy policy) + throws IOException { + try { + final URI uri = cloudBlob.getStorageUri().getPrimaryUri(); + + Map<String, String[]> queryString = PathUtility.parseQueryString(this.azureStorageContainerSASToken); + + UriQueryBuilder builder = new UriQueryBuilder(); + for (Map.Entry<String, String[]> entry : queryString.entrySet()) { + for (String value : entry.getValue()) { + builder.add(entry.getKey(), value); + } + } + + URI result = builder.addToURI(uri); + LOG.log(Level.INFO, "Here's the URI: " + result); + + return result; + } catch (StorageException | URISyntaxException e) { + throw new IOException("Failed to generated a Shared Access Signature.", e); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/StorageKeyCloudBlobProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/StorageKeyCloudBlobProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/StorageKeyCloudBlobProvider.java new file mode 100644 index 0000000..ae8613d --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/StorageKeyCloudBlobProvider.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.azbatch.util.storage; + +import com.microsoft.windowsazure.storage.CloudStorageAccount; +import com.microsoft.windowsazure.storage.StorageException; +import com.microsoft.windowsazure.storage.blob.CloudBlob; +import com.microsoft.windowsazure.storage.blob.CloudBlobClient; +import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy; +import org.apache.reef.annotations.audience.Private; +import org.apache.reef.runtime.azbatch.parameters.AzureStorageAccountKey; +import org.apache.reef.runtime.azbatch.parameters.AzureStorageAccountName; +import org.apache.reef.tang.annotations.Parameter; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.InvalidKeyException; + +/** + * Cloud Blob client provider that uses Azure Storage Shared Key authorization. + */ +@Private +public final class StorageKeyCloudBlobProvider implements ICloudBlobClientProvider { + + private static final String AZURE_STORAGE_CONNECTION_STRING_FORMAT = + "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s"; + + private final String azureStorageAccountName; + private final String azureStorageAccountKey; + + @Inject + StorageKeyCloudBlobProvider( + @Parameter(AzureStorageAccountName.class) final String azureStorageAccountName, + @Parameter(AzureStorageAccountKey.class) final String azureStorageAccountKey) { + this.azureStorageAccountName = azureStorageAccountName; + this.azureStorageAccountKey = azureStorageAccountKey; + } + + /** + * Returns an instance of {@link CloudBlobClient} based on available authentication mechanism. + * @return an instance of {@link CloudBlobClient}. + * @throws IOException + */ + @Override + public CloudBlobClient getCloudBlobClient() throws IOException { + String connectionString = String.format(AZURE_STORAGE_CONNECTION_STRING_FORMAT, + this.azureStorageAccountName, this.azureStorageAccountKey); + try { + return CloudStorageAccount.parse(connectionString).createCloudBlobClient(); + } catch (URISyntaxException | InvalidKeyException e) { + throw new IOException("Failed to create a Cloud Storage Account.", e); + } + } + + /** + * Generates a Shared Access Key URI for the given {@link CloudBlob}. + * @param cloudBlob cloud blob to create a Shared Access Key URI for. + * @param policy an instance of {@link SharedAccessBlobPolicy} that specifies permissions and signature's + * validity time period. + * @return a Shared Access Key URI for the given {@link CloudBlob}. + * @throws IOException + */ + @Override + public URI generateSharedAccessSignature(final CloudBlob cloudBlob, final SharedAccessBlobPolicy policy) + throws IOException { + try { + final String sas = cloudBlob.generateSharedAccessSignature(policy, null); + final String uri = cloudBlob.getStorageUri().getPrimaryUri().toString(); + return new URI(uri + "?" + sas); + } catch (StorageException | InvalidKeyException | URISyntaxException e) { + throw new IOException("Failed to generated a Shared Access Signature.", e); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/package-info.java new file mode 100644 index 0000000..adbe05f --- /dev/null +++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Azure Storage utilities for Azure Batch runtime. + */ +package org.apache.reef.runtime.azbatch.util.storage;
