HADOOP-11959. WASB should configure client side socket timeout in storage client blob request options. Contributed by Ivan Mitic.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44f4df57 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44f4df57 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44f4df57 Branch: refs/heads/YARN-2928 Commit: 44f4df57bfe7a078894cc4c05b07c0e3a292412f Parents: 4b4aeba Author: cnauroth <cnaur...@apache.org> Authored: Thu May 28 12:31:06 2015 -0700 Committer: Zhijie Shen <zjs...@apache.org> Committed: Tue Jun 2 16:12:56 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-project/pom.xml | 2 +- .../hadoop/fs/azure/AzureNativeFileSystemStore.java | 13 ++----------- .../org/apache/hadoop/fs/azure/StorageInterface.java | 6 +++--- .../apache/hadoop/fs/azure/StorageInterfaceImpl.java | 4 ++-- .../apache/hadoop/fs/azure/MockStorageInterface.java | 4 ++-- .../fs/azure/TestAzureFileSystemErrorConditions.java | 1 + .../apache/hadoop/fs/azure/TestBlobDataValidation.java | 1 + 8 files changed, 15 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6aa224c..aba9087 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -791,6 +791,9 @@ Release 2.8.0 - UNRELEASED HADOOP-11930. test-patch in offline mode should tell maven to be in offline mode (Sean Busbey via aw) + HADOOP-11959. WASB should configure client side socket timeout in storage + client blob request options. (Ivan Mitic via cnauroth) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 94e1f5e..a5d4ed8 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -942,7 +942,7 @@ <dependency> <groupId>com.microsoft.azure</groupId> <artifactId>azure-storage</artifactId> - <version>2.0.0</version> + <version>2.2.0</version> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 3267d8b..69bda06 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -2434,15 +2434,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { // CloudBlobWrapper dstBlob = getBlobReference(dstKey); - // TODO: Remove at the time when we move to Azure Java SDK 1.2+. - // This is the workaround provided by Azure Java SDK team to - // mitigate the issue with un-encoded x-ms-copy-source HTTP - // request header. Azure sdk version before 1.2+ does not encode this - // header what causes all URIs that have special (category "other") - // characters in the URI not to work with startCopyFromBlob when - // specified as source (requests fail with HTTP 403). - URI srcUri = new URI(srcBlob.getUri().toASCIIString()); - // Rename the source blob to the destination blob by copying it to // the destination blob then deleting it. // @@ -2451,7 +2442,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { // a more intensive exponential retry policy when the cluster is getting // throttled. try { - dstBlob.startCopyFromBlob(srcUri, null, getInstrumentedContext()); + dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext()); } catch (StorageException se) { if (se.getErrorCode().equals( StorageErrorCode.SERVER_BUSY.toString())) { @@ -2475,7 +2466,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { options.setRetryPolicyFactory(new RetryExponentialRetry( copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, copyBlobMaxRetries)); - dstBlob.startCopyFromBlob(srcUri, options, getInstrumentedContext()); + dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext()); } else { throw se; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java index e89151d..ce5f749 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java @@ -381,8 +381,8 @@ abstract class StorageInterface { * Copies an existing blob's contents, properties, and metadata to this instance of the <code>CloudBlob</code> * class, using the specified operation context. * - * @param source - * A <code>java.net.URI</code> The URI of a source blob. + * @param sourceBlob + * A <code>CloudBlob</code> object that represents the source blob to copy. * @param options * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying * <code>null</code> will use the default request options from the associated service client ( @@ -397,7 +397,7 @@ abstract class StorageInterface { * @throws URISyntaxException * */ - public abstract void startCopyFromBlob(URI source, + public abstract void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options, OperationContext opContext) throws StorageException, URISyntaxException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java index 90d4d88..382ff66 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java @@ -393,10 +393,10 @@ class StorageInterfaceImpl extends StorageInterface { } @Override - public void startCopyFromBlob(URI source, BlobRequestOptions options, + public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options, OperationContext opContext) throws StorageException, URISyntaxException { - getBlob().startCopyFromBlob(source, + getBlob().startCopyFromBlob(((CloudBlobWrapperImpl)sourceBlob).blob, null, null, options, opContext); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java index cde0e38..9f84f4b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java @@ -429,9 +429,9 @@ public class MockStorageInterface extends StorageInterface { } @Override - public void startCopyFromBlob(URI source, BlobRequestOptions options, + public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options, OperationContext opContext) throws StorageException, URISyntaxException { - backingStore.copy(convertUriToDecodedString(source), convertUriToDecodedString(uri)); + backingStore.copy(convertUriToDecodedString(sourceBlob.getUri()), convertUriToDecodedString(uri)); //TODO: set the backingStore.properties.CopyState and // update azureNativeFileSystemStore.waitForCopyToComplete } http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java index ace57dc..810bcf7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java @@ -205,6 +205,7 @@ public class TestAzureFileSystemErrorConditions { @Override public boolean isTargetConnection(HttpURLConnection connection) { return connection.getRequestMethod().equals("PUT") + && connection.getURL().getQuery() != null && connection.getURL().getQuery().contains("blocklist"); } }); http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java index 9237ade..c40b7b5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java @@ -191,6 +191,7 @@ public class TestBlobDataValidation { private static boolean isPutBlock(HttpURLConnection connection) { return connection.getRequestMethod().equals("PUT") + && connection.getURL().getQuery() != null && connection.getURL().getQuery().contains("blockid"); }
