HADOOP-11959. WASB should configure client side socket timeout in storage 
client blob request options. Contributed by Ivan Mitic.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44f4df57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44f4df57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44f4df57

Branch: refs/heads/YARN-2928
Commit: 44f4df57bfe7a078894cc4c05b07c0e3a292412f
Parents: 4b4aeba
Author: cnauroth <cnaur...@apache.org>
Authored: Thu May 28 12:31:06 2015 -0700
Committer: Zhijie Shen <zjs...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt        |  3 +++
 hadoop-project/pom.xml                                 |  2 +-
 .../hadoop/fs/azure/AzureNativeFileSystemStore.java    | 13 ++-----------
 .../org/apache/hadoop/fs/azure/StorageInterface.java   |  6 +++---
 .../apache/hadoop/fs/azure/StorageInterfaceImpl.java   |  4 ++--
 .../apache/hadoop/fs/azure/MockStorageInterface.java   |  4 ++--
 .../fs/azure/TestAzureFileSystemErrorConditions.java   |  1 +
 .../apache/hadoop/fs/azure/TestBlobDataValidation.java |  1 +
 8 files changed, 15 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6aa224c..aba9087 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -791,6 +791,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-11930. test-patch in offline mode should tell maven to be in
     offline mode (Sean Busbey via aw)
 
+    HADOOP-11959. WASB should configure client side socket timeout in storage
+    client blob request options. (Ivan Mitic via cnauroth)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 94e1f5e..a5d4ed8 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -942,7 +942,7 @@
       <dependency>
         <groupId>com.microsoft.azure</groupId>
         <artifactId>azure-storage</artifactId>
-        <version>2.0.0</version>
+        <version>2.2.0</version>
      </dependency>
 
      <dependency>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 3267d8b..69bda06 100644
--- 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -2434,15 +2434,6 @@ public class AzureNativeFileSystemStore implements 
NativeFileSystemStore {
       //
       CloudBlobWrapper dstBlob = getBlobReference(dstKey);
 
-      // TODO: Remove at the time when we move to Azure Java SDK 1.2+.
-      // This is the workaround provided by Azure Java SDK team to
-      // mitigate the issue with un-encoded x-ms-copy-source HTTP
-      // request header. Azure sdk version before 1.2+ does not encode this
-      // header what causes all URIs that have special (category "other")
-      // characters in the URI not to work with startCopyFromBlob when
-      // specified as source (requests fail with HTTP 403).
-      URI srcUri = new URI(srcBlob.getUri().toASCIIString());
-
       // Rename the source blob to the destination blob by copying it to
       // the destination blob then deleting it.
       //
@@ -2451,7 +2442,7 @@ public class AzureNativeFileSystemStore implements 
NativeFileSystemStore {
       // a more intensive exponential retry policy when the cluster is getting 
       // throttled.
       try {
-        dstBlob.startCopyFromBlob(srcUri, null, getInstrumentedContext());
+        dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext());
       } catch (StorageException se) {
         if (se.getErrorCode().equals(
                  StorageErrorCode.SERVER_BUSY.toString())) {
@@ -2475,7 +2466,7 @@ public class AzureNativeFileSystemStore implements 
NativeFileSystemStore {
           options.setRetryPolicyFactory(new RetryExponentialRetry(
             copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, 
                        copyBlobMaxRetries));
-          dstBlob.startCopyFromBlob(srcUri, options, getInstrumentedContext());
+          dstBlob.startCopyFromBlob(srcBlob, options, 
getInstrumentedContext());
         } else {
           throw se;
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
----------------------------------------------------------------------
diff --git 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
index e89151d..ce5f749 100644
--- 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
+++ 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
@@ -381,8 +381,8 @@ abstract class StorageInterface {
      * Copies an existing blob's contents, properties, and metadata to this 
instance of the <code>CloudBlob</code>
      * class, using the specified operation context.
      *
-     * @param source
-     *            A <code>java.net.URI</code> The URI of a source blob.
+     * @param sourceBlob
+     *            A <code>CloudBlob</code> object that represents the source 
blob to copy.
      * @param options
      *            A {@link BlobRequestOptions} object that specifies any 
additional options for the request. Specifying
      *            <code>null</code> will use the default request options from 
the associated service client (
@@ -397,7 +397,7 @@ abstract class StorageInterface {
      * @throws URISyntaxException
      *
      */
-    public abstract void startCopyFromBlob(URI source,
+    public abstract void startCopyFromBlob(CloudBlobWrapper sourceBlob,
         BlobRequestOptions options, OperationContext opContext)
         throws StorageException, URISyntaxException;
     

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
index 90d4d88..382ff66 100644
--- 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
+++ 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
@@ -393,10 +393,10 @@ class StorageInterfaceImpl extends StorageInterface {
     }
 
     @Override
-    public void startCopyFromBlob(URI source, BlobRequestOptions options,
+    public void startCopyFromBlob(CloudBlobWrapper sourceBlob, 
BlobRequestOptions options,
         OperationContext opContext)
             throws StorageException, URISyntaxException {
-      getBlob().startCopyFromBlob(source,
+      getBlob().startCopyFromBlob(((CloudBlobWrapperImpl)sourceBlob).blob,
           null, null, options, opContext);
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
----------------------------------------------------------------------
diff --git 
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
 
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
index cde0e38..9f84f4b 100644
--- 
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
+++ 
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
@@ -429,9 +429,9 @@ public class MockStorageInterface extends StorageInterface {
     }
 
     @Override
-    public void startCopyFromBlob(URI source, BlobRequestOptions options,
+    public void startCopyFromBlob(CloudBlobWrapper sourceBlob, 
BlobRequestOptions options,
         OperationContext opContext) throws StorageException, 
URISyntaxException {
-      backingStore.copy(convertUriToDecodedString(source), 
convertUriToDecodedString(uri));
+      backingStore.copy(convertUriToDecodedString(sourceBlob.getUri()), 
convertUriToDecodedString(uri));
       //TODO: set the backingStore.properties.CopyState and
       //      update azureNativeFileSystemStore.waitForCopyToComplete
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
----------------------------------------------------------------------
diff --git 
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
 
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
index ace57dc..810bcf7 100644
--- 
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
+++ 
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
@@ -205,6 +205,7 @@ public class TestAzureFileSystemErrorConditions {
         @Override
         public boolean isTargetConnection(HttpURLConnection connection) {
           return connection.getRequestMethod().equals("PUT")
+              && connection.getURL().getQuery() != null
               && connection.getURL().getQuery().contains("blocklist");
         }
       });

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
----------------------------------------------------------------------
diff --git 
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
 
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
index 9237ade..c40b7b5 100644
--- 
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
+++ 
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
@@ -191,6 +191,7 @@ public class TestBlobDataValidation {
 
     private static boolean isPutBlock(HttpURLConnection connection) {
       return connection.getRequestMethod().equals("PUT")
+          && connection.getURL().getQuery() != null
           && connection.getURL().getQuery().contains("blockid");
     }
 

Reply via email to