HADOOP-11642. Upgrade azure sdk version from 0.6.0 to 2.0.0. Contributed by Shashank Khandelwal and Ivan Mitic.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd633373 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd633373 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd633373 Branch: refs/heads/HDFS-7285 Commit: fd63337314557e9c8078e0e30ce7e43a05698594 Parents: a2f91d9 Author: cnauroth <cnaur...@apache.org> Authored: Fri Mar 6 14:59:09 2015 -0800 Committer: Jing Zhao <ji...@apache.org> Committed: Mon Mar 9 13:11:27 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + hadoop-project/pom.xml | 6 +- hadoop-tools/hadoop-azure/pom.xml | 7 +- .../fs/azure/AzureNativeFileSystemStore.java | 37 ++++++----- .../hadoop/fs/azure/NativeAzureFileSystem.java | 10 +-- .../hadoop/fs/azure/PageBlobFormatHelpers.java | 2 +- .../hadoop/fs/azure/PageBlobInputStream.java | 8 +-- .../hadoop/fs/azure/PageBlobOutputStream.java | 8 +-- .../hadoop/fs/azure/SelfRenewingLease.java | 6 +- .../fs/azure/SelfThrottlingIntercept.java | 10 +-- .../hadoop/fs/azure/SendRequestIntercept.java | 16 +++-- .../hadoop/fs/azure/StorageInterface.java | 24 +++---- .../hadoop/fs/azure/StorageInterfaceImpl.java | 46 +++++++------ .../fs/azure/metrics/ErrorMetricUpdater.java | 8 +-- .../metrics/ResponseReceivedMetricUpdater.java | 10 +-- .../fs/azure/AzureBlobStorageTestAccount.java | 28 ++++---- .../hadoop/fs/azure/MockStorageInterface.java | 70 ++++++++++++++------ .../fs/azure/NativeAzureFileSystemBaseTest.java | 6 +- .../TestAzureFileSystemErrorConditions.java | 6 +- .../hadoop/fs/azure/TestBlobDataValidation.java | 20 +++--- .../hadoop/fs/azure/TestContainerChecks.java | 6 +- .../TestOutOfBandAzureBlobOperationsLive.java | 4 +- .../fs/azure/TestWasbUriAndConfiguration.java | 4 +- 23 files changed, 190 insertions(+), 155 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 628faa3..14cd75a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -653,6 +653,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11648. Set DomainSocketWatcher thread name explicitly. (Liang Xie via ozawa) + HADOOP-11642. Upgrade azure sdk version from 0.6.0 to 2.0.0. + (Shashank Khandelwal and Ivan Mitic via cnauroth) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 2c0f03a..a6127c7 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -902,9 +902,9 @@ </dependency> <dependency> - <groupId>com.microsoft.windowsazure.storage</groupId> - <artifactId>microsoft-windowsazure-storage-sdk</artifactId> - <version>0.6.0</version> + <groupId>com.microsoft.azure</groupId> + <artifactId>azure-storage</artifactId> + <version>2.0.0</version> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index d39dd76..e9b3af7 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -140,12 +140,13 @@ <artifactId>httpclient</artifactId> <scope>compile</scope> </dependency> - + <dependency> - <groupId>com.microsoft.windowsazure.storage</groupId> - <artifactId>microsoft-windowsazure-storage-sdk</artifactId> + <groupId>com.microsoft.azure</groupId> + <artifactId>azure-storage</artifactId> <scope>compile</scope> </dependency> + <dependency> <groupId>com.google.guava</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index c0c03b3..b664fe7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -65,23 +65,23 @@ import org.apache.hadoop.fs.permission.PermissionStatus; import org.mortbay.util.ajax.JSON; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryExponentialRetry; -import com.microsoft.windowsazure.storage.RetryNoRetry; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.windowsazure.storage.StorageCredentialsSharedAccessSignature; -import com.microsoft.windowsazure.storage.StorageErrorCode; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CopyStatus; -import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.core.Utility; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryExponentialRetry; +import com.microsoft.azure.storage.RetryNoRetry; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; +import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.azure.storage.StorageErrorCode; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CopyStatus; +import com.microsoft.azure.storage.blob.DeleteSnapshotsOption; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.core.Utility; /** * Core implementation of Windows Azure Filesystem for Hadoop. @@ -2543,7 +2543,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { try { checkContainer(ContainerAccessType.ReadThenWrite); CloudBlobWrapper blob = getBlobReference(key); - blob.getProperties().setLastModified(lastModified); + //setLastModified function is not available in 2.0.0 version. blob.uploadProperties automatically updates last modified + //timestamp to current time blob.uploadProperties(getInstrumentedContext(), folderLease); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 0248b85..e39b37d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -74,11 +74,11 @@ import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.core.*; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.core.*; /** * <p> http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java index ad11aac..9a316a5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java @@ -20,7 +20,7 @@ package org.apache.hadoop.fs.azure; import java.nio.ByteBuffer; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.BlobRequestOptions; /** * Constants and helper methods for ASV's custom data format in page blobs. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java index 62b47ee..468ac65 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java @@ -33,10 +33,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.PageRange; /** * An input stream that reads file data from a page blob stored http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java index 4d1d5c8..2b8846c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java @@ -44,10 +44,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudPageBlob; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudPageBlob; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java index bda6006..06f32ce 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java @@ -22,9 +22,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlob; import java.util.concurrent.atomic.AtomicInteger; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java index d18a144..a9e3df9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java @@ -25,11 +25,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RequestResult; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.SendingRequestEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RequestResult; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageEvent; /* * Self throttling is implemented by hooking into send & response callbacks http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java index 18f173e..4d564d5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java @@ -25,12 +25,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.Constants.HeaderConstants; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.SendingRequestEvent; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageEvent; -import com.microsoft.windowsazure.storage.StorageException; +import com.microsoft.azure.storage.Constants.HeaderConstants; +import com.microsoft.azure.storage.core.StorageCredentialsHelper; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageEvent; +import com.microsoft.azure.storage.StorageException; /** * Manages the lifetime of binding on the operation contexts to intercept send @@ -146,7 +147,8 @@ public final class SendRequestIntercept extends StorageEvent<SendingRequestEvent try { // Sign the request. GET's have no payload so the content length is // zero. - getCredentials().signBlobAndQueueRequest(urlConnection, -1L, getOperationContext()); + StorageCredentialsHelper.signBlobAndQueueRequest(getCredentials(), + urlConnection, -1L, getOperationContext()); } catch (InvalidKeyException e) { // Log invalid key exception to track signing error before the send // fails. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java index 8d0229d..91928a2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java @@ -29,18 +29,18 @@ import java.util.HashMap; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryPolicyFactory; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CopyState; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryPolicyFactory; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CopyState; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.PageRange; /** * This is a very thin layer over the methods exposed by the Windows Azure http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java index e44823c..2120536 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java @@ -30,26 +30,26 @@ import java.util.Iterator; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryPolicyFactory; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.StorageUri; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CloudBlobClient; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlobDirectory; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; -import com.microsoft.windowsazure.storage.blob.CloudPageBlob; -import com.microsoft.windowsazure.storage.blob.CopyState; -import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryPolicyFactory; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageUri; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.CloudPageBlob; +import com.microsoft.azure.storage.blob.CopyState; +import com.microsoft.azure.storage.blob.DeleteSnapshotsOption; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.PageRange; /** * A real implementation of the Azure interaction layer that just redirects @@ -61,12 +61,14 @@ class StorageInterfaceImpl extends StorageInterface { @Override public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) { - serviceClient.setRetryPolicyFactory(retryPolicyFactory); + serviceClient.getDefaultRequestOptions().setRetryPolicyFactory( + retryPolicyFactory); } @Override public void setTimeoutInMs(int timeoutInMs) { - serviceClient.setTimeoutInMs(timeoutInMs); + serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs( + timeoutInMs); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java index d33e8c4..dc23354 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java @@ -24,10 +24,10 @@ import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; //500 import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RequestResult; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RequestResult; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.StorageEvent; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java index 676adb9..de503bf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java @@ -24,11 +24,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.Constants.HeaderConstants; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RequestResult; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.Constants.HeaderConstants; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RequestResult; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.StorageEvent; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java index b8ff912..635c024 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java @@ -40,20 +40,20 @@ import org.apache.hadoop.metrics2.MetricsSink; import org.apache.hadoop.metrics2.MetricsTag; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.windowsazure.storage.StorageCredentialsAnonymous; -import com.microsoft.windowsazure.storage.blob.BlobContainerPermissions; -import com.microsoft.windowsazure.storage.blob.BlobContainerPublicAccessType; -import com.microsoft.windowsazure.storage.blob.BlobOutputStream; -import com.microsoft.windowsazure.storage.blob.CloudBlobClient; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; -import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPermissions; -import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy; -import com.microsoft.windowsazure.storage.core.Base64; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; +import com.microsoft.azure.storage.StorageCredentialsAnonymous; +import com.microsoft.azure.storage.blob.BlobContainerPermissions; +import com.microsoft.azure.storage.blob.BlobContainerPublicAccessType; +import com.microsoft.azure.storage.blob.BlobOutputStream; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions; +import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy; +import com.microsoft.azure.storage.core.Base64; /** * Helper class to create WASB file systems backed by either a mock in-memory http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java index 047ea1b..c51c05b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java @@ -22,10 +22,12 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Calendar; +import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -35,21 +37,21 @@ import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.lang.NotImplementedException; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryPolicyFactory; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.StorageUri; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlobDirectory; -import com.microsoft.windowsazure.storage.blob.CopyState; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryPolicyFactory; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageUri; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import com.microsoft.azure.storage.blob.CopyState; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.PageRange; import javax.ws.rs.core.UriBuilder; import javax.ws.rs.core.UriBuilderException; @@ -357,18 +359,42 @@ public class MockStorageInterface extends StorageInterface { this.uri = uri; this.metadata = metadata; this.properties = new BlobProperties(); - this.properties.setLength(length); - this.properties.setLastModified( - Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime()); + + this.properties=updateLastModifed(this.properties); + this.properties=updateLength(this.properties,length); + } + + protected BlobProperties updateLastModifed(BlobProperties properties){ + try{ + Method setLastModified =properties.getClass(). + getDeclaredMethod("setLastModified", Date.class); + setLastModified.setAccessible(true); + setLastModified.invoke(this.properties, + Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime()); + }catch(Exception e){ + throw new RuntimeException(e); + } + return properties; } - + + protected BlobProperties updateLength(BlobProperties properties,int length) { + try{ + Method setLength =properties.getClass(). + getDeclaredMethod("setLength", long.class); + setLength.setAccessible(true); + setLength.invoke(this.properties, length); + }catch (Exception e){ + throw new RuntimeException(e); + } + return properties; + } + protected void refreshProperties(boolean getMetadata) { if (backingStore.exists(convertUriToDecodedString(uri))) { byte[] content = backingStore.getContent(convertUriToDecodedString(uri)); properties = new BlobProperties(); - properties.setLength(content.length); - properties.setLastModified( - Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime()); + this.properties=updateLastModifed(this.properties); + this.properties=updateLength(this.properties, content.length); if (getMetadata) { metadata = backingStore.getMetadata(convertUriToDecodedString(uri)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java index 01cf713..9ce6cc9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java @@ -58,9 +58,9 @@ import org.junit.Test; import org.apache.hadoop.fs.azure.AzureException; import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlob; /* * Tests the Native Azure file system (WASB) against an actual blob store if http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java index febb605..ace57dc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java @@ -37,9 +37,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext; import org.junit.Test; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.SendingRequestEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageEvent; public class TestAzureFileSystemErrorConditions { private static final int ALL_THREE_FILE_SIZE = 1024; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java index 25bd338..9237ade 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java @@ -39,16 +39,16 @@ import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationCo import org.junit.After; import org.junit.Test; -import com.microsoft.windowsazure.storage.Constants; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.StorageErrorCodeStrings; -import com.microsoft.windowsazure.storage.StorageEvent; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlockEntry; -import com.microsoft.windowsazure.storage.blob.BlockSearchMode; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; -import com.microsoft.windowsazure.storage.core.Base64; +import com.microsoft.azure.storage.Constants; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.StorageErrorCodeStrings; +import com.microsoft.azure.storage.StorageEvent; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlockEntry; +import com.microsoft.azure.storage.blob.BlockSearchMode; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.core.Base64; /** * Test that we do proper data integrity validation with MD5 checks as http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java index 727f540..56ec881 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java @@ -32,9 +32,9 @@ import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions; import org.junit.After; import org.junit.Test; -import com.microsoft.windowsazure.storage.blob.BlobOutputStream; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.BlobOutputStream; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; /** * Tests that WASB creates containers only if needed. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java index 9ac67e7..60b01c6 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java @@ -29,8 +29,8 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import com.microsoft.windowsazure.storage.blob.BlobOutputStream; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.BlobOutputStream; +import com.microsoft.azure.storage.blob.CloudBlockBlob; public class TestOutOfBandAzureBlobOperationsLive { private FileSystem fs; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd633373/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java index 0360e32..a4ca6fd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java @@ -42,8 +42,8 @@ import org.junit.After; import org.junit.Assert; import org.junit.Test; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; public class TestWasbUriAndConfiguration {
