Updated Branches: refs/heads/master 73464c387 -> 06b8cd7ac
JCLOUDS-305. Disable S3 virtual host buckets for generic S3 Not all S3-compatible providers support virtual host buckets and thus we should disable this feature by default. Continue to enable virtual host buckets for AWS-S3 which supports this although this feature suffers from DNS settling issues. Ran ran integration tests against AWS-S3 and Scality using its S3 API. Project: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/commit/06b8cd7a Tree: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/tree/06b8cd7a Diff: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/diff/06b8cd7a Branch: refs/heads/master Commit: 06b8cd7ac5420b0e3668bca6116ad5a3a29bbca8 Parents: 73464c3 Author: Andrew Gaul <[email protected]> Authored: Mon Aug 26 23:26:15 2013 -0700 Committer: Adrian Cole <[email protected]> Committed: Fri Sep 27 16:22:42 2013 -0700 ---------------------------------------------------------------------- .../main/java/org/jclouds/s3/S3ApiMetadata.java | 2 +- .../java/org/jclouds/s3/S3ClientExpectTest.java | 5 +-- .../java/org/jclouds/s3/S3ClientMockTest.java | 1 - .../s3/blobstore/S3BlobSignerExpectTest.java | 18 +++------ .../filters/RequestAuthorizeSignatureTest.java | 14 ++++--- ...rizeSignatureWithSessionCredentialsTest.java | 6 +-- .../S3RedirectionRetryHandlerExpectTest.java | 12 +++--- .../rest/internal/BaseRestApiExpectTest.java | 4 +- .../org/jclouds/aws/s3/AWSS3ApiMetadata.java | 2 + .../s3/blobstore/AWSS3BlobSignerExpectTest.java | 39 ++++++++++++++++++++ 10 files changed, 70 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java index 92ec427..281d97a 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java @@ -85,7 +85,7 @@ public class S3ApiMetadata extends BaseRestApiMetadata { properties.setProperty(PROPERTY_AUTH_TAG, "AWS"); properties.setProperty(PROPERTY_HEADER_TAG, S3Headers.DEFAULT_AMAZON_HEADERTAG); properties.setProperty(PROPERTY_S3_SERVICE_PATH, "/"); - properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true"); + properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false"); properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true"); properties.setProperty(PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX, "/"); properties.setProperty(PROPERTY_USER_METADATA_PREFIX, String.format("x-${%s}-meta-", PROPERTY_HEADER_TAG)); http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java b/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java index e932cd9..9203394 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java @@ -36,11 +36,10 @@ public class S3ClientExpectTest extends BaseS3ClientExpectTest { public void testBucketExistsReturnsTrueOn200AndFalseOn404() { HttpRequest bucketFooExists = HttpRequest.builder().method("GET").endpoint( - URI.create("https://foo.s3.amazonaws.com/?max-keys=0";)).headers( + URI.create("https://s3.amazonaws.com/foo?max-keys=0";)).headers( ImmutableMultimap.<String, String> builder() - .put("Host", "foo.s3.amazonaws.com") .put("Date", CONSTANT_DATE) - .put("Authorization", "AWS identity:ZWVz2v/jGB+ZMmijoyfH9mFMPo0=") + .put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=") .build()).build(); S3Client clientWhenBucketExists = requestSendsResponse(bucketFooExists, HttpResponse.builder().statusCode(200).build()); http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java b/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java index 748a6b4..ccdf326 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java @@ -54,7 +54,6 @@ public class S3ClientMockTest { static S3Client getS3Client(URL server) { Properties overrides = new Properties(); - overrides.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false"); // prevent expect-100 bug http://code.google.com/p/mockwebserver/issues/detail?id=6 overrides.setProperty(PROPERTY_SO_TIMEOUT, "0"); overrides.setProperty(PROPERTY_MAX_RETRIES, "1"); http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java index 18f046c..522381c 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java @@ -44,8 +44,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest getBlob() { return HttpRequest.builder().method("GET") - .endpoint("https://container.s3.amazonaws.com/name";) - .addHeader("Host", "container.s3.amazonaws.com") + .endpoint("https://s3.amazonaws.com/container/name";) .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); } @@ -60,8 +59,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest getBlobWithTime() { return HttpRequest.builder().method("GET") - .endpoint("https://container.s3.amazonaws.com/name";) - .addHeader("Host", "container.s3.amazonaws.com") + .endpoint("https://s3.amazonaws.com/container/name";) .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); } @@ -69,9 +67,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest getBlobWithOptions() { return HttpRequest.builder().method("GET") - .endpoint("https://container.s3.amazonaws.com/name";) + .endpoint("https://s3.amazonaws.com/container/name";) .addHeader("Range", "bytes=0-1") - .addHeader("Host", "container.s3.amazonaws.com") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); } @@ -79,9 +76,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest putBlob() { return HttpRequest.builder().method("PUT") - .endpoint("https://container.s3.amazonaws.com/name";) + .endpoint("https://s3.amazonaws.com/container/name";) .addHeader("Expect", "100-continue") - .addHeader("Host", "container.s3.amazonaws.com") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=").build(); } @@ -96,9 +92,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest putBlobWithTime() { return HttpRequest.builder().method("PUT") - .endpoint("https://container.s3.amazonaws.com/name";) + .endpoint("https://s3.amazonaws.com/container/name";) .addHeader("Expect", "100-continue") - .addHeader("Host", "container.s3.amazonaws.com") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); } @@ -106,8 +101,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest removeBlob() { return HttpRequest.builder().method("DELETE") - .endpoint("https://container.s3.amazonaws.com/name";) - .addHeader("Host", "container.s3.amazonaws.com") + .endpoint("https://s3.amazonaws.com/container/name";) .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=").build(); } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java index 1298003..a680103 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java @@ -20,6 +20,7 @@ import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL; import static org.jclouds.reflect.Reflection2.method; import static org.testng.Assert.assertEquals; +import java.net.URI; import java.util.Properties; import javax.ws.rs.core.HttpHeaders; @@ -48,6 +49,7 @@ import com.google.common.collect.TreeMultimap; // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "RequestAuthorizeSignatureTest") public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3AsyncClient> { + String bucketName = "bucket"; @DataProvider(parallel = true) public Object[][] dataProvider() throws NoSuchMethodException { @@ -89,7 +91,7 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3Async ImmutableList.<Object> of("bucket")); StringBuilder builder = new StringBuilder(); filter.appendBucketName(request, builder); - assertEquals(builder.toString(), "/bucket"); + assertEquals(builder.toString(), ""); } @Test @@ -97,7 +99,7 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3Async HttpRequest request = putBucketAcl(); StringBuilder builder = new StringBuilder(); filter.appendUriPath(request, builder); - assertEquals(builder.toString(), "/?acl"); + assertEquals(builder.toString(), "/" + bucketName + "?acl"); } private GeneratedHttpRequest putBucketAcl() throws NoSuchMethodException { @@ -140,11 +142,13 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3Async } @Test - void testAppendBucketNameURIHost() throws SecurityException, NoSuchMethodException { + void testAppendBucketNameInURIPath() throws SecurityException, NoSuchMethodException { GeneratedHttpRequest request = processor.createRequest( method(S3AsyncClient.class, "getBucketLocation", String.class), - ImmutableList.<Object> of("bucket")); - assertEquals(request.getEndpoint().getHost(), "bucket.s3.amazonaws.com"); + ImmutableList.<Object> of(bucketName)); + URI uri = request.getEndpoint(); + assertEquals(uri.getHost(), "s3.amazonaws.com"); + assertEquals(uri.getPath(), "/" + bucketName); } @Override http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java index 267eec3..d80394f 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java @@ -79,14 +79,14 @@ public class RequestAuthorizeSignatureWithSessionCredentialsTest { HttpRequest bucketFooExists = GeneratedHttpRequest.builder().method("GET") .invocation(invocation) - .endpoint("https://foo.s3.amazonaws.com/?max-keys=0";) - .addHeader("Host", "foo.s3.amazonaws.com").build(); + .endpoint("https://s3.amazonaws.com/foo?max-keys=0";) + .build(); @Test void testAddsSecurityToken() { HttpRequest filtered = filter(temporaryCredentials).filter(bucketFooExists); assertEquals(filtered.getFirstHeaderOrNull("Authorization"), - "AWS AKIAIOSFODNN7EXAMPLE:0fUhWTaRBcIvIAndg2C+5eLfE24="); + "AWS AKIAIOSFODNN7EXAMPLE:sbAAy3Gh/sD1zwO7Ut75YJFCs4U="); assertEquals(filtered.getFirstHeaderOrNull("x-amz-security-token"), temporaryCredentials.getSessionToken()); } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java b/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java index 78a6f12..ec2d6cc 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java @@ -36,17 +36,17 @@ public class S3RedirectionRetryHandlerExpectTest extends BaseS3ClientExpectTest public void testRedirectOnHeadBucketChangesRequestToGetBucket() { HttpRequest bucketFooExists = HttpRequest.builder().method("HEAD").endpoint( - URI.create("https://foo.s3.amazonaws.com/?max-keys=0";)).headers( - ImmutableMultimap.<String, String> builder().put("Host", "foo.s3.amazonaws.com").put("Date", - CONSTANT_DATE).put("Authorization", "AWS identity:86P4BBb7xT+gBqq7jxM8Tc28ktY=").build()) + URI.create("https://s3.amazonaws.com/foo?max-keys=0";)).headers( + ImmutableMultimap.<String, String> builder().put("Date", + CONSTANT_DATE).put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=").build()) .build(); HttpResponse redirectResponse = HttpResponse.builder().statusCode(301).build(); HttpRequest bucketFooExistsNowUsesGET = HttpRequest.builder().method("GET").endpoint( - URI.create("https://foo.s3.amazonaws.com/?max-keys=0";)).headers( - ImmutableMultimap.<String, String> builder().put("Host", "foo.s3.amazonaws.com").put("Date", - CONSTANT_DATE).put("Authorization", "AWS identity:ZWVz2v/jGB+ZMmijoyfH9mFMPo0=").build()) + URI.create("https://s3.amazonaws.com/foo?max-keys=0";)).headers( + ImmutableMultimap.<String, String> builder().put("Date", + CONSTANT_DATE).put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=").build()) .build(); HttpResponse success = HttpResponse.builder().statusCode(200).build(); http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java b/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java index e4a24ad..7618622 100644 --- a/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java +++ b/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java @@ -103,8 +103,8 @@ import com.google.inject.TypeLiteral; * <pre> * * HttpRequest bucketFooExists = HttpRequest.builder().method("HEAD").endpoint( - * URI.create("https://foo.s3.amazonaws.com/?max-keys=0")).headers( - * ImmutableMultimap.<String, String> builder().put("Host", "foo.s3.amazonaws.com").put("Date", CONSTANT_DATE) + * URI.create("https://s3.amazonaws.com/foo?max-keys=0")).headers( + * ImmutableMultimap.<String, String> builder().put("Date", CONSTANT_DATE) * .put("Authorization", "AWS identity:86P4BBb7xT+gBqq7jxM8Tc28ktY=").build()).build(); * * S3Client clientWhenBucketExists = requestSendsResponse(bucketFooExists, HttpResponse.builder().statusCode(200).build()); http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java ---------------------------------------------------------------------- diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java index dc616f8..d896a9f 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java @@ -17,6 +17,7 @@ package org.jclouds.aws.s3; import static org.jclouds.reflect.Reflection2.typeToken; +import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS; import java.util.Properties; @@ -61,6 +62,7 @@ public class AWSS3ApiMetadata extends S3ApiMetadata { public static Properties defaultProperties() { Properties properties = S3ApiMetadata.defaultProperties(); + properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true"); return properties; } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/06b8cd7a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java ---------------------------------------------------------------------- diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java index 0b38b0b..aa36785 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java @@ -36,12 +36,22 @@ import com.google.inject.Module; @Test(groups = "unit", testName = "AWSS3BlobSignerExpectTest") public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest { private static final String DATE = "Thu, 05 Jun 2008 16:38:19 GMT"; + private static final String HOST = "container.s3.amazonaws.com"; public AWSS3BlobSignerExpectTest() { provider = "aws-s3"; } @Override + protected HttpRequest getBlob() { + return HttpRequest.builder().method("GET") + .endpoint("https://container.s3.amazonaws.com/name";) + .addHeader("Host", HOST) + .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") + .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); + } + + @Override protected HttpRequest getBlobWithTime() { return HttpRequest.builder().method("GET") .endpoint("https://container.s3.amazonaws.com/name"; + @@ -50,6 +60,16 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest { .addHeader("Date", DATE).build(); } + @Override + protected HttpRequest getBlobWithOptions() { + return HttpRequest.builder().method("GET") + .endpoint("https://container.s3.amazonaws.com/name";) + .addHeader("Host", HOST) + .addHeader("Range", "bytes=0-1") + .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") + .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); + } + @Test public void testSignGetBlobWithTime() { BlobStore getBlobWithTime = requestsSendResponses(init()); @@ -59,6 +79,16 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest { } @Override + protected HttpRequest putBlob() { + return HttpRequest.builder().method("PUT") + .endpoint("https://container.s3.amazonaws.com/name";) + .addHeader("Host", HOST) + .addHeader("Expect", "100-continue") + .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") + .addHeader("Authorization", "AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=").build(); + } + + @Override protected HttpRequest putBlobWithTime() { return HttpRequest.builder().method("PUT") .endpoint("https://container.s3.amazonaws.com/name"; + @@ -68,6 +98,15 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest { .addHeader("Date", DATE).build(); } + @Override + protected HttpRequest removeBlob() { + return HttpRequest.builder().method("DELETE") + .endpoint("https://container.s3.amazonaws.com/name";) + .addHeader("Host", HOST) + .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") + .addHeader("Authorization", "AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=").build(); + } + @Test public void testSignPutBlobWithTime() throws Exception { BlobStore signPutBloblWithTime = requestsSendResponses(init());
