http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java deleted file mode 100644 index 8aad9e9..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java +++ /dev/null @@ -1,569 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.io.retry.RetryUtils; -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.HttpEntity; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.ProtocolVersion; -import org.apache.http.ParseException; -import org.apache.http.HeaderElement; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; - -import java.io.ByteArrayInputStream; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.charset.StandardCharsets; - -import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.atLeast; -import static org.mockito.Mockito.times; - -/** - * Test class to hold all WasbRemoteCallHelper tests - */ -public class TestWasbRemoteCallHelper - extends AbstractWasbTestBase { - public static final String EMPTY_STRING = ""; - private static final int INVALID_HTTP_STATUS_CODE_999 = 999; - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - Configuration conf = new Configuration(); - conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true"); - conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URLS, "http://localhost1/,http://localhost2/,http://localhost:8080"); - return AzureBlobStorageTestAccount.create(conf); - } - - @Before - public void beforeMethod() { - boolean useSecureMode = fs.getConf().getBoolean(KEY_USE_SECURE_MODE, false); - boolean useAuthorization = fs.getConf().getBoolean(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, false); - Assume.assumeTrue("Test valid when both SecureMode and Authorization are enabled .. skipping", - useSecureMode && useAuthorization); - - Assume.assumeTrue( - useSecureMode && useAuthorization - ); - } - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - /** - * Test invalid status-code - * @throws Throwable - */ - @Test // (expected = WasbAuthorizationException.class) - public void testInvalidStatusCode() throws Throwable { - - setupExpectations(); - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse); - Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(INVALID_HTTP_STATUS_CODE_999)); - // finished setting up mocks - - performop(mockHttpClient); - } - - /** - * Test invalid Content-Type - * @throws Throwable - */ - @Test // (expected = WasbAuthorizationException.class) - public void testInvalidContentType() throws Throwable { - - setupExpectations(); - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse); - Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(HttpStatus.SC_OK)); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "text/plain")); - // finished setting up mocks - - performop(mockHttpClient); - } - - /** - * Test missing Content-Length - * @throws Throwable - */ - @Test // (expected = WasbAuthorizationException.class) - public void testMissingContentLength() throws Throwable { - - setupExpectations(); - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse); - Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(HttpStatus.SC_OK)); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - // finished setting up mocks - - performop(mockHttpClient); - } - - /** - * Test Content-Length exceeds max - * @throws Throwable - */ - @Test // (expected = WasbAuthorizationException.class) - public void testContentLengthExceedsMax() throws Throwable { - - setupExpectations(); - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse); - Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(HttpStatus.SC_OK)); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "2048")); - // finished setting up mocks - - performop(mockHttpClient); - } - - /** - * Test invalid Content-Length value - * @throws Throwable - */ - @Test // (expected = WasbAuthorizationException.class) - public void testInvalidContentLengthValue() throws Throwable { - - setupExpectations(); - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse); - Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(HttpStatus.SC_OK)); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "20abc48")); - // finished setting up mocks - - performop(mockHttpClient); - } - - /** - * Test valid JSON response - * @throws Throwable - */ - @Test - public void testValidJSONResponse() throws Throwable { - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - - HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); - HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class); - - Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse); - Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(HttpStatus.SC_OK)); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity); - Mockito.when(mockHttpEntity.getContent()) - .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8))) - .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8))) - .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8))); - // finished setting up mocks - - performop(mockHttpClient); - } - - /** - * Test malformed JSON response - * @throws Throwable - */ - @Test // (expected = WasbAuthorizationException.class) - public void testMalFormedJSONResponse() throws Throwable { - - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("com.fasterxml.jackson.core.JsonParseException: Unexpected end-of-input in FIELD_NAME"); - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - - HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); - HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class); - - Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse); - Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(HttpStatus.SC_OK)); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity); - Mockito.when(mockHttpEntity.getContent()) - .thenReturn(new ByteArrayInputStream(malformedJsonResponse().getBytes(StandardCharsets.UTF_8))); - // finished setting up mocks - - performop(mockHttpClient); - } - - /** - * Test valid JSON response failure response code - * @throws Throwable - */ - @Test // (expected = WasbAuthorizationException.class) - public void testFailureCodeJSONResponse() throws Throwable { - - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("Remote authorization service encountered an error Unauthorized"); - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - - HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); - HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class); - - Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse); - Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(HttpStatus.SC_OK)); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity); - Mockito.when(mockHttpEntity.getContent()) - .thenReturn(new ByteArrayInputStream(failureCodeJsonResponse().getBytes(StandardCharsets.UTF_8))); - // finished setting up mocks - - performop(mockHttpClient); - } - - @Test - public void testWhenOneInstanceIsDown() throws Throwable { - - boolean isAuthorizationCachingEnabled = fs.getConf().getBoolean(CachingAuthorizer.KEY_AUTH_SERVICE_CACHING_ENABLE, false); - - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class); - - HttpResponse mockHttpResponseService1 = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpResponseService1.getStatusLine()) - .thenReturn(newStatusLine(HttpStatus.SC_INTERNAL_SERVER_ERROR)); - Mockito.when(mockHttpResponseService1.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponseService1.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponseService1.getEntity()) - .thenReturn(mockHttpEntity); - - HttpResponse mockHttpResponseService2 = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpResponseService2.getStatusLine()) - .thenReturn(newStatusLine(HttpStatus.SC_OK)); - Mockito.when(mockHttpResponseService2.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponseService2.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponseService2.getEntity()) - .thenReturn(mockHttpEntity); - - HttpResponse mockHttpResponseServiceLocal = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpResponseServiceLocal.getStatusLine()) - .thenReturn(newStatusLine(HttpStatus.SC_INTERNAL_SERVER_ERROR)); - Mockito.when(mockHttpResponseServiceLocal.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponseServiceLocal.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponseServiceLocal.getEntity()) - .thenReturn(mockHttpEntity); - - - - class HttpGetForService1 extends ArgumentMatcher<HttpGet>{ - @Override public boolean matches(Object o) { - return checkHttpGetMatchHost((HttpGet) o, "localhost1"); - } - } - class HttpGetForService2 extends ArgumentMatcher<HttpGet>{ - @Override public boolean matches(Object o) { - return checkHttpGetMatchHost((HttpGet) o, "localhost2"); - } - } - class HttpGetForServiceLocal extends ArgumentMatcher<HttpGet>{ - @Override public boolean matches(Object o) { - try { - return checkHttpGetMatchHost((HttpGet) o, InetAddress.getLocalHost().getCanonicalHostName()); - } catch (UnknownHostException e) { - return checkHttpGetMatchHost((HttpGet) o, "localhost"); - } - } - } - Mockito.when(mockHttpClient.execute(argThat(new HttpGetForService1()))) - .thenReturn(mockHttpResponseService1); - Mockito.when(mockHttpClient.execute(argThat(new HttpGetForService2()))) - .thenReturn(mockHttpResponseService2); - Mockito.when(mockHttpClient.execute(argThat(new HttpGetForServiceLocal()))) - .thenReturn(mockHttpResponseServiceLocal); - - //Need 2 times because performop() does 2 fs operations. - Mockito.when(mockHttpEntity.getContent()) - .thenReturn(new ByteArrayInputStream(validJsonResponse() - .getBytes(StandardCharsets.UTF_8))) - .thenReturn(new ByteArrayInputStream(validJsonResponse() - .getBytes(StandardCharsets.UTF_8))) - .thenReturn(new ByteArrayInputStream(validJsonResponse() - .getBytes(StandardCharsets.UTF_8))); - // finished setting up mocks - - performop(mockHttpClient); - - int expectedNumberOfInvocations = isAuthorizationCachingEnabled ? 1 : 2; - Mockito.verify(mockHttpClient, times(expectedNumberOfInvocations)).execute(Mockito.argThat(new HttpGetForServiceLocal())); - Mockito.verify(mockHttpClient, times(expectedNumberOfInvocations)).execute(Mockito.argThat(new HttpGetForService2())); - } - - @Test - public void testWhenServiceInstancesAreDown() throws Throwable { - //expectedEx.expect(WasbAuthorizationException.class); - // set up mocks - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); - HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class); - - HttpResponse mockHttpResponseService1 = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpResponseService1.getStatusLine()) - .thenReturn(newStatusLine(HttpStatus.SC_INTERNAL_SERVER_ERROR)); - Mockito.when(mockHttpResponseService1.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponseService1.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponseService1.getEntity()) - .thenReturn(mockHttpEntity); - - HttpResponse mockHttpResponseService2 = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpResponseService2.getStatusLine()) - .thenReturn(newStatusLine( - HttpStatus.SC_INTERNAL_SERVER_ERROR)); - Mockito.when(mockHttpResponseService2.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponseService2.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponseService2.getEntity()) - .thenReturn(mockHttpEntity); - - HttpResponse mockHttpResponseService3 = Mockito.mock(HttpResponse.class); - Mockito.when(mockHttpResponseService3.getStatusLine()) - .thenReturn(newStatusLine( - HttpStatus.SC_INTERNAL_SERVER_ERROR)); - Mockito.when(mockHttpResponseService3.getFirstHeader("Content-Type")) - .thenReturn(newHeader("Content-Type", "application/json")); - Mockito.when(mockHttpResponseService3.getFirstHeader("Content-Length")) - .thenReturn(newHeader("Content-Length", "1024")); - Mockito.when(mockHttpResponseService3.getEntity()) - .thenReturn(mockHttpEntity); - - class HttpGetForService1 extends ArgumentMatcher<HttpGet>{ - @Override public boolean matches(Object o) { - return checkHttpGetMatchHost((HttpGet) o, "localhost1"); - } - } - class HttpGetForService2 extends ArgumentMatcher<HttpGet>{ - @Override public boolean matches(Object o) { - return checkHttpGetMatchHost((HttpGet) o, "localhost2"); - } - } - class HttpGetForService3 extends ArgumentMatcher<HttpGet> { - @Override public boolean matches(Object o){ - try { - return checkHttpGetMatchHost((HttpGet) o, InetAddress.getLocalHost().getCanonicalHostName()); - } catch (UnknownHostException e) { - return checkHttpGetMatchHost((HttpGet) o, "localhost"); - } - } - } - Mockito.when(mockHttpClient.execute(argThat(new HttpGetForService1()))) - .thenReturn(mockHttpResponseService1); - Mockito.when(mockHttpClient.execute(argThat(new HttpGetForService2()))) - .thenReturn(mockHttpResponseService2); - Mockito.when(mockHttpClient.execute(argThat(new HttpGetForService3()))) - .thenReturn(mockHttpResponseService3); - - //Need 3 times because performop() does 3 fs operations. - Mockito.when(mockHttpEntity.getContent()) - .thenReturn(new ByteArrayInputStream( - validJsonResponse().getBytes(StandardCharsets.UTF_8))) - .thenReturn(new ByteArrayInputStream( - validJsonResponse().getBytes(StandardCharsets.UTF_8))) - .thenReturn(new ByteArrayInputStream( - validJsonResponse().getBytes(StandardCharsets.UTF_8))); - // finished setting up mocks - try { - performop(mockHttpClient); - }catch (WasbAuthorizationException e){ - e.printStackTrace(); - Mockito.verify(mockHttpClient, atLeast(2)) - .execute(argThat(new HttpGetForService1())); - Mockito.verify(mockHttpClient, atLeast(2)) - .execute(argThat(new HttpGetForService2())); - Mockito.verify(mockHttpClient, atLeast(3)) - .execute(argThat(new HttpGetForService3())); - Mockito.verify(mockHttpClient, times(7)).execute(Mockito.<HttpGet>any()); - } - } - - private void setupExpectations() { - expectedEx.expect(WasbAuthorizationException.class); - - class MatchesPattern extends TypeSafeMatcher<String> { - private String pattern; - - MatchesPattern(String pattern) { - this.pattern = pattern; - } - - @Override protected boolean matchesSafely(String item) { - return item.matches(pattern); - } - - @Override public void describeTo(Description description) { - description.appendText("matches pattern ").appendValue(pattern); - } - - @Override protected void describeMismatchSafely(String item, - Description mismatchDescription) { - mismatchDescription.appendText("does not match"); - } - } - - expectedEx.expectMessage(new MatchesPattern( - "org\\.apache\\.hadoop\\.fs\\.azure\\.WasbRemoteCallException: " - + "Encountered error while making remote call to " - + "http:\\/\\/localhost1\\/,http:\\/\\/localhost2\\/,http:\\/\\/localhost:8080 retried 6 time\\(s\\)\\.")); - } - - private void performop(HttpClient mockHttpClient) throws Throwable { - - Path testPath = new Path("/", "test.dat"); - - RemoteWasbAuthorizerImpl authorizer = new RemoteWasbAuthorizerImpl(); - authorizer.init(fs.getConf()); - WasbRemoteCallHelper mockWasbRemoteCallHelper = new WasbRemoteCallHelper( - RetryUtils.getMultipleLinearRandomRetry(new Configuration(), - EMPTY_STRING, true, - EMPTY_STRING, "1000,3,10000,2")); - mockWasbRemoteCallHelper.updateHttpClient(mockHttpClient); - authorizer.updateWasbRemoteCallHelper(mockWasbRemoteCallHelper); - fs.updateWasbAuthorizer(authorizer); - - fs.create(testPath); - ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); - fs.delete(testPath, false); - } - - private String validJsonResponse() { - return "{" - + "\"responseCode\": 0," - + "\"authorizationResult\": true," - + "\"responseMessage\": \"Authorized\"" - + "}"; - } - - private String malformedJsonResponse() { - return "{" - + "\"responseCode\": 0," - + "\"authorizationResult\": true," - + "\"responseMessage\":"; - } - - private String failureCodeJsonResponse() { - return "{" - + "\"responseCode\": 1," - + "\"authorizationResult\": false," - + "\"responseMessage\": \"Unauthorized\"" - + "}"; - } - - private StatusLine newStatusLine(int statusCode) { - return new StatusLine() { - @Override - public ProtocolVersion getProtocolVersion() { - return new ProtocolVersion("HTTP", 1, 1); - } - - @Override - public int getStatusCode() { - return statusCode; - } - - @Override - public String getReasonPhrase() { - return "Reason Phrase"; - } - }; - } - - private Header newHeader(String name, String value) { - return new Header() { - @Override - public String getName() { - return name; - } - - @Override - public String getValue() { - return value; - } - - @Override - public HeaderElement[] getElements() throws ParseException { - return new HeaderElement[0]; - } - }; - } - - /** Check that a HttpGet request is with given remote host. */ - private static boolean checkHttpGetMatchHost(HttpGet g, String h) { - return g != null && g.getURI().getHost().equals(h); - } - -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java deleted file mode 100644 index 672ed9c..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java +++ /dev/null @@ -1,617 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeNotNull; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.util.Date; -import java.util.EnumSet; -import java.io.File; - -import org.apache.hadoop.security.ProviderUtils; -import org.apache.hadoop.security.alias.CredentialProvider; -import org.apache.hadoop.security.alias.CredentialProviderFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.AbstractFileSystem; -import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions; -import org.junit.After; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlockBlob; - -public class TestWasbUriAndConfiguration { - - private static final int FILE_SIZE = 4096; - private static final String PATH_DELIMITER = "/"; - - protected String accountName; - protected String accountKey; - protected static Configuration conf = null; - private boolean runningInSASMode = false; - @Rule - public final TemporaryFolder tempDir = new TemporaryFolder(); - - private AzureBlobStorageTestAccount testAccount; - - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - } - } - - @Before - public void setMode() { - runningInSASMode = AzureBlobStorageTestAccount.createTestConfiguration(). - getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, false); - } - - private boolean validateIOStreams(Path filePath) throws IOException { - // Capture the file system from the test account. - FileSystem fs = testAccount.getFileSystem(); - return validateIOStreams(fs, filePath); - } - - private boolean validateIOStreams(FileSystem fs, Path filePath) - throws IOException { - - // Create and write a file - OutputStream outputStream = fs.create(filePath); - outputStream.write(new byte[FILE_SIZE]); - outputStream.close(); - - // Return true if the the count is equivalent to the file size. - return (FILE_SIZE == readInputStream(fs, filePath)); - } - - private int readInputStream(Path filePath) throws IOException { - // Capture the file system from the test account. - FileSystem fs = testAccount.getFileSystem(); - return readInputStream(fs, filePath); - } - - private int readInputStream(FileSystem fs, Path filePath) throws IOException { - // Read the file - InputStream inputStream = fs.open(filePath); - int count = 0; - while (inputStream.read() >= 0) { - count++; - } - inputStream.close(); - - // Return true if the the count is equivalent to the file size. - return count; - } - - // Positive tests to exercise making a connection with to Azure account using - // account key. - @Test - public void testConnectUsingKey() throws Exception { - - testAccount = AzureBlobStorageTestAccount.create(); - assumeNotNull(testAccount); - - // Validate input and output on the connection. - assertTrue(validateIOStreams(new Path("/wasb_scheme"))); - } - - @Test - public void testConnectUsingSAS() throws Exception { - - Assume.assumeFalse(runningInSASMode); - // Create the test account with SAS credentials. - testAccount = AzureBlobStorageTestAccount.create("", - EnumSet.of(CreateOptions.UseSas, CreateOptions.CreateContainer)); - assumeNotNull(testAccount); - // Validate input and output on the connection. - // NOTE: As of 4/15/2013, Azure Storage has a deficiency that prevents the - // full scenario from working (CopyFromBlob doesn't work with SAS), so - // just do a minor check until that is corrected. - assertFalse(testAccount.getFileSystem().exists(new Path("/IDontExist"))); - //assertTrue(validateIOStreams(new Path("/sastest.txt"))); - } - - @Test - public void testConnectUsingSASReadonly() throws Exception { - - Assume.assumeFalse(runningInSASMode); - // Create the test account with SAS credentials. - testAccount = AzureBlobStorageTestAccount.create("", EnumSet.of( - CreateOptions.UseSas, CreateOptions.CreateContainer, - CreateOptions.Readonly)); - assumeNotNull(testAccount); - - // Create a blob in there - final String blobKey = "blobForReadonly"; - CloudBlobContainer container = testAccount.getRealContainer(); - CloudBlockBlob blob = container.getBlockBlobReference(blobKey); - ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[] { 1, - 2, 3 }); - blob.upload(inputStream, 3); - inputStream.close(); - - // Make sure we can read it from the file system - Path filePath = new Path("/" + blobKey); - FileSystem fs = testAccount.getFileSystem(); - assertTrue(fs.exists(filePath)); - byte[] obtained = new byte[3]; - DataInputStream obtainedInputStream = fs.open(filePath); - obtainedInputStream.readFully(obtained); - obtainedInputStream.close(); - assertEquals(3, obtained[2]); - } - - @Test - public void testConnectUsingAnonymous() throws Exception { - - // Create test account with anonymous credentials - testAccount = AzureBlobStorageTestAccount.createAnonymous("testWasb.txt", - FILE_SIZE); - assumeNotNull(testAccount); - - // Read the file from the public folder using anonymous credentials. - assertEquals(FILE_SIZE, readInputStream(new Path("/testWasb.txt"))); - } - - @Test - public void testConnectToEmulator() throws Exception { - testAccount = AzureBlobStorageTestAccount.createForEmulator(); - assumeNotNull(testAccount); - assertTrue(validateIOStreams(new Path("/testFile"))); - } - - /** - * Tests that we can connect to fully qualified accounts outside of - * blob.core.windows.net - */ - @Test - public void testConnectToFullyQualifiedAccountMock() throws Exception { - Configuration conf = new Configuration(); - AzureBlobStorageTestAccount.setMockAccountKey(conf, - "mockAccount.mock.authority.net"); - AzureNativeFileSystemStore store = new AzureNativeFileSystemStore(); - MockStorageInterface mockStorage = new MockStorageInterface(); - store.setAzureStorageInteractionLayer(mockStorage); - NativeAzureFileSystem fs = new NativeAzureFileSystem(store); - fs.initialize( - new URI("wasb://mockcontai...@mockaccount.mock.authority.net"), conf); - fs.createNewFile(new Path("/x")); - assertTrue(mockStorage.getBackingStore().exists( - "http://mockAccount.mock.authority.net/mockContainer/x")); - fs.close(); - } - - public void testConnectToRoot() throws Exception { - - // Set up blob names. - final String blobPrefix = String.format("wasbtests-%s-%tQ-blob", - System.getProperty("user.name"), new Date()); - final String inblobName = blobPrefix + "_In" + ".txt"; - final String outblobName = blobPrefix + "_Out" + ".txt"; - - // Create test account with default root access. - testAccount = AzureBlobStorageTestAccount.createRoot(inblobName, FILE_SIZE); - assumeNotNull(testAccount); - - // Read the file from the default container. - assertEquals(FILE_SIZE, readInputStream(new Path(PATH_DELIMITER - + inblobName))); - - try { - // Capture file system. - FileSystem fs = testAccount.getFileSystem(); - - // Create output path and open an output stream to the root folder. - Path outputPath = new Path(PATH_DELIMITER + outblobName); - OutputStream outputStream = fs.create(outputPath); - fail("Expected an AzureException when writing to root folder."); - outputStream.write(new byte[FILE_SIZE]); - outputStream.close(); - } catch (AzureException e) { - assertTrue(true); - } catch (Exception e) { - String errMsg = String.format( - "Expected AzureException but got %s instead.", e); - assertTrue(errMsg, false); - } - } - - // Positive tests to exercise throttling I/O path. Connections are made to an - // Azure account using account key. - // - public void testConnectWithThrottling() throws Exception { - - testAccount = AzureBlobStorageTestAccount.createThrottled(); - - // Validate input and output on the connection. - assertTrue(validateIOStreams(new Path("/wasb_scheme"))); - } - - /** - * Creates a file and writes a single byte with the given value in it. - */ - private static void writeSingleByte(FileSystem fs, Path testFile, int toWrite) - throws Exception { - OutputStream outputStream = fs.create(testFile); - outputStream.write(toWrite); - outputStream.close(); - } - - /** - * Reads the file given and makes sure that it's a single-byte file with the - * given value in it. - */ - private static void assertSingleByteValue(FileSystem fs, Path testFile, - int expectedValue) throws Exception { - InputStream inputStream = fs.open(testFile); - int byteRead = inputStream.read(); - assertTrue("File unexpectedly empty: " + testFile, byteRead >= 0); - assertTrue("File has more than a single byte: " + testFile, - inputStream.read() < 0); - inputStream.close(); - assertEquals("Unxpected content in: " + testFile, expectedValue, byteRead); - } - - @Test - public void testMultipleContainers() throws Exception { - AzureBlobStorageTestAccount firstAccount = AzureBlobStorageTestAccount - .create("first"), secondAccount = AzureBlobStorageTestAccount - .create("second"); - assumeNotNull(firstAccount); - assumeNotNull(secondAccount); - try { - FileSystem firstFs = firstAccount.getFileSystem(), - secondFs = secondAccount.getFileSystem(); - Path testFile = new Path("/testWasb"); - assertTrue(validateIOStreams(firstFs, testFile)); - assertTrue(validateIOStreams(secondFs, testFile)); - // Make sure that we're really dealing with two file systems here. - writeSingleByte(firstFs, testFile, 5); - writeSingleByte(secondFs, testFile, 7); - assertSingleByteValue(firstFs, testFile, 5); - assertSingleByteValue(secondFs, testFile, 7); - } finally { - firstAccount.cleanup(); - secondAccount.cleanup(); - } - } - - @Test - public void testDefaultKeyProvider() throws Exception { - Configuration conf = new Configuration(); - String account = "testacct"; - String key = "testkey"; - - conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, key); - - String result = AzureNativeFileSystemStore.getAccountKeyFromConfiguration( - account, conf); - assertEquals(key, result); - } - - @Test - public void testCredsFromCredentialProvider() throws Exception { - - Assume.assumeFalse(runningInSASMode); - String account = "testacct"; - String key = "testkey"; - // set up conf to have a cred provider - final Configuration conf = new Configuration(); - final File file = tempDir.newFile("test.jks"); - final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider( - file.toURI()); - conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, - jks.toString()); - - provisionAccountKey(conf, account, key); - - // also add to configuration as clear text that should be overridden - conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, - key + "cleartext"); - - String result = AzureNativeFileSystemStore.getAccountKeyFromConfiguration( - account, conf); - // result should contain the credential provider key not the config key - assertEquals("AccountKey incorrect.", key, result); - } - - void provisionAccountKey( - final Configuration conf, String account, String key) throws Exception { - // add our creds to the provider - final CredentialProvider provider = - CredentialProviderFactory.getProviders(conf).get(0); - provider.createCredentialEntry( - SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, key.toCharArray()); - provider.flush(); - } - - @Test - public void testValidKeyProvider() throws Exception { - Configuration conf = new Configuration(); - String account = "testacct"; - String key = "testkey"; - - conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, key); - conf.setClass("fs.azure.account.keyprovider." + account, - SimpleKeyProvider.class, KeyProvider.class); - String result = AzureNativeFileSystemStore.getAccountKeyFromConfiguration( - account, conf); - assertEquals(key, result); - } - - @Test - public void testInvalidKeyProviderNonexistantClass() throws Exception { - Configuration conf = new Configuration(); - String account = "testacct"; - - conf.set("fs.azure.account.keyprovider." + account, - "org.apache.Nonexistant.Class"); - try { - AzureNativeFileSystemStore.getAccountKeyFromConfiguration(account, conf); - Assert.fail("Nonexistant key provider class should have thrown a " - + "KeyProviderException"); - } catch (KeyProviderException e) { - } - } - - @Test - public void testInvalidKeyProviderWrongClass() throws Exception { - Configuration conf = new Configuration(); - String account = "testacct"; - - conf.set("fs.azure.account.keyprovider." + account, "java.lang.String"); - try { - AzureNativeFileSystemStore.getAccountKeyFromConfiguration(account, conf); - Assert.fail("Key provider class that doesn't implement KeyProvider " - + "should have thrown a KeyProviderException"); - } catch (KeyProviderException e) { - } - } - - /** - * Tests the cases when the URI is specified with no authority, i.e. - * wasb:///path/to/file. - */ - @Test - public void testNoUriAuthority() throws Exception { - // For any combination of default FS being asv(s)/wasb(s)://c@a/ and - // the actual URI being asv(s)/wasb(s):///, it should work. - - String[] wasbAliases = new String[] { "wasb", "wasbs" }; - for (String defaultScheme : wasbAliases) { - for (String wantedScheme : wasbAliases) { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI(defaultScheme, authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - // Add references to file system implementations for wasb and wasbs. - conf.addResource("azure-test.xml"); - URI wantedUri = new URI(wantedScheme + ":///random/path"); - NativeAzureFileSystem obtained = (NativeAzureFileSystem) FileSystem - .get(wantedUri, conf); - assertNotNull(obtained); - assertEquals(new URI(wantedScheme, authority, null, null, null), - obtained.getUri()); - // Make sure makeQualified works as expected - Path qualified = obtained.makeQualified(new Path(wantedUri)); - assertEquals(new URI(wantedScheme, authority, wantedUri.getPath(), - null, null), qualified.toUri()); - // Cleanup for the next iteration to not cache anything in FS - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - // If the default FS is not a WASB FS, then specifying a URI without - // authority for the Azure file system should throw. - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - conf.set(FS_DEFAULT_NAME_KEY, "file:///"); - try { - FileSystem.get(new URI("wasb:///random/path"), conf); - fail("Should've thrown."); - } catch (IllegalArgumentException e) { - } - } - - @Test - public void testWasbAsDefaultFileSystemHasNoPort() throws Exception { - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasb", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - conf.addResource("azure-test.xml"); - - FileSystem fs = FileSystem.get(conf); - assertTrue(fs instanceof NativeAzureFileSystem); - assertEquals(-1, fs.getUri().getPort()); - - AbstractFileSystem afs = FileContext.getFileContext(conf) - .getDefaultFileSystem(); - assertTrue(afs instanceof Wasb); - assertEquals(-1, afs.getUri().getPort()); - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - - /** - * Tests the cases when the scheme specified is 'wasbs'. - */ - @Test - public void testAbstractFileSystemImplementationForWasbsScheme() throws Exception { - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs"); - conf.addResource("azure-test.xml"); - - FileSystem fs = FileSystem.get(conf); - assertTrue(fs instanceof NativeAzureFileSystem); - assertEquals("wasbs", fs.getScheme()); - - AbstractFileSystem afs = FileContext.getFileContext(conf) - .getDefaultFileSystem(); - assertTrue(afs instanceof Wasbs); - assertEquals(-1, afs.getUri().getPort()); - assertEquals("wasbs", afs.getUri().getScheme()); - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - - @Test - public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception { - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - - FileSystem fs = FileSystem.get(conf); - assertTrue(fs instanceof NativeAzureFileSystem); - assertEquals("wasbs", fs.getScheme()); - - // should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified - try{ - FileContext.getFileContext(conf).getDefaultFileSystem(); - fail("Should've thrown."); - }catch(UnsupportedFileSystemException e){ - } - - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - - @Test - public void testCredentialProviderPathExclusions() throws Exception { - String providerPath = - "user:///,jceks://wasb/user/hrt_qa/sqoopdbpasswd.jceks," + - "jceks://h...@nn1.example.com/my/path/test.jceks"; - Configuration config = new Configuration(); - config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, - providerPath); - String newPath = "user:///,jceks://h...@nn1.example.com/my/path/test.jceks"; - - excludeAndTestExpectations(config, newPath); - } - - @Test - public void testExcludeAllProviderTypesFromConfig() throws Exception { - String providerPath = - "jceks://wasb/tmp/test.jceks," + - "jceks://wasb@/my/path/test.jceks"; - Configuration config = new Configuration(); - config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, - providerPath); - String newPath = null; - - excludeAndTestExpectations(config, newPath); - } - - void excludeAndTestExpectations(Configuration config, String newPath) - throws Exception { - Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders( - config, NativeAzureFileSystem.class); - String effectivePath = conf.get( - CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, null); - assertEquals(newPath, effectivePath); - } - - @Test - public void testUserAgentConfig() throws Exception { - // Set the user agent - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs"); - - conf.set(AzureNativeFileSystemStore.USER_AGENT_ID_KEY, "TestClient"); - - FileSystem fs = FileSystem.get(conf); - AbstractFileSystem afs = FileContext.getFileContext(conf).getDefaultFileSystem(); - - assertTrue(afs instanceof Wasbs); - assertEquals(-1, afs.getUri().getPort()); - assertEquals("wasbs", afs.getUri().getScheme()); - - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - - // Unset the user agent - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs"); - - conf.unset(AzureNativeFileSystemStore.USER_AGENT_ID_KEY); - - FileSystem fs = FileSystem.get(conf); - AbstractFileSystem afs = FileContext.getFileContext(conf).getDefaultFileSystem(); - assertTrue(afs instanceof Wasbs); - assertEquals(-1, afs.getUri().getPort()); - assertEquals("wasbs", afs.getUri().getScheme()); - - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java new file mode 100644 index 0000000..fd21bd2 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; + +/** + * Append test, skipping one of them. + */ + +public class ITestAzureNativeContractAppend extends AbstractContractAppendTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } + + @Override + public void testRenameFileBeingAppended() throws Throwable { + skip("Skipping as renaming an opened file is not supported"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java new file mode 100644 index 0000000..0ac046a --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java new file mode 100644 index 0000000..4c6dd48 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractDelete extends AbstractContractDeleteTest { + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java new file mode 100644 index 0000000..7769570 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azure.integration.AzureTestConstants; +import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; + +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.assumeScaleTestsEnabled; + +/** + * Contract test suite covering WASB integration with DistCp. + */ +public class ITestAzureNativeContractDistCp extends AbstractContractDistCpTest { + + @Override + protected int getTestTimeoutMillis() { + return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS; + } + + @Override + protected NativeAzureFileSystemContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } + + @Override + public void setup() throws Exception { + super.setup(); + assumeScaleTestsEnabled(getContract().getConf()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java new file mode 100644 index 0000000..9c09c0d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractGetFileStatus + extends AbstractContractGetFileStatusTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java new file mode 100644 index 0000000..71654b8 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractMkdir extends AbstractContractMkdirTest { + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java new file mode 100644 index 0000000..0b174e6 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java new file mode 100644 index 0000000..474b874 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java new file mode 100644 index 0000000..673d5f8 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractSeek extends AbstractContractSeekTest{ + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java index 28c13ea..a264aca 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java @@ -18,15 +18,21 @@ package org.apache.hadoop.fs.azure.contract; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.integration.AzureTestUtils; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; +/** + * Azure Contract. Test paths are created using any maven fork + * identifier, if defined. This guarantees paths unique to tests + * running in parallel. + */ public class NativeAzureFileSystemContract extends AbstractBondedFSContract { public static final String CONTRACT_XML = "wasb.xml"; - protected NativeAzureFileSystemContract(Configuration conf) { - super(conf); - //insert the base features + public NativeAzureFileSystemContract(Configuration conf) { + super(conf); //insert the base features addConfResource(CONTRACT_XML); } @@ -34,4 +40,9 @@ public class NativeAzureFileSystemContract extends AbstractBondedFSContract { public String getScheme() { return "wasb"; } -} \ No newline at end of file + + @Override + public Path getTestPath() { + return AzureTestUtils.createTestPath(super.getTestPath()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractAppend.java deleted file mode 100644 index 8a2341e..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractAppend.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractAppendTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.junit.Test; -import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; - -public class TestAzureNativeContractAppend extends AbstractContractAppendTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } - - @Override - public void testRenameFileBeingAppended() throws Throwable { - skip("Skipping as renaming an opened file is not supported"); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractCreate.java deleted file mode 100644 index 531552d..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractCreate.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractCreateTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractCreate extends AbstractContractCreateTest{ - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDelete.java deleted file mode 100644 index 5e5c13b..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDelete.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractDelete extends AbstractContractDeleteTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDistCp.java deleted file mode 100644 index a3750d4..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDistCp.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; - -/** - * Contract test suite covering WASB integration with DistCp. - */ -public class TestAzureNativeContractDistCp extends AbstractContractDistCpTest { - - @Override - protected NativeAzureFileSystemContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractGetFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractGetFileStatus.java deleted file mode 100644 index b0c59ee..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractGetFileStatus.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractGetFileStatus extends AbstractContractGetFileStatusTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractMkdir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractMkdir.java deleted file mode 100644 index 36df041..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractMkdir.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractMkdir extends AbstractContractMkdirTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractOpen.java deleted file mode 100644 index d5147ac..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractOpen.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractOpenTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractOpen extends AbstractContractOpenTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractRename.java deleted file mode 100644 index 4d8b2b5..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractRename.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractRenameTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractRename extends AbstractContractRenameTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractSeek.java deleted file mode 100644 index 30046dc..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractSeek.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractSeekTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractSeek extends AbstractContractSeekTest{ - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java new file mode 100644 index 0000000..062d073 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.integration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.azure.AbstractWasbTestBase; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; + +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.*; + +/** + * Scale tests are only executed if the scale profile + * is set; the setup method will check this and skip + * tests if not. + * + */ +public abstract class AbstractAzureScaleTest + extends AbstractWasbTestBase implements Sizes { + + protected static final Logger LOG = + LoggerFactory.getLogger(AbstractAzureScaleTest.class); + + @Override + protected int getTestTimeoutMillis() { + return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS; + } + + @Override + public void setUp() throws Exception { + super.setUp(); + LOG.debug("Scale test operation count = {}", getOperationCount()); + assumeScaleTestsEnabled(getConfiguration()); + } + + /** + * Create the test account. + * @return a test account + * @throws Exception on any failure to create the account. + */ + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + return AzureBlobStorageTestAccount.create(createConfiguration()); + } + + protected long getOperationCount() { + return getConfiguration().getLong(KEY_OPERATION_COUNT, + DEFAULT_OPERATION_COUNT); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org