Repository: hadoop Updated Branches: refs/heads/HDFS-7240 b3044db40 -> fedb22d9b
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fedb22d9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 218058c..da6d9c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -23,6 +23,10 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_STORAGE_RPC_ADDRESS_KE import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URISyntaxException; +import java.util.Random; + +import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +41,8 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolPB; import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.ozone.storage.StorageContainerManager; +import org.apache.hadoop.ozone.web.client.OzoneClient; +import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; @@ -53,6 +59,8 @@ public class MiniOzoneCluster extends MiniDFSCluster implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(MiniOzoneCluster.class); + private static final String USER_AUTH = "hdfs"; + private final OzoneConfiguration conf; private final StorageContainerManager scm; @@ -126,24 +134,26 @@ public class MiniOzoneCluster extends MiniDFSCluster implements Closeable { } /** - * Waits for the Ozone cluster to be ready for processing requests. + * Creates an {@link OzoneClient} connected to this cluster's REST service. + * Callers take ownership of the client and must close it when done. + * + * @return OzoneClient connected to this cluster's REST service + * @throws OzoneException if Ozone encounters an error creating the client */ - public void waitOzoneReady() { - long begin = Time.monotonicNow(); - while (scm.getDatanodeReport(DatanodeReportType.LIVE).length < - numDataNodes) { - if (Time.monotonicNow() - begin > 20000) { - throw new IllegalStateException( - "Timed out waiting for Ozone cluster to become ready."); - } - LOG.info("Waiting for Ozone cluster to become ready"); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException( - "Interrupted while waiting for Ozone cluster to become ready."); - } + public OzoneClient createOzoneClient() throws OzoneException { + Preconditions.checkState(!getDataNodes().isEmpty(), + "Cannot create OzoneClient if the cluster has no DataNodes."); + // An Ozone request may originate at any DataNode, so pick one at random. + int dnIndex = new Random().nextInt(getDataNodes().size()); + String uri = String.format("http://127.0.0.1:%d";, + getDataNodes().get(dnIndex).getInfoPort()); + LOG.info("Creating Ozone client to DataNode {} with URI {} and user {}", + dnIndex, uri, USER_AUTH); + try { + return new OzoneClient(uri, USER_AUTH); + } catch (URISyntaxException e) { + // We control the REST service URI, so it should never be invalid. + throw new IllegalStateException("Unexpected URISyntaxException", e); } } @@ -155,14 +165,39 @@ public class MiniOzoneCluster extends MiniDFSCluster implements Closeable { * @return RPC proxy for accessing container location information * @throws IOException if there is an I/O error */ - protected StorageContainerLocationProtocolClientSideTranslatorPB + public StorageContainerLocationProtocolClientSideTranslatorPB createStorageContainerLocationClient() throws IOException { long version = RPC.getProtocolVersion( StorageContainerLocationProtocolPB.class); InetSocketAddress address = scm.getStorageContainerLocationRpcAddress(); + LOG.info( + "Creating StorageContainerLocationProtocol RPC client with address {}", + address); return new StorageContainerLocationProtocolClientSideTranslatorPB( RPC.getProxy(StorageContainerLocationProtocolPB.class, version, address, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), Client.getTimeout(conf))); } + + /** + * Waits for the Ozone cluster to be ready for processing requests. + */ + public void waitOzoneReady() { + long begin = Time.monotonicNow(); + while (scm.getDatanodeReport(DatanodeReportType.LIVE).length < + numDataNodes) { + if (Time.monotonicNow() - begin > 20000) { + throw new IllegalStateException( + "Timed out waiting for Ozone cluster to become ready."); + } + LOG.info("Waiting for Ozone cluster to become ready"); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted while waiting for Ozone cluster to become ready."); + } + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fedb22d9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java new file mode 100644 index 0000000..315f59d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.web; + +import static com.google.common.base.Charsets.UTF_8; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.ozone.OzoneConsts.CHUNK_SIZE; +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneConfiguration; +import org.apache.hadoop.ozone.web.client.OzoneBucket; +import org.apache.hadoop.ozone.web.client.OzoneClient; +import org.apache.hadoop.ozone.web.client.OzoneVolume; +import org.apache.hadoop.ozone.web.request.OzoneQuota; + +/** + * End-to-end testing of Ozone REST operations. + */ +public class TestOzoneRestWithMiniCluster { + + private static MiniOzoneCluster cluster; + private static OzoneConfiguration conf; + private static int idSuffix; + private static OzoneClient ozoneClient; + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, true); + conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "distributed"); + conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_TRACE_ENABLED_KEY, true); + cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(3).build(); + cluster.waitOzoneReady(); + ozoneClient = cluster.createOzoneClient(); + } + + @AfterClass + public static void shutdown() throws InterruptedException { + IOUtils.cleanup(null, ozoneClient, cluster); + } + + @Test + public void testCreateAndGetVolume() throws Exception { + String volumeName = nextId("volume"); + OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB"); + assertNotNull(volume); + assertEquals(volumeName, volume.getVolumeName()); + assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby()); + assertEquals("bilbo", volume.getOwnerName()); + assertNotNull(volume.getQuota()); + assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), + volume.getQuota().sizeInBytes()); + volume = ozoneClient.getVolume(volumeName); + assertNotNull(volume); + assertEquals(volumeName, volume.getVolumeName()); + assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby()); + assertEquals("bilbo", volume.getOwnerName()); + assertNotNull(volume.getQuota()); + assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), + volume.getQuota().sizeInBytes()); + } + + @Test + public void testCreateAndGetBucket() throws Exception { + String volumeName = nextId("volume"); + String bucketName = nextId("bucket"); + OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB"); + assertNotNull(volume); + assertEquals(volumeName, volume.getVolumeName()); + assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby()); + assertEquals("bilbo", volume.getOwnerName()); + assertNotNull(volume.getQuota()); + assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), + volume.getQuota().sizeInBytes()); + OzoneBucket bucket = volume.createBucket(bucketName); + assertNotNull(bucket); + assertEquals(bucketName, bucket.getBucketName()); + bucket = volume.getBucket(bucketName); + assertNotNull(bucket); + assertEquals(bucketName, bucket.getBucketName()); + } + + @Test + public void testPutAndGetKey() throws Exception { + String volumeName = nextId("volume"); + String bucketName = nextId("bucket"); + String keyName = nextId("key"); + String keyData = nextId("data"); + OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB"); + assertNotNull(volume); + assertEquals(volumeName, volume.getVolumeName()); + assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby()); + assertEquals("bilbo", volume.getOwnerName()); + assertNotNull(volume.getQuota()); + assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), + volume.getQuota().sizeInBytes()); + OzoneBucket bucket = volume.createBucket(bucketName); + assertNotNull(bucket); + assertEquals(bucketName, bucket.getBucketName()); + bucket.putKey(keyName, keyData); + assertEquals(keyData, bucket.getKey(keyName)); + } + + @Test + public void testPutAndGetEmptyKey() throws Exception { + String volumeName = nextId("volume"); + String bucketName = nextId("bucket"); + String keyName = nextId("key"); + String keyData = ""; + OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB"); + assertNotNull(volume); + assertEquals(volumeName, volume.getVolumeName()); + assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby()); + assertEquals("bilbo", volume.getOwnerName()); + assertNotNull(volume.getQuota()); + assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), + volume.getQuota().sizeInBytes()); + OzoneBucket bucket = volume.createBucket(bucketName); + assertNotNull(bucket); + assertEquals(bucketName, bucket.getBucketName()); + bucket.putKey(keyName, keyData); + assertEquals(keyData, bucket.getKey(keyName)); + } + + @Test + public void testPutAndGetMultiChunkKey() throws Exception { + String volumeName = nextId("volume"); + String bucketName = nextId("bucket"); + String keyName = nextId("key"); + int keyDataLen = 3 * CHUNK_SIZE; + String keyData = buildKeyData(keyDataLen); + OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB"); + assertNotNull(volume); + assertEquals(volumeName, volume.getVolumeName()); + assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby()); + assertEquals("bilbo", volume.getOwnerName()); + assertNotNull(volume.getQuota()); + assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), + volume.getQuota().sizeInBytes()); + OzoneBucket bucket = volume.createBucket(bucketName); + assertNotNull(bucket); + assertEquals(bucketName, bucket.getBucketName()); + bucket.putKey(keyName, keyData); + assertEquals(keyData, bucket.getKey(keyName)); + } + + @Test + public void testPutAndGetMultiChunkKeyLastChunkPartial() throws Exception { + String volumeName = nextId("volume"); + String bucketName = nextId("bucket"); + String keyName = nextId("key"); + int keyDataLen = (int)(2.5 * CHUNK_SIZE); + String keyData = buildKeyData(keyDataLen); + OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB"); + assertNotNull(volume); + assertEquals(volumeName, volume.getVolumeName()); + assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby()); + assertEquals("bilbo", volume.getOwnerName()); + assertNotNull(volume.getQuota()); + assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), + volume.getQuota().sizeInBytes()); + OzoneBucket bucket = volume.createBucket(bucketName); + assertNotNull(bucket); + assertEquals(bucketName, bucket.getBucketName()); + bucket.putKey(keyName, keyData); + assertEquals(keyData, bucket.getKey(keyName)); + } + + @Test + public void testReplaceKey() throws Exception { + String volumeName = nextId("volume"); + String bucketName = nextId("bucket"); + String keyName = nextId("key"); + int keyDataLen = (int)(2.5 * CHUNK_SIZE); + String keyData = buildKeyData(keyDataLen); + OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB"); + assertNotNull(volume); + assertEquals(volumeName, volume.getVolumeName()); + assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby()); + assertEquals("bilbo", volume.getOwnerName()); + assertNotNull(volume.getQuota()); + assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), + volume.getQuota().sizeInBytes()); + OzoneBucket bucket = volume.createBucket(bucketName); + assertNotNull(bucket); + assertEquals(bucketName, bucket.getBucketName()); + bucket.putKey(keyName, keyData); + assertEquals(keyData, bucket.getKey(keyName)); + + // Replace key with data consisting of fewer chunks. + keyDataLen = (int)(1.5 * CHUNK_SIZE); + keyData = buildKeyData(keyDataLen); + bucket.putKey(keyName, keyData); + assertEquals(keyData, bucket.getKey(keyName)); + + // Replace key with data consisting of more chunks. + keyDataLen = (int)(3.5 * CHUNK_SIZE); + keyData = buildKeyData(keyDataLen); + bucket.putKey(keyName, keyData); + assertEquals(keyData, bucket.getKey(keyName)); + } + + /** + * Creates sample key data of the specified length. The data is a string of + * printable ASCII characters. This makes it easy to debug through visual + * inspection of the chunk files if a test fails. + * + * @param keyDataLen desired length of key data + * @return string of printable ASCII characters of the specified length + */ + private static String buildKeyData(int keyDataLen) { + return new String(dataset(keyDataLen, 33, 93), UTF_8); + } + + /** + * Generates identifiers unique enough for use in tests, so that individual + * tests don't collide on each others' data in the shared mini-cluster. + * + * @param idPrefix prefix to put in front of ID + * @return unique ID generated by appending a suffix to the given prefix + */ + private static String nextId(String idPrefix) { + return idPrefix + ++idSuffix; + } +}
