HDDS-374. Support to configure container size in units lesser than GB. Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12b2f362 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12b2f362 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12b2f362 Branch: refs/heads/HDFS-12943 Commit: 12b2f362cc9a370904da724a68ba015cd3a99eff Parents: 91836f0 Author: Nanda kumar <na...@apache.org> Authored: Mon Aug 27 18:29:32 2018 +0530 Committer: Nanda kumar <na...@apache.org> Committed: Mon Aug 27 18:29:32 2018 +0530 ---------------------------------------------------------------------- .../org/apache/hadoop/ozone/OzoneConsts.java | 2 +- .../container/common/impl/ContainerData.java | 24 ++++++++++---------- .../common/impl/ContainerDataYaml.java | 5 ++-- .../container/common/impl/HddsDispatcher.java | 6 +---- .../container/keyvalue/KeyValueContainer.java | 2 +- .../keyvalue/KeyValueContainerData.java | 10 ++++---- .../container/keyvalue/KeyValueHandler.java | 15 ++++++------ .../common/TestKeyValueContainerData.java | 5 ++-- .../common/impl/TestContainerDataYaml.java | 7 +++--- .../container/common/impl/TestContainerSet.java | 7 ++++-- .../common/impl/TestHddsDispatcher.java | 3 ++- .../keyvalue/TestChunkManagerImpl.java | 4 +++- .../container/keyvalue/TestKeyManagerImpl.java | 4 +++- .../keyvalue/TestKeyValueBlockIterator.java | 4 +++- .../keyvalue/TestKeyValueContainer.java | 13 +++++++---- .../container/keyvalue/TestKeyValueHandler.java | 4 +++- .../container/ozoneimpl/TestOzoneContainer.java | 5 ++-- .../test/resources/additionalfields.container | 4 ++-- .../test/resources/incorrect.checksum.container | 2 +- .../src/test/resources/incorrect.container | 2 +- .../ozone/container/ContainerTestHelper.java | 4 +++- .../common/TestBlockDeletingService.java | 2 +- .../TestContainerDeletionChoosingPolicy.java | 8 +++---- .../common/impl/TestContainerPersistence.java | 3 +-- 24 files changed, 79 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index f912f02..320a3ed 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -185,7 +185,7 @@ public final class OzoneConsts { public static final String CONTAINER_TYPE = "containerType"; public static final String STATE = "state"; public static final String METADATA = "metadata"; - public static final String MAX_SIZE_GB = "maxSizeGB"; + public static final String MAX_SIZE = "maxSize"; public static final String METADATA_PATH = "metadataPath"; public static final String CHUNKS_PATH = "chunksPath"; public static final String CONTAINER_DB_TYPE = "containerDBType"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java index afd1407..efea20b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java @@ -40,7 +40,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.CHECKSUM; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ID; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_TYPE; import static org.apache.hadoop.ozone.OzoneConsts.LAYOUTVERSION; -import static org.apache.hadoop.ozone.OzoneConsts.MAX_SIZE_GB; +import static org.apache.hadoop.ozone.OzoneConsts.MAX_SIZE; import static org.apache.hadoop.ozone.OzoneConsts.METADATA; import static org.apache.hadoop.ozone.OzoneConsts.STATE; @@ -67,7 +67,7 @@ public abstract class ContainerData { // State of the Container private ContainerLifeCycleState state; - private final int maxSizeGB; + private final long maxSize; /** parameters for read/write statistics on the container. **/ private final AtomicLong readBytes; @@ -92,16 +92,16 @@ public abstract class ContainerData { LAYOUTVERSION, STATE, METADATA, - MAX_SIZE_GB, + MAX_SIZE, CHECKSUM)); /** * Creates a ContainerData Object, which holds metadata of the container. * @param type - ContainerType * @param containerId - ContainerId - * @param size - container maximum size + * @param size - container maximum size in bytes */ - protected ContainerData(ContainerType type, long containerId, int size) { + protected ContainerData(ContainerType type, long containerId, long size) { this(type, containerId, ChunkLayOutVersion.getLatestVersion().getVersion(), size); } @@ -111,10 +111,10 @@ public abstract class ContainerData { * @param type - ContainerType * @param containerId - ContainerId * @param layOutVersion - Container layOutVersion - * @param size - Container maximum size in GB + * @param size - Container maximum size in bytes */ protected ContainerData(ContainerType type, long containerId, - int layOutVersion, int size) { + int layOutVersion, long size) { Preconditions.checkNotNull(type); this.containerType = type; @@ -128,7 +128,7 @@ public abstract class ContainerData { this.writeBytes = new AtomicLong(0L); this.bytesUsed = new AtomicLong(0L); this.keyCount = new AtomicLong(0L); - this.maxSizeGB = size; + this.maxSize = size; setChecksumTo0ByteArray(); } @@ -171,11 +171,11 @@ public abstract class ContainerData { } /** - * Return's maximum size of the container in GB. - * @return maxSizeGB + * Return's maximum size of the container in bytes. + * @return maxSize in bytes */ - public int getMaxSizeGB() { - return maxSizeGB; + public long getMaxSize() { + return maxSize; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java index ec6d642..65262d4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java @@ -236,12 +236,11 @@ public final class ContainerDataYaml { long layOutVersion = (long) nodes.get(OzoneConsts.LAYOUTVERSION); int lv = (int) layOutVersion; - long size = (long) nodes.get(OzoneConsts.MAX_SIZE_GB); - int maxSize = (int) size; + long size = (long) nodes.get(OzoneConsts.MAX_SIZE); //When a new field is added, it needs to be added here. KeyValueContainerData kvData = new KeyValueContainerData( - (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, maxSize); + (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, size); kvData.setContainerDBType((String)nodes.get( OzoneConsts.CONTAINER_DB_TYPE)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index d92eb17..ebc2628 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -21,12 +21,8 @@ package org.apache.hadoop.ozone.container.common.impl; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerInfo; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerAction; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -166,7 +162,7 @@ public class HddsDispatcher implements ContainerDispatcher { if (isOpen) { ContainerData containerData = container.getContainerData(); double containerUsedPercentage = 1.0f * containerData.getBytesUsed() / - StorageUnit.GB.toBytes(containerData.getMaxSizeGB()); + containerData.getMaxSize(); if (containerUsedPercentage >= containerCloseThreshold) { ContainerAction action = ContainerAction.newBuilder() .setContainerID(containerData.getContainerID()) http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java index 8108a11..f60c5e9 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java @@ -109,7 +109,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData> { File containerMetaDataPath = null; //acquiring volumeset lock and container lock volumeSet.acquireLock(); - long maxSize = (containerData.getMaxSizeGB() * 1024L * 1024L * 1024L); + long maxSize = containerData.getMaxSize(); try { HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet .getVolumesList(), maxSize); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java index e4cb5f3..7ffdbf5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java @@ -84,9 +84,9 @@ public class KeyValueContainerData extends ContainerData { /** * Constructs KeyValueContainerData object. * @param id - ContainerId - * @param size - maximum size in GB of the container + * @param size - maximum size of the container in bytes */ - public KeyValueContainerData(long id, int size) { + public KeyValueContainerData(long id, long size) { super(ContainerProtos.ContainerType.KeyValueContainer, id, size); this.numPendingDeletionBlocks = new AtomicInteger(0); this.deleteTransactionId = 0; @@ -96,9 +96,9 @@ public class KeyValueContainerData extends ContainerData { * Constructs KeyValueContainerData object. * @param id - ContainerId * @param layOutVersion - * @param size - maximum size in GB of the container + * @param size - maximum size of the container in bytes */ - public KeyValueContainerData(long id, int layOutVersion, int size) { + public KeyValueContainerData(long id, int layOutVersion, long size) { super(ContainerProtos.ContainerType.KeyValueContainer, id, layOutVersion, size); this.numPendingDeletionBlocks = new AtomicInteger(0); @@ -272,7 +272,7 @@ public class KeyValueContainerData extends ContainerData { ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT); KeyValueContainerData data = new KeyValueContainerData( protoData.getContainerID(), - (int)storageSize.getUnit().toBytes(storageSize.getValue())); + (long)storageSize.getUnit().toBytes(storageSize.getValue())); for (int x = 0; x < protoData.getMetadataCount(); x++) { data.addMetadata(protoData.getMetadata(x).getKey(), protoData.getMetadata(x).getValue()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index 8409561..e254833 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -81,8 +81,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .Result.CLOSED_CONTAINER_RETRY; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Result.CONTAINER_INTERNAL_ERROR; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Result.CLOSED_CONTAINER_IO; @@ -124,8 +122,8 @@ public class KeyValueHandler extends Handler { private final KeyManager keyManager; private final ChunkManager chunkManager; private final BlockDeletingService blockDeletingService; - private VolumeChoosingPolicy volumeChoosingPolicy; - private final int maxContainerSizeGB; + private final VolumeChoosingPolicy volumeChoosingPolicy; + private final long maxContainerSize; private final AutoCloseableLock handlerLock; private final OpenContainerBlockMap openContainerBlockMap; @@ -150,9 +148,9 @@ public class KeyValueHandler extends Handler { volumeChoosingPolicy = ReflectionUtils.newInstance(conf.getClass( HDDS_DATANODE_VOLUME_CHOOSING_POLICY, RoundRobinVolumeChoosingPolicy .class, VolumeChoosingPolicy.class), conf); - maxContainerSizeGB = (int)config.getStorageSize( + maxContainerSize = (long)config.getStorageSize( ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.GB); + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); // this handler lock is used for synchronizing createContainer Requests, // so using a fair lock here. handlerLock = new AutoCloseableLock(new ReentrantLock(true)); @@ -215,8 +213,9 @@ public class KeyValueHandler extends Handler { return handleGetSmallFile(request, kvContainer); case GetCommittedBlockLength: return handleGetCommittedBlockLength(request, kvContainer); + default: + return null; } - return null; } @VisibleForTesting @@ -247,7 +246,7 @@ public class KeyValueHandler extends Handler { long containerID = request.getContainerID(); KeyValueContainerData newContainerData = new KeyValueContainerData( - containerID, maxContainerSizeGB); + containerID, maxContainerSize); // TODO: Add support to add metadataList to ContainerData. Add metadata // to container during creation. KeyValueContainer newContainer = new KeyValueContainer( http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java index 12ce163..f991520 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.common; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.junit.Test; @@ -31,7 +32,7 @@ import java.util.concurrent.atomic.AtomicLong; */ public class TestKeyValueContainerData { - private static final int MAXSIZE = 5; + private static final long MAXSIZE = (long) StorageUnit.GB.toBytes(5); @Test public void testKeyValueData() { long containerId = 1L; @@ -58,7 +59,7 @@ public class TestKeyValueContainerData { assertEquals(val.get(), kvData.getWriteCount()); assertEquals(val.get(), kvData.getKeyCount()); assertEquals(val.get(), kvData.getNumPendingDeletionBlocks()); - assertEquals(MAXSIZE, kvData.getMaxSizeGB()); + assertEquals(MAXSIZE, kvData.getMaxSize()); kvData.setState(state); kvData.setContainerDBType(containerDBType); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java index fd51db3..c7b9e0a 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.common.impl; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; @@ -42,7 +43,7 @@ public class TestContainerDataYaml { private static String testRoot = new FileSystemTestHelper().getTestRootDir(); - private static final int MAXSIZE = 5; + private static final long MAXSIZE = (long) StorageUnit.GB.toBytes(5); /** * Creates a .container file. cleanup() should be called at the end of the @@ -94,7 +95,7 @@ public class TestContainerDataYaml { .getState()); assertEquals(1, kvData.getLayOutVersion()); assertEquals(0, kvData.getMetadata().size()); - assertEquals(MAXSIZE, kvData.getMaxSizeGB()); + assertEquals(MAXSIZE, kvData.getMaxSize()); // Update ContainerData. kvData.addMetadata("VOLUME", "hdfs"); @@ -122,7 +123,7 @@ public class TestContainerDataYaml { assertEquals(2, kvData.getMetadata().size()); assertEquals("hdfs", kvData.getMetadata().get("VOLUME")); assertEquals("ozone", kvData.getMetadata().get("OWNER")); - assertEquals(MAXSIZE, kvData.getMaxSizeGB()); + assertEquals(MAXSIZE, kvData.getMaxSize()); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java index ae670e0..af322ea 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.container.common.impl; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto @@ -53,7 +54,8 @@ public class TestContainerSet { ContainerProtos.ContainerLifeCycleState state = ContainerProtos .ContainerLifeCycleState.CLOSED; - KeyValueContainerData kvData = new KeyValueContainerData(containerId, 5); + KeyValueContainerData kvData = new KeyValueContainerData(containerId, + (long) StorageUnit.GB.toBytes(5)); kvData.setState(state); KeyValueContainer keyValueContainer = new KeyValueContainer(kvData, new OzoneConfiguration()); @@ -163,7 +165,8 @@ public class TestContainerSet { private ContainerSet createContainerSet() throws StorageContainerException { ContainerSet containerSet = new ContainerSet(); for (int i=0; i<10; i++) { - KeyValueContainerData kvData = new KeyValueContainerData(i, 5); + KeyValueContainerData kvData = new KeyValueContainerData(i, + (long) StorageUnit.GB.toBytes(5)); if (i%2 == 0) { kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSED); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java index b107782..fc622b2 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java @@ -70,7 +70,8 @@ public class TestHddsDispatcher { ContainerSet containerSet = new ContainerSet(); VolumeSet volumeSet = new VolumeSet(dd.getUuidString(), conf); StateContext context = Mockito.mock(StateContext.class); - KeyValueContainerData containerData = new KeyValueContainerData(1L, 1); + KeyValueContainerData containerData = new KeyValueContainerData(1L, + (long) StorageUnit.GB.toBytes(1)); Container container = new KeyValueContainer(containerData, conf); container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), scmId.toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java index 760d873..9664052 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.keyvalue; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; @@ -79,7 +80,8 @@ public class TestChunkManagerImpl { Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) .thenReturn(hddsVolume); - keyValueContainerData = new KeyValueContainerData(1L, 5); + keyValueContainerData = new KeyValueContainerData(1L, + (long) StorageUnit.GB.toBytes(5)); keyValueContainer = new KeyValueContainer(keyValueContainerData, config); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java index a90cf80..b05dbca 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.keyvalue; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; @@ -79,7 +80,8 @@ public class TestKeyManagerImpl { Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) .thenReturn(hddsVolume); - keyValueContainerData = new KeyValueContainerData(1L, 5); + keyValueContainerData = new KeyValueContainerData(1L, + (long) StorageUnit.GB.toBytes(5)); keyValueContainer = new KeyValueContainer( keyValueContainerData, config); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java index ba57c3f..f1fe88e 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.keyvalue; import com.google.common.primitives.Longs; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -244,7 +245,8 @@ public class TestKeyValueBlockIterator { private void createContainerWithBlocks(long containerId, int normalBlocks, int deletedBlocks) throws Exception { - containerData = new KeyValueContainerData(containerId, 1); + containerData = new KeyValueContainerData(containerId, + (long) StorageUnit.GB.toBytes(1)); container = new KeyValueContainer(containerData, conf); container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID .randomUUID().toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java index 7359868..f84ba7d 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.container.keyvalue; import com.google.common.primitives.Longs; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; @@ -94,7 +95,8 @@ public class TestKeyValueContainer { Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) .thenReturn(hddsVolume); - keyValueContainerData = new KeyValueContainerData(1L, 5); + keyValueContainerData = new KeyValueContainerData(1L, + (long) StorageUnit.GB.toBytes(5)); keyValueContainer = new KeyValueContainer( keyValueContainerData, conf); @@ -103,7 +105,8 @@ public class TestKeyValueContainer { @Test public void testBlockIterator() throws Exception{ - keyValueContainerData = new KeyValueContainerData(100L, 1); + keyValueContainerData = new KeyValueContainerData(100L, + (long) StorageUnit.GB.toBytes(1)); keyValueContainer = new KeyValueContainer( keyValueContainerData, conf); keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); @@ -213,7 +216,7 @@ public class TestKeyValueContainer { //create a new one KeyValueContainerData containerData = new KeyValueContainerData(containerId, 1, - keyValueContainerData.getMaxSizeGB()); + keyValueContainerData.getMaxSize()); KeyValueContainer container = new KeyValueContainer(containerData, conf); HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet @@ -234,8 +237,8 @@ public class TestKeyValueContainer { containerData.getKeyCount()); Assert.assertEquals(keyValueContainerData.getLayOutVersion(), containerData.getLayOutVersion()); - Assert.assertEquals(keyValueContainerData.getMaxSizeGB(), - containerData.getMaxSizeGB()); + Assert.assertEquals(keyValueContainerData.getMaxSize(), + containerData.getMaxSize()); Assert.assertEquals(keyValueContainerData.getBytesUsed(), containerData.getBytesUsed()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java index ce12e1f..d91bbf7 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.keyvalue; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; @@ -262,7 +263,8 @@ public class TestKeyValueHandler { public void testCloseInvalidContainer() { long containerID = 1234L; Configuration conf = new Configuration(); - KeyValueContainerData kvData = new KeyValueContainerData(containerID, 1); + KeyValueContainerData kvData = new KeyValueContainerData(containerID, + (long) StorageUnit.GB.toBytes(1)); KeyValueContainer container = new KeyValueContainer(kvData, conf); kvData.setState(ContainerProtos.ContainerLifeCycleState.INVALID); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 19ec6a2..173a8b2 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -19,9 +19,9 @@ package org.apache.hadoop.ozone.container.ozoneimpl; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; @@ -77,7 +77,8 @@ public class TestOzoneContainer { // Add containers to disk for (int i=0; i<10; i++) { - keyValueContainerData = new KeyValueContainerData(i, 1); + keyValueContainerData = new KeyValueContainerData(i, + (long) StorageUnit.GB.toBytes(1)); keyValueContainer = new KeyValueContainer( keyValueContainerData, conf); keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/resources/additionalfields.container ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/resources/additionalfields.container b/hadoop-hdds/container-service/src/test/resources/additionalfields.container index 73cf5f3..fff5304 100644 --- a/hadoop-hdds/container-service/src/test/resources/additionalfields.container +++ b/hadoop-hdds/container-service/src/test/resources/additionalfields.container @@ -5,8 +5,8 @@ containerID: 9223372036854775807 containerType: KeyValueContainer metadataPath: /hdds/current/aed-fg4-hji-jkl/containerDir0/1 layOutVersion: 1 -maxSizeGB: 5 +maxSize: 5368709120 metadata: {OWNER: ozone, VOLUME: hdfs} state: CLOSED aclEnabled: true -checksum: 1bbff32aeaa8fadc0b80c5c1e0597036e96acd8ae4bddbed188a2162762251a2 \ No newline at end of file +checksum: c5b5373b8755c4e7199478dcaded9d996f9aca089704e08950259cdb0f290680 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container b/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container index feeeadc..d06ba57 100644 --- a/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container +++ b/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container @@ -5,7 +5,7 @@ containerID: 9223372036854775807 containerType: KeyValueContainer metadataPath: /hdds/current/aed-fg4-hji-jkl/containerdir0/1 layOutVersion: 1 -maxSizeGB: 5 +maxSize: 5368709120 metadata: {OWNER: ozone, VOLUME: hdfs} state: OPEN checksum: 08bc9d390f9183aeed3cf33c789e2a07310bba60f3cf55941caccc939db8670f \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/resources/incorrect.container ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.container b/hadoop-hdds/container-service/src/test/resources/incorrect.container index 8aeb30c..0053ab2 100644 --- a/hadoop-hdds/container-service/src/test/resources/incorrect.container +++ b/hadoop-hdds/container-service/src/test/resources/incorrect.container @@ -5,7 +5,7 @@ containerID: 9223372036854775807 containerType: KeyValueContainer metadataPath: /hdds/current/aed-fg4-hji-jkl/containerDir0/1 layOutVersion: 1 -maxSizeGB: 5 +maxSize: 5368709120 metadata: {OWNER: ozone, VOLUME: hdfs} state: INVALID checksum: 08bc9d390f9183aeed3cf33c789e2a07310bba60f3cf55941caccc939db8670f \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java index ff68863..ca92110 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.container; import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID; import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.commons.codec.binary.Hex; @@ -59,7 +60,8 @@ public final class ContainerTestHelper { ContainerTestHelper.class); private static Random r = new Random(); - public static final int CONTAINER_MAX_SIZE_GB = 1; + public static final long CONTAINER_MAX_SIZE = + (long) StorageUnit.GB.toBytes(1); /** * Never constructed. http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index dcf4022..25c8c6b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -110,7 +110,7 @@ public class TestBlockDeletingService { conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath()); long containerID = ContainerTestHelper.getTestContainerID(); KeyValueContainerData data = new KeyValueContainerData(containerID, - ContainerTestHelper.CONTAINER_MAX_SIZE_GB); + ContainerTestHelper.CONTAINER_MAX_SIZE); Container container = new KeyValueContainer(data, conf); container.create(new VolumeSet(scmId, clusterID, conf), new RoundRobinVolumeChoosingPolicy(), scmId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java index b2e4c9a..0d46ecf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java @@ -75,8 +75,8 @@ public class TestContainerDeletionChoosingPolicy { int numContainers = 10; for (int i = 0; i < numContainers; i++) { - KeyValueContainerData data = new KeyValueContainerData(new Long(i), - ContainerTestHelper.CONTAINER_MAX_SIZE_GB); + KeyValueContainerData data = new KeyValueContainerData(i, + ContainerTestHelper.CONTAINER_MAX_SIZE); KeyValueContainer container = new KeyValueContainer(data, conf); containerSet.addContainer(container); Assert.assertTrue( @@ -128,8 +128,8 @@ public class TestContainerDeletionChoosingPolicy { for (int i = 0; i <= numContainers; i++) { long containerId = RandomUtils.nextLong(); KeyValueContainerData data = - new KeyValueContainerData(new Long(containerId), - ContainerTestHelper.CONTAINER_MAX_SIZE_GB); + new KeyValueContainerData(containerId, + ContainerTestHelper.CONTAINER_MAX_SIZE); if (i != numContainers) { int deletionBlocks = random.nextInt(numContainers) + 1; data.incrPendingDeletionBlocks(deletionBlocks); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index 016b94c..8e11a97 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.container.common.impl; import com.google.common.collect.Maps; import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto @@ -193,7 +192,7 @@ public class TestContainerPersistence { private Container addContainer(ContainerSet containerSet, long containerID) throws IOException { KeyValueContainerData data = new KeyValueContainerData(containerID, - ContainerTestHelper.CONTAINER_MAX_SIZE_GB); + ContainerTestHelper.CONTAINER_MAX_SIZE); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner)", "bilbo"); KeyValueContainer container = new KeyValueContainer(data, conf); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org