This is an automated email from the ASF dual-hosted git repository. sammichen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push: new d08a4c1 HDDS-3728. Bucket space: check quotaUsageInBytes when write key and allocate block. (#1458) d08a4c1 is described below commit d08a4c1d973f7752875436f497c4a007bea9f250 Author: micah zhao <micahz...@tencent.com> AuthorDate: Fri Oct 9 19:42:53 2020 +0800 HDDS-3728. Bucket space: check quotaUsageInBytes when write key and allocate block. (#1458) --- .../client/rpc/TestOzoneRpcClientAbstract.java | 67 ++++++++++++++++++++-- .../ozone/om/request/file/OMFileCreateRequest.java | 3 +- .../om/request/key/OMAllocateBlockRequest.java | 3 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 3 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 21 +++++++ 5 files changed, 88 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index c9551fd..b7b75a4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -815,6 +815,7 @@ public abstract class TestOzoneRpcClientAbstract { } @Test + @SuppressWarnings("methodlength") public void testCheckUsedBytesQuota() throws IOException { String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); @@ -828,13 +829,15 @@ public abstract class TestOzoneRpcClientAbstract { store.createVolume(volumeName); volume = store.getVolume(volumeName); + + // Test volume quota. // Set quota In Bytes for a smaller value store.getVolume(volumeName).setQuota( OzoneQuota.parseQuota("1 Bytes", 100)); volume.createBucket(bucketName); OzoneBucket bucket = volume.getBucket(bucketName); - // Test write key. + // Test volume quota: write key. // The remaining quota does not satisfy a block size, so the write fails. try { writeKey(bucket, UUID.randomUUID().toString(), ONE, value, valueLength); @@ -845,7 +848,7 @@ public abstract class TestOzoneRpcClientAbstract { // Write failed, volume usedBytes should be 0 Assert.assertEquals(0L, store.getVolume(volumeName).getUsedBytes()); - // Test write file. + // Test volume quota: write file. // The remaining quota does not satisfy a block size, so the write fails. try { writeFile(bucket, UUID.randomUUID().toString(), ONE, value, 0); @@ -856,7 +859,7 @@ public abstract class TestOzoneRpcClientAbstract { // Write failed, volume usedBytes should be 0 Assert.assertEquals(0L, store.getVolume(volumeName).getUsedBytes()); - // Write a key(with two blocks), test allocateBlock fails. + // Test volume quota: write key(with two blocks), test allocateBlock fails. store.getVolume(volumeName).setQuota( OzoneQuota.parseQuota(blockSize + "Bytes", 100)); try { @@ -873,8 +876,8 @@ public abstract class TestOzoneRpcClientAbstract { // AllocateBlock failed, volume usedBytes should be 1 * blockSize. Assert.assertEquals(blockSize, store.getVolume(volumeName).getUsedBytes()); - // Write large key(with five blocks), the first four blocks will succeed, - // while the later block will fail. + // Test volume quota: write large key(with five blocks), the first four + // blocks will succeed,while the later block will fail. store.getVolume(volumeName).setQuota( OzoneQuota.parseQuota(5 * blockSize + "Bytes", 100)); try { @@ -892,7 +895,59 @@ public abstract class TestOzoneRpcClientAbstract { Assert.assertEquals(5 * blockSize, store.getVolume(volumeName).getUsedBytes()); - Assert.assertEquals(4, countException); + // Test bucket quota. + // Set quota In Bytes for a smaller value + store.getVolume(volumeName).setQuota( + OzoneQuota.parseQuota(Long.MAX_VALUE + " Bytes", 100)); + bucketName = UUID.randomUUID().toString(); + volume.createBucket(bucketName); + bucket = volume.getBucket(bucketName); + bucket.setQuota(OzoneQuota.parseQuota("1 Bytes", 100)); + + // Test bucket quota: write key. + // The remaining quota does not satisfy a block size, so the write fails. + try { + writeKey(bucket, UUID.randomUUID().toString(), ONE, value, valueLength); + } catch (IOException ex) { + countException++; + GenericTestUtils.assertExceptionContains("QUOTA_EXCEEDED", ex); + } + // Write failed, bucket usedBytes should be 0 + Assert.assertEquals(0L, + store.getVolume(volumeName).getBucket(bucketName).getUsedBytes()); + + // Test bucket quota: write file. + // The remaining quota does not satisfy a block size, so the write fails. + try { + writeFile(bucket, UUID.randomUUID().toString(), ONE, value, 0); + } catch (IOException ex) { + countException++; + GenericTestUtils.assertExceptionContains("QUOTA_EXCEEDED", ex); + } + // Write failed, bucket usedBytes should be 0 + Assert.assertEquals(0L, + store.getVolume(volumeName).getBucket(bucketName).getUsedBytes()); + + // Test bucket quota: write large key(with five blocks), the first four + // blocks will succeed,while the later block will fail. + bucket.setQuota(OzoneQuota.parseQuota( + 4 * blockSize + " Bytes", 100)); + try { + OzoneOutputStream out = bucket.createKey(UUID.randomUUID().toString(), + valueLength, STAND_ALONE, ONE, new HashMap<>()); + for (int i = 0; i <= (4 * blockSize) / value.length(); i++) { + out.write(value.getBytes()); + } + out.close(); + } catch (IOException ex) { + countException++; + GenericTestUtils.assertExceptionContains("QUOTA_EXCEEDED", ex); + } + // AllocateBlock failed, bucket usedBytes should be 4 * blockSize + Assert.assertEquals(4 * blockSize, + store.getVolume(volumeName).getBucket(bucketName).getUsedBytes()); + + Assert.assertEquals(7, countException); } @Test diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 367e4ba..9a7f31a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -279,10 +279,11 @@ public class OMFileCreateRequest extends OMKeyRequest { omVolumeArgs = getVolumeInfo(omMetadataManager, volumeName); omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); - // check volume quota + // check bucket and volume quota long preAllocatedSpace = newLocationList.size() * ozoneManager.getScmBlockSize() * omKeyInfo.getFactor().getNumber(); + checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace); checkVolumeQuotaInBytes(omVolumeArgs, preAllocatedSpace); // Add to cache entry can be done outside of lock for this openKey. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index afd6162..194e7ef 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -196,10 +196,11 @@ public class OMAllocateBlockRequest extends OMKeyRequest { OmKeyLocationInfo.getFromProtobuf(blockLocation)); omVolumeArgs = getVolumeInfo(omMetadataManager, volumeName); omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); - // check volume quota + // check bucket and volume quota long preAllocatedSpace = newLocationList.size() * ozoneManager.getScmBlockSize() * openKeyInfo.getFactor().getNumber(); + checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace); checkVolumeQuotaInBytes(omVolumeArgs, preAllocatedSpace); // Append new block openKeyInfo.appendNewBlocks(newLocationList, false); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index f16153d..5ec79b5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -298,7 +298,8 @@ public class OMKeyCreateRequest extends OMKeyRequest { long preAllocatedSpace = newLocationList.size() * ozoneManager.getScmBlockSize() * omKeyInfo.getFactor().getNumber(); - // check volume quota + // check bucket and volume quota + checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace); checkVolumeQuotaInBytes(omVolumeArgs, preAllocatedSpace); // Add to cache entry can be done outside of lock for this openKey. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index e24f4e2..bf99169 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -556,6 +556,27 @@ public abstract class OMKeyRequest extends OMClientRequest { } /** + * Check bucket quota in bytes. + * @param omBucketInfo + * @param allocateSize + * @throws IOException + */ + protected void checkBucketQuotaInBytes(OmBucketInfo omBucketInfo, + long allocateSize) throws IOException { + if (omBucketInfo.getQuotaInBytes() > OzoneConsts.QUOTA_RESET) { + long usedBytes = omBucketInfo.getUsedBytes().sum(); + long quotaInBytes = omBucketInfo.getQuotaInBytes(); + if (quotaInBytes - usedBytes < allocateSize) { + throw new OMException("The DiskSpace quota of bucket:" + + omBucketInfo.getBucketName() + "exceeded: quotaInBytes: " + + quotaInBytes + " Bytes but diskspace consumed: " + (usedBytes + + allocateSize) + " Bytes.", + OMException.ResultCodes.QUOTA_EXCEEDED); + } + } + } + + /** * Check directory exists. If exists return true, else false. * @param volumeName * @param bucketName --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org