This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 206686837b HDDS-10046. Replace PrecomputedVolumeSpace with
SpaceUsageSource.Fixed (#5904)
206686837b is described below
commit 206686837bd7fc3a470652d97b749efca08fd0e1
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Jan 5 08:25:02 2024 +0100
HDDS-10046. Replace PrecomputedVolumeSpace with SpaceUsageSource.Fixed
(#5904)
---
.../apache/hadoop/hdds/fs/SpaceUsageSource.java | 43 ++++++++++++++++++++++
.../hadoop/hdds/fs/MockSpaceUsageSource.java | 30 +--------------
.../container/common/impl/HddsDispatcher.java | 6 +--
.../container/common/volume/StorageVolume.java | 10 ++---
.../ozone/container/common/volume/VolumeInfo.java | 8 ++--
.../ozone/container/common/volume/VolumeUsage.java | 40 ++++----------------
6 files changed, 64 insertions(+), 73 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java
index f3e54f375c..c25c0a40c5 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java
@@ -40,4 +40,47 @@ public interface SpaceUsageSource {
long getCapacity();
long getAvailable();
+
+ default SpaceUsageSource snapshot() {
+ return new Fixed(getCapacity(), getAvailable(), getUsedSpace());
+ }
+
+ SpaceUsageSource UNKNOWN = new Fixed(0, 0, 0);
+
+ /**
+ * A static source of space usage. Can be a point in time snapshot of a
+ * real volume usage, or can be used for testing.
+ */
+ final class Fixed implements SpaceUsageSource {
+
+ private final long capacity;
+ private final long available;
+ private final long used;
+
+ Fixed(long capacity, long available, long used) {
+ this.capacity = capacity;
+ this.available = available;
+ this.used = used;
+ }
+
+ @Override
+ public long getCapacity() {
+ return capacity;
+ }
+
+ @Override
+ public long getAvailable() {
+ return available;
+ }
+
+ @Override
+ public long getUsedSpace() {
+ return used;
+ }
+
+ @Override
+ public SpaceUsageSource snapshot() {
+ return this; // immutable
+ }
+ }
}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java
index 4055f080b9..26c861dc68 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java
@@ -36,35 +36,7 @@ public final class MockSpaceUsageSource {
public static SpaceUsageSource fixed(long capacity, long available,
long used) {
- return new Fixed(capacity, available, used);
- }
-
- private static final class Fixed implements SpaceUsageSource {
-
- private final long capacity;
- private final long available;
- private final long used;
-
- Fixed(long capacity, long available, long used) {
- this.capacity = capacity;
- this.available = available;
- this.used = used;
- }
-
- @Override
- public long getCapacity() {
- return capacity;
- }
-
- @Override
- public long getAvailable() {
- return available;
- }
-
- @Override
- public long getUsedSpace() {
- return used;
- }
+ return new SpaceUsageSource.Fixed(capacity, available, used);
}
private MockSpaceUsageSource() {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 1b0ef29c77..eeb3548d5f 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
@@ -78,7 +79,6 @@ import java.util.TreeMap;
import static
org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.malformedRequest;
import static
org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
import static
org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult;
-import static
org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace;
/**
* Ozone Container dispatcher takes a call from the netty server and routes it
@@ -585,8 +585,8 @@ public class HddsDispatcher implements ContainerDispatcher,
Auditor {
.orElse(Boolean.FALSE);
if (isOpen) {
HddsVolume volume = container.getContainerData().getVolume();
- PrecomputedVolumeSpace precomputedVolumeSpace =
- volume.getPrecomputedVolumeSpace();
+ SpaceUsageSource precomputedVolumeSpace =
+ volume.getCurrentUsage();
long volumeCapacity = precomputedVolumeSpace.getCapacity();
long volumeFreeSpaceToSpare =
VolumeUsage.getMinVolumeFreeSpace(conf, volumeCapacity);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
index fc329ebeb0..44ae1c0e79 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.hdfs.server.datanode.checker.Checkable;
import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
@@ -50,7 +51,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import static
org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion.getLatestVersion;
-import static
org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace;
/**
@@ -456,14 +456,14 @@ public abstract class StorageVolume
}
- public long getAvailable(PrecomputedVolumeSpace precomputedVolumeSpace) {
+ public long getAvailable(SpaceUsageSource precomputedVolumeSpace) {
return volumeInfo.map(info -> info.getAvailable(precomputedVolumeSpace))
.orElse(0L);
}
- public PrecomputedVolumeSpace getPrecomputedVolumeSpace() {
- return volumeInfo.map(VolumeInfo::getPrecomputedVolumeSpace)
- .orElse(new PrecomputedVolumeSpace(0L, 0L));
+ public SpaceUsageSource getCurrentUsage() {
+ return volumeInfo.map(VolumeInfo::getCurrentUsage)
+ .orElse(SpaceUsageSource.UNKNOWN);
}
public long getUsedSpace() {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
index 61c0b422c7..6ee35ba6b0 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -29,10 +29,10 @@ import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
import org.apache.hadoop.hdds.fs.SpaceUsageCheckParams;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static
org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT;
@@ -240,13 +240,13 @@ public final class VolumeInfo {
return Math.max(Math.min(avail, usage.getAvailable()), 0);
}
- public long getAvailable(PrecomputedVolumeSpace precomputedValues) {
+ public long getAvailable(SpaceUsageSource precomputedValues) {
long avail = precomputedValues.getCapacity() - usage.getUsedSpace();
return Math.max(Math.min(avail, usage.getAvailable(precomputedValues)), 0);
}
- public PrecomputedVolumeSpace getPrecomputedVolumeSpace() {
- return usage.getPrecomputedVolumeSpace();
+ public SpaceUsageSource getCurrentUsage() {
+ return usage.snapshot();
}
public void incrementUsedSpace(long usedSpace) {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
index 57cf0a8b9d..b2a66ba16b 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
@@ -65,7 +65,7 @@ public class VolumeUsage implements SpaceUsageSource {
return source.getAvailable() - getRemainingReserved();
}
- public long getAvailable(PrecomputedVolumeSpace precomputedVolumeSpace) {
+ public long getAvailable(SpaceUsageSource precomputedVolumeSpace) {
long available = precomputedVolumeSpace.getAvailable();
return available - getRemainingReserved(precomputedVolumeSpace);
}
@@ -75,6 +75,11 @@ public class VolumeUsage implements SpaceUsageSource {
return source.getUsedSpace();
}
+ @Override
+ public SpaceUsageSource snapshot() {
+ return source.snapshot();
+ }
+
public void incrementUsedSpace(long usedSpace) {
source.incrementUsedSpace(usedSpace);
}
@@ -94,7 +99,7 @@ public class VolumeUsage implements SpaceUsageSource {
return Math.max(totalUsed - source.getUsedSpace(), 0L);
}
- private long getOtherUsed(PrecomputedVolumeSpace precomputedVolumeSpace) {
+ private long getOtherUsed(SpaceUsageSource precomputedVolumeSpace) {
long totalUsed = precomputedVolumeSpace.getCapacity() -
precomputedVolumeSpace.getAvailable();
return Math.max(totalUsed - source.getUsedSpace(), 0L);
@@ -105,7 +110,7 @@ public class VolumeUsage implements SpaceUsageSource {
}
private long getRemainingReserved(
- PrecomputedVolumeSpace precomputedVolumeSpace) {
+ SpaceUsageSource precomputedVolumeSpace) {
return Math.max(reservedInBytes - getOtherUsed(precomputedVolumeSpace),
0L);
}
@@ -169,33 +174,4 @@ public class VolumeUsage implements SpaceUsageSource {
return (volumeAvailableSpace - volumeCommittedBytesCount) >
Math.max(requiredSpace, volumeFreeSpaceToSpare);
}
-
- /**
- * Class representing precomputed space values of a volume.
- * This class is intended to store precomputed values, such as capacity
- * and available space of a volume, to avoid recalculating these
- * values multiple times and to make method signatures simpler.
- */
- public static class PrecomputedVolumeSpace {
- private final long capacity;
- private final long available;
-
- public PrecomputedVolumeSpace(long capacity, long available) {
- this.capacity = capacity;
- this.available = available;
- }
-
- public long getCapacity() {
- return capacity;
- }
-
- public long getAvailable() {
- return available;
- }
- }
-
- public PrecomputedVolumeSpace getPrecomputedVolumeSpace() {
- return new PrecomputedVolumeSpace(source.getCapacity(),
- source.getAvailable());
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
