fapifta commented on a change in pull request #2910:
URL: https://github.com/apache/ozone/pull/2910#discussion_r768266757
##########
File path:
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MultiNodePipelineBlockAllocator.java
##########
@@ -21,68 +21,103 @@
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import java.util.ArrayList;
import java.util.List;
-import java.util.Random;
+import java.util.Set;
/**
* Allocates the block with required number of nodes in the pipeline.
*/
public class MultiNodePipelineBlockAllocator implements MockBlockAllocator {
- public static final Random RANDOM = new Random();
private long blockId;
private int requiredNodes;
private final ConfigurationSource conf;
+ private List<HddsProtos.DatanodeDetailsProto> clusterDns = new ArrayList<>();
+ private int start = 0;
public MultiNodePipelineBlockAllocator(OzoneConfiguration conf,
- int requiredNodes) {
+ int requiredNodes, int clusterSize) {
Review comment:
I think we should document a bit better how the block allocator works,
as the code for it starts to get a bit more large to make it just easy to
understand. It would be really useful to easily understand what is happening
here, and how this code interacts with the failure injection logic inside
MockXCieverClientFactory.
##########
File path:
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MultiNodePipelineBlockAllocator.java
##########
@@ -21,68 +21,103 @@
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import java.util.ArrayList;
import java.util.List;
-import java.util.Random;
+import java.util.Set;
/**
* Allocates the block with required number of nodes in the pipeline.
*/
public class MultiNodePipelineBlockAllocator implements MockBlockAllocator {
- public static final Random RANDOM = new Random();
private long blockId;
private int requiredNodes;
private final ConfigurationSource conf;
+ private List<HddsProtos.DatanodeDetailsProto> clusterDns = new ArrayList<>();
+ private int start = 0;
public MultiNodePipelineBlockAllocator(OzoneConfiguration conf,
- int requiredNodes) {
+ int requiredNodes, int clusterSize) {
this.requiredNodes = requiredNodes;
this.conf = conf;
+ for (int i = 0; i < clusterSize; i++) {
+ clusterDns.add(HddsProtos.DatanodeDetailsProto.newBuilder().setUuid128(
+ HddsProtos.UUID.newBuilder().setLeastSigBits(i).setMostSigBits(i)
+ .build()).setHostName("localhost").setIpAddress("1.2.3.4")
+ .addPorts(HddsProtos.Port.newBuilder().setName("RATIS").setValue(i)
+ .build()).build());
+ }
+ }
+
+ public List<HddsProtos.DatanodeDetailsProto> getClusterDns(){
+ return this.clusterDns;
}
@Override
public Iterable<? extends OzoneManagerProtocolProtos.KeyLocation>
- allocateBlock(OzoneManagerProtocolProtos.KeyArgs keyArgs) {
- HddsProtos.Pipeline.Builder builder =
- HddsProtos.Pipeline.newBuilder().setFactor(keyArgs.getFactor())
-
.setType(keyArgs.getType()).setId(HddsProtos.PipelineID.newBuilder()
- .setUuid128(HddsProtos.UUID.newBuilder().setLeastSigBits(1L)
- .setMostSigBits(1L).build()).build());
- final int rand = RANDOM.nextInt(); // used for port and UUID combination.
- // It's ok here for port number limit as don't really create any socket
- // connection.
- for (int i = 1; i <= requiredNodes; i++) {
- builder.addMembers(HddsProtos.DatanodeDetailsProto.newBuilder()
- .setUuid128(HddsProtos.UUID.newBuilder().setLeastSigBits(rand)
- .setMostSigBits(i).build()).setHostName("localhost")
- .setIpAddress("1.2.3.4").addPorts(
- HddsProtos.Port.newBuilder().setName("RATIS").setValue(rand)
- .build()).build());
- if (keyArgs.getType() == HddsProtos.ReplicationType.EC) {
- builder.addMemberReplicaIndexes(i);
- }
- }
- if (keyArgs.getType() == HddsProtos.ReplicationType.EC) {
- builder.setEcReplicationConfig(keyArgs.getEcReplicationConfig());
- }
- final HddsProtos.Pipeline pipeline = builder.build();
-
+ allocateBlock(OzoneManagerProtocolProtos.KeyArgs keyArgs,
+ ExcludeList excludeList) {
long blockSize = (long) conf
.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
-
+ long blockGroupLen = keyArgs.getEcReplicationConfig().getData() *
blockSize;
+ long dataSize = keyArgs.getDataSize();
List<OzoneManagerProtocolProtos.KeyLocation> results = new ArrayList<>();
- results.add(OzoneManagerProtocolProtos.KeyLocation.newBuilder()
- .setPipeline(pipeline).setBlockID(
- HddsProtos.BlockID.newBuilder().setBlockCommitSequenceId(1L)
- .setContainerBlockID(
- HddsProtos.ContainerBlockID.newBuilder().setContainerID(1L)
- .setLocalID(blockId++).build()).build()).setOffset(0L)
- .setLength(blockSize).build());
+ long numbBlkGroups = dataSize / blockGroupLen + 1;
Review comment:
As I understand the code, we are creating multiple block groups here,
and return it as preallocated blocks.
If I am right, then I would as if you think it would worth it to allocate
less block groupss then then data size, instead of a spare block groups, so
that our tests will test the code path where the client has to allocate a new
block and can not simply finish the write into pre-allocated ones.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
