szetszwo commented on code in PR #12061:
URL: https://github.com/apache/ozone/pull/12061#discussion_r4654917412


##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ChainedPipelineProvider.java:
##########
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.hdds.client.ChainedReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+
+/**
+ * Implements Api for creating CHAINED (V3) pipelines.
+ * Only factor THREE is supported.
+ * All the datanodes in a CHAINED pipeline must have the
+ * {@link DatanodeDetails.Port.Name#RATIS_DATASTREAM} port.
+ */
+public class ChainedPipelineProvider extends 
NCopyPipelineProvider<ChainedReplicationConfig> {
+
+  public ChainedPipelineProvider(NodeManager nodeManager,
+                                 PipelineStateManager stateManager,
+                                 ConfigurationSource conf,
+                                 EventPublisher eventPublisher,
+                                 SCMContext scmContext) {
+    super(nodeManager, stateManager, conf, eventPublisher, scmContext);
+  }
+
+  @Override
+  public synchronized Pipeline create(ChainedReplicationConfig 
replicationConfig,
+      List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes)
+      throws IOException {
+    final ReplicationFactor factor = replicationConfig.getReplicationFactor();
+    if (factor != ReplicationFactor.THREE) {
+      throw new IllegalArgumentException("Unsupported factor " + factor + " 
for " + replicationConfig);
+    }

Review Comment:
   We should allow ReplicationFactor.ONE also.  Then V3 pipelines can 
completely replace V1/V2 pipelines in the future.
   



##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/NCopyPipelineProvider.java:
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import static 
org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS_DATASTREAM;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import org.apache.hadoop.hdds.client.ReplicatedReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import 
org.apache.hadoop.hdds.scm.pipeline.leader.choose.algorithms.LeaderChoosePolicy;
+import 
org.apache.hadoop.hdds.scm.pipeline.leader.choose.algorithms.LeaderChoosePolicyFactory;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand;
+import org.apache.ratis.protocol.exceptions.NotLeaderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for providers of pipelines with N copies, such as RATIS and 
CHAINED pipelines.
+ * The datanodes are chosen by {@link PipelinePlacementPolicy},
+ * a leader is suggested by {@link LeaderChoosePolicy}
+ * and the pipelines are created and closed on the datanodes using
+ * {@link CreatePipelineCommand} and {@link ClosePipelineCommand}.
+ */
+public abstract class NCopyPipelineProvider<C extends 
ReplicatedReplicationConfig> extends PipelineProvider<C> {
+
+  private static final Logger LOG = 
LoggerFactory.getLogger(NCopyPipelineProvider.class);
+
+  private final EventPublisher eventPublisher;
+  private final PlacementPolicy placementPolicy;
+  private final LeaderChoosePolicy leaderChoosePolicy;
+  private final SCMContext scmContext;
+  private final long containerSizeBytes;
+  private final long minRatisVolumeSizeBytes;
+
+  protected NCopyPipelineProvider(NodeManager nodeManager,
+                                  PipelineStateManager stateManager,
+                                  ConfigurationSource conf,
+                                  EventPublisher eventPublisher,
+                                  SCMContext scmContext) {
+    super(nodeManager, stateManager);
+    this.eventPublisher = eventPublisher;
+    this.scmContext = scmContext;
+    this.placementPolicy = PipelinePlacementPolicyFactory
+        .getPolicy(nodeManager, stateManager, conf);
+    this.containerSizeBytes = (long) conf.getStorageSize(
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
+        StorageUnit.BYTES);
+    this.minRatisVolumeSizeBytes = (long) conf.getStorageSize(
+        ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
+        ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT,
+        StorageUnit.BYTES);
+    try {
+      leaderChoosePolicy = LeaderChoosePolicyFactory
+          .getPolicy(conf, nodeManager, stateManager);
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @VisibleForTesting
+  public LeaderChoosePolicy getLeaderChoosePolicy() {
+    return leaderChoosePolicy;
+  }
+
+  long getContainerSizeBytes() {
+    return containerSizeBytes;
+  }
+
+  long getMinRatisVolumeSizeBytes() {
+    return minRatisVolumeSizeBytes;
+  }
+
+  @Override
+  public synchronized Pipeline create(C replicationConfig) throws IOException {
+    return create(replicationConfig, Collections.emptyList(), 
Collections.emptyList());
+  }
+
+  @Override
+  public Pipeline create(C replicationConfig, List<DatanodeDetails> nodes) {
+    return newPipelineBuilder(replicationConfig, nodes)
+        .setId(PipelineID.randomId())
+        .build();
+  }
+
+  @Override
+  public Pipeline createForRead(C replicationConfig, Set<ContainerReplica> 
replicas) {
+    // Use insecureRandomId for throwaway read pipeline IDs to avoid
+    // contention on the shared SecureRandom instance.
+    return newPipelineBuilder(replicationConfig, 
ContainerReplica.toDatanodeDetailsList(replicas))
+        .setId(PipelineID.insecureRandomId())
+        .build();
+  }
+
+  /**
+   * Build a new pipeline with the given datanodes and a suggested leader.
+   * Then, send {@link CreatePipelineCommand} to each datanode.
+   */
+  Pipeline createPipeline(C replicationConfig, List<DatanodeDetails> dns) 
throws NotLeaderException {
+    DatanodeDetails suggestedLeader = leaderChoosePolicy.chooseLeader(dns);
+
+    Pipeline pipeline = newPipelineBuilder(replicationConfig, dns)
+        .setId(PipelineID.randomId())
+        .setSuggestedLeaderId(suggestedLeader != null ? 
suggestedLeader.getID() : null)
+        .build();
+
+    // Send command to datanodes to create pipeline
+    final ReplicationFactor factor = replicationConfig.getReplicationFactor();
+    final CreatePipelineCommand createCommand = new CreatePipelineCommand(
+        pipeline.getId(), pipeline.getType(), factor, dns, suggestedLeader);
+    createCommand.setTerm(scmContext.getTermOfLeader());
+
+    dns.forEach(node -> {
+      LOG.info("Sending CreatePipelineCommand for pipeline:{} to datanode:{}",
+          pipeline.getId(), node);
+      eventPublisher.fireEvent(SCMEvents.DATANODE_COMMAND,
+          new CommandForDatanode<>(node, createCommand));
+    });
+
+    return pipeline;
+  }
+
+  /**
+   * Choose datanodes for a factor THREE pipeline,
+   * excluding the datanodes which have reached their pipeline limit.
+   *
+   * @param excludeNonStreamNodes Should the datanodes without the {@link 
DatanodeDetails.Port.Name#RATIS_DATASTREAM}
+   *                              port be excluded?
+   * @param fallback When excludeNonStreamNodes is true, should it fall back 
to choose from all the datanodes
+   *                 if there are not enough datanodes with the stream port?
+   */
+  List<DatanodeDetails> chooseThreeFactorDatanodes(
+      List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes, 
int requiredNode,
+      boolean excludeNonStreamNodes, boolean fallback)
+      throws IOException {
+    final NodeManager nodeManager = getNodeManager();
+    final PipelineStateManager stateManager = getPipelineStateManager();
+    final List<DatanodeDetails> healthyNodes = 
nodeManager.getNodes(NodeStatus.inServiceHealthy());
+    excludedNodes = excludedNodes.isEmpty() ? null : excludedNodes;
+    List<DatanodeDetails> additionalExcludedNodes = null;
+    for (DatanodeDetails d : healthyNodes) {
+      final int count = 
PipelinePlacementPolicy.currentRatisThreePipelineCount(nodeManager, 
stateManager, d);
+      if (count >= nodeManager.pipelineLimit(d)) {

Review Comment:
   - currentRatisThreePipelineCount filtered non-RATIS pipeline in 
isNonClosedRatisThreePipeline.  So CHAINED pipelines are not counted.
   - We need separated pipelineLimit for RATIS and CHAINED.



##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ChainedPipelineProvider.java:
##########
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.hdds.client.ChainedReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+
+/**
+ * Implements Api for creating CHAINED (V3) pipelines.
+ * Only factor THREE is supported.
+ * All the datanodes in a CHAINED pipeline must have the
+ * {@link DatanodeDetails.Port.Name#RATIS_DATASTREAM} port.
+ */
+public class ChainedPipelineProvider extends 
NCopyPipelineProvider<ChainedReplicationConfig> {
+
+  public ChainedPipelineProvider(NodeManager nodeManager,
+                                 PipelineStateManager stateManager,
+                                 ConfigurationSource conf,
+                                 EventPublisher eventPublisher,
+                                 SCMContext scmContext) {
+    super(nodeManager, stateManager, conf, eventPublisher, scmContext);
+  }
+
+  @Override
+  public synchronized Pipeline create(ChainedReplicationConfig 
replicationConfig,
+      List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes)
+      throws IOException {

Review Comment:
   Check HDDSVersion.STREAM_PUTBLOCK_WITHOUT_RAFT and disallow CHAINED pipeline 
before upgrade finalized.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to