[16/50] hadoop git commit: HDDS-238. Add Node2Pipeline Map in SCM to track ratis/standalone pipelines. Contributed by Mukul Kumar Singh.

2018-07-20 Thread zhangduo 
HDDS-238. Add Node2Pipeline Map in SCM to track ratis/standalone pipelines. 
Contributed by Mukul Kumar Singh.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f3f7222
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f3f7222
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f3f7222

Branch: refs/heads/HDFS-13572
Commit: 3f3f72221ffd11cc6bfa0e010e3c5b0e14911102
Parents: f89e265
Author: Xiaoyu Yao 
Authored: Thu Jul 12 22:02:57 2018 -0700
Committer: Xiaoyu Yao 
Committed: Thu Jul 12 22:14:03 2018 -0700

--
 .../container/common/helpers/ContainerInfo.java |  11 ++
 .../hdds/scm/container/ContainerMapping.java|  11 +-
 .../scm/container/ContainerStateManager.java|   6 +
 .../scm/container/states/ContainerStateMap.java |  36 +-
 .../hdds/scm/pipelines/Node2PipelineMap.java| 121 +++
 .../hdds/scm/pipelines/PipelineManager.java |  22 ++--
 .../hdds/scm/pipelines/PipelineSelector.java|  24 +++-
 .../scm/pipelines/ratis/RatisManagerImpl.java   |  11 +-
 .../standalone/StandaloneManagerImpl.java   |   7 +-
 .../hdds/scm/pipeline/TestNode2PipelineMap.java | 117 ++
 10 files changed, 343 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
--
diff --git 
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
 
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
index 9593717..4074b21 100644
--- 
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
+++ 
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
@@ -456,4 +456,15 @@ public class ContainerInfo implements 
Comparator,
   replicationFactor, replicationType);
 }
   }
+
+  /**
+   * Check if a container is in open state, this will check if the
+   * container is either open or allocated or creating. Any containers in
+   * these states is managed as an open container by SCM.
+   */
+  public boolean isContainerOpen() {
+return state == HddsProtos.LifeCycleState.ALLOCATED ||
+state == HddsProtos.LifeCycleState.CREATING ||
+state == HddsProtos.LifeCycleState.OPEN;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
--
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index abad32c..26f4d86 100644
--- 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -477,7 +477,7 @@ public class ContainerMapping implements Mapping {
 List
 containerInfos = reports.getReportsList();
 
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
 containerInfos) {
   byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
   lock.lock();
@@ -498,7 +498,9 @@ public class ContainerMapping implements Mapping {
   containerStore.put(dbKey, newState.toByteArray());
 
   // If the container is closed, then state is already written to SCM
-  Pipeline pipeline = 
pipelineSelector.getPipeline(newState.getPipelineName(), 
newState.getReplicationType());
+  Pipeline pipeline =
+  pipelineSelector.getPipeline(newState.getPipelineName(),
+  newState.getReplicationType());
   if(pipeline == null) {
 pipeline = pipelineSelector
 .getReplicationPipeline(newState.getReplicationType(),
@@ -713,4 +715,9 @@ public class ContainerMapping implements Mapping {
   public MetadataStore getContainerStore() {
 return containerStore;
   }
+
+  @VisibleForTesting
+  public PipelineSelector getPipelineSelector() {
+return pipelineSelector;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
--
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/containe

[09/32] hadoop git commit: HDDS-238. Add Node2Pipeline Map in SCM to track ratis/standalone pipelines. Contributed by Mukul Kumar Singh.

2018-07-17 Thread xkrogen 
HDDS-238. Add Node2Pipeline Map in SCM to track ratis/standalone pipelines. 
Contributed by Mukul Kumar Singh.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f3f7222
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f3f7222
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f3f7222

Branch: refs/heads/HDFS-12943
Commit: 3f3f72221ffd11cc6bfa0e010e3c5b0e14911102
Parents: f89e265
Author: Xiaoyu Yao 
Authored: Thu Jul 12 22:02:57 2018 -0700
Committer: Xiaoyu Yao 
Committed: Thu Jul 12 22:14:03 2018 -0700

--
 .../container/common/helpers/ContainerInfo.java |  11 ++
 .../hdds/scm/container/ContainerMapping.java|  11 +-
 .../scm/container/ContainerStateManager.java|   6 +
 .../scm/container/states/ContainerStateMap.java |  36 +-
 .../hdds/scm/pipelines/Node2PipelineMap.java| 121 +++
 .../hdds/scm/pipelines/PipelineManager.java |  22 ++--
 .../hdds/scm/pipelines/PipelineSelector.java|  24 +++-
 .../scm/pipelines/ratis/RatisManagerImpl.java   |  11 +-
 .../standalone/StandaloneManagerImpl.java   |   7 +-
 .../hdds/scm/pipeline/TestNode2PipelineMap.java | 117 ++
 10 files changed, 343 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
--
diff --git 
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
 
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
index 9593717..4074b21 100644
--- 
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
+++ 
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
@@ -456,4 +456,15 @@ public class ContainerInfo implements 
Comparator,
   replicationFactor, replicationType);
 }
   }
+
+  /**
+   * Check if a container is in open state, this will check if the
+   * container is either open or allocated or creating. Any containers in
+   * these states is managed as an open container by SCM.
+   */
+  public boolean isContainerOpen() {
+return state == HddsProtos.LifeCycleState.ALLOCATED ||
+state == HddsProtos.LifeCycleState.CREATING ||
+state == HddsProtos.LifeCycleState.OPEN;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
--
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index abad32c..26f4d86 100644
--- 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -477,7 +477,7 @@ public class ContainerMapping implements Mapping {
 List
 containerInfos = reports.getReportsList();
 
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
 containerInfos) {
   byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
   lock.lock();
@@ -498,7 +498,9 @@ public class ContainerMapping implements Mapping {
   containerStore.put(dbKey, newState.toByteArray());
 
   // If the container is closed, then state is already written to SCM
-  Pipeline pipeline = 
pipelineSelector.getPipeline(newState.getPipelineName(), 
newState.getReplicationType());
+  Pipeline pipeline =
+  pipelineSelector.getPipeline(newState.getPipelineName(),
+  newState.getReplicationType());
   if(pipeline == null) {
 pipeline = pipelineSelector
 .getReplicationPipeline(newState.getReplicationType(),
@@ -713,4 +715,9 @@ public class ContainerMapping implements Mapping {
   public MetadataStore getContainerStore() {
 return containerStore;
   }
+
+  @VisibleForTesting
+  public PipelineSelector getPipelineSelector() {
+return pipelineSelector;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
--
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/containe

[39/50] [abbrv] hadoop git commit: HDDS-238. Add Node2Pipeline Map in SCM to track ratis/standalone pipelines. Contributed by Mukul Kumar Singh.

2018-07-13 Thread botong 
HDDS-238. Add Node2Pipeline Map in SCM to track ratis/standalone pipelines. 
Contributed by Mukul Kumar Singh.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f3f7222
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f3f7222
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f3f7222

Branch: refs/heads/YARN-7402
Commit: 3f3f72221ffd11cc6bfa0e010e3c5b0e14911102
Parents: f89e265
Author: Xiaoyu Yao 
Authored: Thu Jul 12 22:02:57 2018 -0700
Committer: Xiaoyu Yao 
Committed: Thu Jul 12 22:14:03 2018 -0700

--
 .../container/common/helpers/ContainerInfo.java |  11 ++
 .../hdds/scm/container/ContainerMapping.java|  11 +-
 .../scm/container/ContainerStateManager.java|   6 +
 .../scm/container/states/ContainerStateMap.java |  36 +-
 .../hdds/scm/pipelines/Node2PipelineMap.java| 121 +++
 .../hdds/scm/pipelines/PipelineManager.java |  22 ++--
 .../hdds/scm/pipelines/PipelineSelector.java|  24 +++-
 .../scm/pipelines/ratis/RatisManagerImpl.java   |  11 +-
 .../standalone/StandaloneManagerImpl.java   |   7 +-
 .../hdds/scm/pipeline/TestNode2PipelineMap.java | 117 ++
 10 files changed, 343 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
--
diff --git 
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
 
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
index 9593717..4074b21 100644
--- 
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
+++ 
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
@@ -456,4 +456,15 @@ public class ContainerInfo implements 
Comparator,
   replicationFactor, replicationType);
 }
   }
+
+  /**
+   * Check if a container is in open state, this will check if the
+   * container is either open or allocated or creating. Any containers in
+   * these states is managed as an open container by SCM.
+   */
+  public boolean isContainerOpen() {
+return state == HddsProtos.LifeCycleState.ALLOCATED ||
+state == HddsProtos.LifeCycleState.CREATING ||
+state == HddsProtos.LifeCycleState.OPEN;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
--
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index abad32c..26f4d86 100644
--- 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -477,7 +477,7 @@ public class ContainerMapping implements Mapping {
 List
 containerInfos = reports.getReportsList();
 
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
 containerInfos) {
   byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
   lock.lock();
@@ -498,7 +498,9 @@ public class ContainerMapping implements Mapping {
   containerStore.put(dbKey, newState.toByteArray());
 
   // If the container is closed, then state is already written to SCM
-  Pipeline pipeline = 
pipelineSelector.getPipeline(newState.getPipelineName(), 
newState.getReplicationType());
+  Pipeline pipeline =
+  pipelineSelector.getPipeline(newState.getPipelineName(),
+  newState.getReplicationType());
   if(pipeline == null) {
 pipeline = pipelineSelector
 .getReplicationPipeline(newState.getReplicationType(),
@@ -713,4 +715,9 @@ public class ContainerMapping implements Mapping {
   public MetadataStore getContainerStore() {
 return containerStore;
   }
+
+  @VisibleForTesting
+  public PipelineSelector getPipelineSelector() {
+return pipelineSelector;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
--
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container

hadoop git commit: HDDS-238. Add Node2Pipeline Map in SCM to track ratis/standalone pipelines. Contributed by Mukul Kumar Singh.

2018-07-12 Thread xyao 
Repository: hadoop
Updated Branches:
  refs/heads/trunk f89e26590 -> 3f3f72221


HDDS-238. Add Node2Pipeline Map in SCM to track ratis/standalone pipelines. 
Contributed by Mukul Kumar Singh.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f3f7222
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f3f7222
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f3f7222

Branch: refs/heads/trunk
Commit: 3f3f72221ffd11cc6bfa0e010e3c5b0e14911102
Parents: f89e265
Author: Xiaoyu Yao 
Authored: Thu Jul 12 22:02:57 2018 -0700
Committer: Xiaoyu Yao 
Committed: Thu Jul 12 22:14:03 2018 -0700

--
 .../container/common/helpers/ContainerInfo.java |  11 ++
 .../hdds/scm/container/ContainerMapping.java|  11 +-
 .../scm/container/ContainerStateManager.java|   6 +
 .../scm/container/states/ContainerStateMap.java |  36 +-
 .../hdds/scm/pipelines/Node2PipelineMap.java| 121 +++
 .../hdds/scm/pipelines/PipelineManager.java |  22 ++--
 .../hdds/scm/pipelines/PipelineSelector.java|  24 +++-
 .../scm/pipelines/ratis/RatisManagerImpl.java   |  11 +-
 .../standalone/StandaloneManagerImpl.java   |   7 +-
 .../hdds/scm/pipeline/TestNode2PipelineMap.java | 117 ++
 10 files changed, 343 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
--
diff --git 
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
 
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
index 9593717..4074b21 100644
--- 
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
+++ 
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
@@ -456,4 +456,15 @@ public class ContainerInfo implements 
Comparator,
   replicationFactor, replicationType);
 }
   }
+
+  /**
+   * Check if a container is in open state, this will check if the
+   * container is either open or allocated or creating. Any containers in
+   * these states is managed as an open container by SCM.
+   */
+  public boolean isContainerOpen() {
+return state == HddsProtos.LifeCycleState.ALLOCATED ||
+state == HddsProtos.LifeCycleState.CREATING ||
+state == HddsProtos.LifeCycleState.OPEN;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
--
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index abad32c..26f4d86 100644
--- 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -477,7 +477,7 @@ public class ContainerMapping implements Mapping {
 List
 containerInfos = reports.getReportsList();
 
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
 containerInfos) {
   byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
   lock.lock();
@@ -498,7 +498,9 @@ public class ContainerMapping implements Mapping {
   containerStore.put(dbKey, newState.toByteArray());
 
   // If the container is closed, then state is already written to SCM
-  Pipeline pipeline = 
pipelineSelector.getPipeline(newState.getPipelineName(), 
newState.getReplicationType());
+  Pipeline pipeline =
+  pipelineSelector.getPipeline(newState.getPipelineName(),
+  newState.getReplicationType());
   if(pipeline == null) {
 pipeline = pipelineSelector
 .getReplicationPipeline(newState.getReplicationType(),
@@ -713,4 +715,9 @@ public class ContainerMapping implements Mapping {
   public MetadataStore getContainerStore() {
 return containerStore;
   }
+
+  @VisibleForTesting
+  public PipelineSelector getPipelineSelector() {
+return pipelineSelector;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f3f7222/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
--
diff --git