hbase git commit: HBASE-20741 Split of a region with replicas creates all daughter regions and its replica in same server (Ram)
Repository: hbase
Updated Branches:
refs/heads/branch-2.1 b5f07e0e9 -> 2051b0982
HBASE-20741 Split of a region with replicas creates all daughter regions
and its replica in same server (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2051b098
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2051b098
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2051b098
Branch: refs/heads/branch-2.1
Commit: 2051b0982ddbf575f88d7ca20b9960b43bb3da30
Parents: b5f07e0
Author: Vasudevan
Authored: Thu Sep 6 16:43:50 2018 +0530
Committer: Vasudevan
Committed: Thu Sep 6 16:44:59 2018 +0530
--
.../master/assignment/AssignmentManager.java| 21 ++-
.../assignment/MergeTableRegionsProcedure.java | 45 --
.../assignment/SplitTableRegionProcedure.java | 46 --
.../hbase/master/balancer/BaseLoadBalancer.java | 21 ++-
.../assignment/TestRegionReplicaSplit.java | 157 +++
5 files changed, 260 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2051b098/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 70a9680..f0a723d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -566,6 +566,17 @@ public class AssignmentManager implements ServerListener {
return waitForAssignment(regionInfo, Long.MAX_VALUE);
}
+ /**
+ * Create round-robin assigns. Use on table creation to distribute out
regions across cluster.
+ * @return AssignProcedures made out of the passed in hris and
a call to the balancer
+ * to populate the assigns with targets chosen using round-robin
(default balancer
+ * scheme). If at assign-time, the target chosen is no longer up,
thats fine, the
+ * AssignProcedure will ask the balancer for a new target, and so on.
+ */
+ public AssignProcedure[] createRoundRobinAssignProcedures(List
hris) {
+return createRoundRobinAssignProcedures(hris, null);
+ }
+
@VisibleForTesting
// TODO: Remove this?
public boolean waitForAssignment(final RegionInfo regionInfo, final long
timeout)
@@ -609,15 +620,21 @@ public class AssignmentManager implements ServerListener {
* balancer scheme). If at assign-time, the target chosen is no longer up,
thats fine,
* the AssignProcedure will ask the balancer for a new target, and so on.
*/
- public AssignProcedure[] createRoundRobinAssignProcedures(final
List hris) {
+ public AssignProcedure[] createRoundRobinAssignProcedures(final
List hris,
+ List serversToExclude) {
if (hris.isEmpty()) {
return null;
}
+if (serversToExclude != null
+&& this.master.getServerManager().getOnlineServersList().size() == 1) {
+ LOG.debug("Only one region server found and hence going ahead with the
assignment");
+ serversToExclude = null;
+}
try {
// Ask the balancer to assign our regions. Pass the regions en masse.
The balancer can do
// a better job if it has all the assignments in the one lump.
Map> assignments =
getBalancer().roundRobinAssignment(hris,
- this.master.getServerManager().createDestinationServersList(null));
+
this.master.getServerManager().createDestinationServersList(serversToExclude));
// Return mid-method!
return createAssignProcedures(assignments, hris.size());
} catch (HBaseIOException hioe) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2051b098/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 20ae444..9296222 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -682,16 +683,38 @@ public class
hbase git commit: HBASE-20741 Split of a region with replicas creates all daughter regions and its replica in same server (Ram)
Repository: hbase
Updated Branches:
refs/heads/branch-2 0409c54ba -> 1a5aedab3
HBASE-20741 Split of a region with replicas creates all daughter regions
and its replica in same server (Ram)
Signed-off-by: Huaxiang Sun, Michael Stack, Duo Zhang
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1a5aedab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1a5aedab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1a5aedab
Branch: refs/heads/branch-2
Commit: 1a5aedab3428abf6a2d66fec342a99177562a87d
Parents: 0409c54
Author: Vasudevan
Authored: Tue Sep 4 16:26:11 2018 +0530
Committer: Vasudevan
Committed: Tue Sep 4 16:26:11 2018 +0530
--
.../master/assignment/AssignmentManager.java| 23 ++-
.../assignment/AssignmentManagerUtil.java | 69 +---
.../assignment/MergeTableRegionsProcedure.java | 5 +-
.../assignment/SplitTableRegionProcedure.java | 14 +-
.../hbase/master/balancer/BaseLoadBalancer.java | 21 ++-
.../assignment/TestRegionReplicaSplit.java | 158 +++
6 files changed, 255 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1a5aedab/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 745703f..cf75d73 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -663,16 +663,22 @@ public class AssignmentManager implements ServerListener {
* scheme). If at assign-time, the target chosen is no longer up,
thats fine, the
* AssignProcedure will ask the balancer for a new target, and so on.
*/
- public TransitRegionStateProcedure[] createRoundRobinAssignProcedures(
- List hris) {
+ public TransitRegionStateProcedure[]
createRoundRobinAssignProcedures(List hris,
+ List serversToExclude) {
if (hris.isEmpty()) {
return new TransitRegionStateProcedure[0];
}
+
+if (serversToExclude != null
+&& this.master.getServerManager().getOnlineServersList().size() == 1) {
+ LOG.debug("Only one region server found and hence going ahead with the
assignment");
+ serversToExclude = null;
+}
try {
// Ask the balancer to assign our regions. Pass the regions en masse.
The balancer can do
// a better job if it has all the assignments in the one lump.
Map> assignments =
getBalancer().roundRobinAssignment(hris,
-this.master.getServerManager().createDestinationServersList(null));
+
this.master.getServerManager().createDestinationServersList(serversToExclude));
// Return mid-method!
return createAssignProcedures(assignments);
} catch (HBaseIOException hioe) {
@@ -682,6 +688,17 @@ public class AssignmentManager implements ServerListener {
return createAssignProcedures(hris);
}
+ /**
+ * Create round-robin assigns. Use on table creation to distribute out
regions across cluster.
+ * @return AssignProcedures made out of the passed in hris and
a call to the balancer
+ * to populate the assigns with targets chosen using round-robin
(default balancer
+ * scheme). If at assign-time, the target chosen is no longer up,
thats fine, the
+ * AssignProcedure will ask the balancer for a new target, and so on.
+ */
+ public TransitRegionStateProcedure[]
createRoundRobinAssignProcedures(List hris) {
+return createRoundRobinAssignProcedures(hris, null);
+ }
+
@VisibleForTesting
static int compare(TransitRegionStateProcedure left,
TransitRegionStateProcedure right) {
if (left.getRegion().isMetaRegion()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/1a5aedab/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
index d0dca09..7f2d11a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
@@ -18,12 +18,15 @@
package org.apache.hadoop.hbase.master.assignment;
import java.io.IOException;
+import
hbase git commit: HBASE-20741 Split of a region with replicas creates all daughter regions and its replica in same server (Ram)
Repository: hbase
Updated Branches:
refs/heads/master e4c4035ed -> 83131b1ac
HBASE-20741 Split of a region with replicas creates all daughter regions
and its replica in same server (Ram)
Signed-off-by: Huaxiang Sun, Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/83131b1a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/83131b1a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/83131b1a
Branch: refs/heads/master
Commit: 83131b1ac40233f6f76c23b2a15e4fa07b4ee020
Parents: e4c4035
Author: Vasudevan
Authored: Tue Sep 4 11:18:51 2018 +0530
Committer: Vasudevan
Committed: Tue Sep 4 11:18:51 2018 +0530
--
.../master/assignment/AssignmentManager.java| 23 ++-
.../assignment/AssignmentManagerUtil.java | 69 +---
.../assignment/MergeTableRegionsProcedure.java | 5 +-
.../assignment/SplitTableRegionProcedure.java | 14 ++--
.../hbase/master/balancer/BaseLoadBalancer.java | 21 +-
5 files changed, 97 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/83131b1a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 745703f..cf75d73 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -663,16 +663,22 @@ public class AssignmentManager implements ServerListener {
* scheme). If at assign-time, the target chosen is no longer up,
thats fine, the
* AssignProcedure will ask the balancer for a new target, and so on.
*/
- public TransitRegionStateProcedure[] createRoundRobinAssignProcedures(
- List hris) {
+ public TransitRegionStateProcedure[]
createRoundRobinAssignProcedures(List hris,
+ List serversToExclude) {
if (hris.isEmpty()) {
return new TransitRegionStateProcedure[0];
}
+
+if (serversToExclude != null
+&& this.master.getServerManager().getOnlineServersList().size() == 1) {
+ LOG.debug("Only one region server found and hence going ahead with the
assignment");
+ serversToExclude = null;
+}
try {
// Ask the balancer to assign our regions. Pass the regions en masse.
The balancer can do
// a better job if it has all the assignments in the one lump.
Map> assignments =
getBalancer().roundRobinAssignment(hris,
-this.master.getServerManager().createDestinationServersList(null));
+
this.master.getServerManager().createDestinationServersList(serversToExclude));
// Return mid-method!
return createAssignProcedures(assignments);
} catch (HBaseIOException hioe) {
@@ -682,6 +688,17 @@ public class AssignmentManager implements ServerListener {
return createAssignProcedures(hris);
}
+ /**
+ * Create round-robin assigns. Use on table creation to distribute out
regions across cluster.
+ * @return AssignProcedures made out of the passed in hris and
a call to the balancer
+ * to populate the assigns with targets chosen using round-robin
(default balancer
+ * scheme). If at assign-time, the target chosen is no longer up,
thats fine, the
+ * AssignProcedure will ask the balancer for a new target, and so on.
+ */
+ public TransitRegionStateProcedure[]
createRoundRobinAssignProcedures(List hris) {
+return createRoundRobinAssignProcedures(hris, null);
+ }
+
@VisibleForTesting
static int compare(TransitRegionStateProcedure left,
TransitRegionStateProcedure right) {
if (left.getRegion().isMetaRegion()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/83131b1a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
index d0dca09..77e3a97 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
@@ -18,12 +18,15 @@
package org.apache.hadoop.hbase.master.assignment;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import
hbase git commit: HBASE-20741 Split of a region with replicas creates all daughter regions and its replica in same server (Ram)
Repository: hbase
Updated Branches:
refs/heads/master 245bb4470 -> e4c4035ed
HBASE-20741 Split of a region with replicas creates all daughter regions
and its replica in same server (Ram)
Signed-off-by: Huaxiang Sun, Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e4c4035e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e4c4035e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e4c4035e
Branch: refs/heads/master
Commit: e4c4035ed88a031d3dcecfdf4578205fd37b0a73
Parents: 245bb44
Author: Vasudevan
Authored: Tue Sep 4 11:08:03 2018 +0530
Committer: Vasudevan
Committed: Tue Sep 4 11:08:03 2018 +0530
--
.../assignment/TestRegionReplicaSplit.java | 163 +++
1 file changed, 163 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e4c4035e/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionReplicaSplit.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionReplicaSplit.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionReplicaSplit.java
new file mode 100644
index 000..315ef83
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionReplicaSplit.java
@@ -0,0 +1,163 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.assignment;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.RegionSplitter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category({ RegionServerTests.class, LargeTests.class })
+public class TestRegionReplicaSplit {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRegionReplicaSplit.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(TestRegionReplicaSplit.class);
+
+ private static final int NB_SERVERS = 4;
+ private static Table table;
+
+ private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
+ private static final byte[] f = HConstants.CATALOG_FAMILY;
+
+ @BeforeClass
+ public static void before() throws Exception {
+
HTU.getConfiguration().setInt("hbase.master.wait.on.regionservers.mintostart",
3);
+HTU.startMiniCluster(NB_SERVERS);
+final TableName tableName =
TableName.valueOf(TestRegionReplicaSplit.class.getSimpleName());
+
+// Create table then get the single region for our new table.
+createTableDirectlyFromHTD(tableName);
+ }
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static void createTableDirectlyFromHTD(final TableName tableName)
throws IOException {
+HTableDescriptor htd = new HTableDescriptor(tableName);
+
