[hbase] branch branch-2 updated: HBASE-23737 [Flakey Tests] TestFavoredNodeTableImport fails 30% of the time; AMENDMENT This is actual fix; previous added debug to test.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new df03306 HBASE-23737 [Flakey Tests] TestFavoredNodeTableImport fails
30% of the time; AMENDMENT This is actual fix; previous added debug to test.
df03306 is described below
commit df033067feccfc73ca46e7b47732c18c024605c0
Author: stack
AuthorDate: Sat Jan 25 17:01:10 2020 -0800
HBASE-23737 [Flakey Tests] TestFavoredNodeTableImport fails 30% of the
time; AMENDMENT
This is actual fix; previous added debug to test.
---
.../hadoop/hbase/favored/FavoredNodesManager.java | 57 +-
.../hadoop/hbase/favored/FavoredNodesPlan.java | 42 +---
.../hbase/master/RegionPlacementMaintainer.java| 2 +-
3 files changed, 50 insertions(+), 51 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java
index 67d4071..77334db 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java
@@ -1,5 +1,4 @@
-/**
- *
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -23,7 +22,6 @@ import static
org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper.FAVORE
import static
org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.PRIMARY;
import static
org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.SECONDARY;
import static
org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.TERTIARY;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -31,9 +29,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.master.MasterServices;
@@ -56,22 +53,21 @@ import
org.apache.hbase.thirdparty.com.google.common.collect.Sets;
* should be done through this class. There should only be one instance of
* {@link FavoredNodesManager} in Master. {@link FavoredNodesPlan} and favored
node information
* from {@link SnapshotOfRegionAssignmentFromMeta} should not be used outside
this class (except
- * for may be tools that only read or test cases). All other classes including
Favored balancers
+ * for tools that only read or fortest cases). All other classes including
Favored balancers
* and {@link FavoredNodeAssignmentHelper} should use {@link
FavoredNodesManager} for any
* read/write/deletes to favored nodes.
*/
@InterfaceAudience.Private
public class FavoredNodesManager {
-
private static final Logger LOG =
LoggerFactory.getLogger(FavoredNodesManager.class);
- private FavoredNodesPlan globalFavoredNodesAssignmentPlan;
- private Map> primaryRSToRegionMap;
- private Map> secondaryRSToRegionMap;
- private Map> teritiaryRSToRegionMap;
+ private final FavoredNodesPlan globalFavoredNodesAssignmentPlan;
+ private final Map> primaryRSToRegionMap;
+ private final Map> secondaryRSToRegionMap;
+ private final Map> teritiaryRSToRegionMap;
- private MasterServices masterServices;
- private RackManager rackManager;
+ private final MasterServices masterServices;
+ private final RackManager rackManager;
/**
* Datanode port to be used for Favored Nodes.
@@ -87,15 +83,19 @@ public class FavoredNodesManager {
this.rackManager = new RackManager(masterServices.getConfiguration());
}
- public void initialize(SnapshotOfRegionAssignmentFromMeta
snapshotOfRegionAssignment)
- throws HBaseIOException {
-globalFavoredNodesAssignmentPlan =
snapshotOfRegionAssignment.getExistingAssignmentPlan();
-primaryRSToRegionMap =
snapshotOfRegionAssignment.getPrimaryToRegionInfoMap();
-secondaryRSToRegionMap =
snapshotOfRegionAssignment.getSecondaryToRegionInfoMap();
-teritiaryRSToRegionMap =
snapshotOfRegionAssignment.getTertiaryToRegionInfoMap();
-datanodeDataTransferPort = getDataNodePort();
+ public synchronized void initialize(SnapshotOfRegionAssignmentFromMeta
snapshot) {
+// Add snapshot to structures made on creation. Current structures may
have picked
+// up data between construction and the scan of meta needed before this
method
+// is called. See HBASE-23737 "[Flakey Tests] TestFavoredNodeTableImport
fails 30% of the time"
+this.globalFavoredNodesAssignmentPlan.
+
[hbase] branch branch-2 updated: HBASE-23737 [Flakey Tests] TestFavoredNodeTableImport fails 30% of the time
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 9cf57a7 HBASE-23737 [Flakey Tests] TestFavoredNodeTableImport fails
30% of the time
9cf57a7 is described below
commit 9cf57a7db6217151fab9344e6eade4d0843bd405
Author: stack
AuthorDate: Fri Jan 24 17:56:42 2020 -0800
HBASE-23737 [Flakey Tests] TestFavoredNodeTableImport fails 30% of the time
---
.../balancer/TestFavoredNodeTableImport.java | 27 +++---
1 file changed, 19 insertions(+), 8 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodeTableImport.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodeTableImport.java
index 6958ed2..29f0708 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodeTableImport.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodeTableImport.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.balancer;
import static
org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper.FAVORED_NODES_NUM;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import java.util.List;
import java.util.Set;
@@ -34,11 +35,13 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.favored.FavoredNodesManager;
+import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.After;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
@@ -66,7 +69,6 @@ public class TestFavoredNodeTableImport {
@After
public void stopCluster() throws Exception {
-UTIL.cleanupTestDir();
UTIL.shutdownMiniCluster();
}
@@ -81,13 +83,14 @@ public class TestFavoredNodeTableImport {
Threads.sleep(1);
}
Admin admin = UTIL.getAdmin();
-admin.setBalancerRunning(false, true);
+admin.balancerSwitch(false, true);
String tableName = "testFNImport";
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, Bytes.toBytes("a"), Bytes.toBytes("z"),
REGION_NUM);
UTIL.waitTableAvailable(desc.getTableName());
+admin.balancerSwitch(true, true);
LOG.info("Shutting down cluster");
UTIL.shutdownMiniHBaseCluster();
@@ -97,18 +100,26 @@ public class TestFavoredNodeTableImport {
UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
FavoredStochasticBalancer.class.getName());
UTIL.restartHBaseCluster(SLAVES);
-while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
+HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+while (!master.isInitialized()) {
Threads.sleep(1);
}
-admin = UTIL.getAdmin();
-
UTIL.waitTableAvailable(desc.getTableName());
+UTIL.waitUntilNoRegionsInTransition(1);
+assertTrue(master.isBalancerOn());
-FavoredNodesManager fnm =
UTIL.getHBaseCluster().getMaster().getFavoredNodesManager();
+FavoredNodesManager fnm = master.getFavoredNodesManager();
+assertNotNull(fnm);
+admin = UTIL.getAdmin();
List regionsOfTable =
admin.getTableRegions(TableName.valueOf(tableName));
for (HRegionInfo rInfo : regionsOfTable) {
- Set favNodes = Sets.newHashSet(fnm.getFavoredNodes(rInfo));
+ assertNotNull(rInfo);
+ assertNotNull(fnm);
+ List fns = fnm.getFavoredNodes(rInfo);
+ LOG.info("FNS {} {}", rInfo, fns);
+ assertNotNull(rInfo.toString(), fns);
+ Set favNodes = Sets.newHashSet(fns);
assertNotNull(favNodes);
assertEquals("Required no of favored nodes not found.",
FAVORED_NODES_NUM, favNodes.size());
for (ServerName fn : favNodes) {
