This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/main by this push:
     new 05abf9f  Fix #1791 - Fix SuspendedTabletsIT (#1888)
05abf9f is described below

commit 05abf9fefbb9de46f7d27b6f93d1a6a839dc6963
Author: Dom G <47725857+domgargu...@users.noreply.github.com>
AuthorDate: Thu Feb 25 13:35:09 2021 -0500

    Fix #1791 - Fix SuspendedTabletsIT (#1888)
    
    * ensure metadata table tablets are on a single server
    * removed server hosting metadata tablets from list to shutdown
    * pre-split tables at creation time instead of immediately after
---
 .../accumulo/test/manager/SuspendedTabletsIT.java  | 71 +++++++++++++---------
 1 file changed, 42 insertions(+), 29 deletions(-)

diff --git 
a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java 
b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
index 92f6401..770648e 100644
--- 
a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
+++ 
b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
@@ -19,10 +19,8 @@
 package org.apache.accumulo.test.manager;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.security.SecureRandom;
@@ -45,8 +43,10 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.accumulo.core.client.Accumulo;
 import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
 import org.apache.accumulo.core.clientImpl.ClientContext;
 import org.apache.accumulo.core.clientImpl.ManagerClient;
+import org.apache.accumulo.core.clientImpl.TabletLocator;
 import org.apache.accumulo.core.conf.ClientProperty;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Range;
@@ -54,6 +54,7 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
 import org.apache.accumulo.core.metadata.MetadataTable;
 import org.apache.accumulo.core.metadata.TServerInstance;
 import org.apache.accumulo.core.metadata.TabletLocationState;
+import org.apache.accumulo.core.spi.balancer.HostRegexTableLoadBalancer;
 import org.apache.accumulo.core.util.HostAndPort;
 import org.apache.accumulo.minicluster.ServerType;
 import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
@@ -77,7 +78,7 @@ public class SuspendedTabletsIT extends ConfigurableMacBase {
   private static ExecutorService THREAD_POOL;
 
   public static final int TSERVERS = 3;
-  public static final long SUSPEND_DURATION = 20;
+  public static final long SUSPEND_DURATION = 80;
   public static final int TABLETS = 30;
 
   @Override
@@ -91,6 +92,11 @@ public class SuspendedTabletsIT extends ConfigurableMacBase {
     cfg.setClientProperty(ClientProperty.INSTANCE_ZOOKEEPERS_TIMEOUT, "5s");
     cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
     cfg.setNumTservers(TSERVERS);
+    // config custom balancer to keep all metadata on one server
+    cfg.setProperty(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + 
MetadataTable.NAME, "*");
+    cfg.setProperty(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, 
"7s");
+    cfg.setProperty(Property.TABLE_LOAD_BALANCER.getKey(),
+        HostRegexTableLoadBalancer.class.getName());
   }
 
   @Test
@@ -115,13 +121,30 @@ public class SuspendedTabletsIT extends 
ConfigurableMacBase {
     // Run the test body. When we get to the point where we need tservers to 
go away, stop them via
     // a clean shutdown.
     suspensionTestBody((ctx, locs, count) -> {
-      Set<TServerInstance> tserversSet = new HashSet<>();
+      Set<TServerInstance> tserverSet = new HashSet<>();
+      Set<TServerInstance> metadataServerSet = new HashSet<>();
+
+      TabletLocator tl = TabletLocator.getLocator(ctx, MetadataTable.ID);
       for (TabletLocationState tls : locs.locationStates.values()) {
         if (tls.current != null) {
-          tserversSet.add(tls.current);
+          // add to set of all servers
+          tserverSet.add(tls.current);
+
+          // get server that the current tablets metadata is on
+          TabletLocator.TabletLocation tab =
+              tl.locateTablet(ctx, tls.extent.toMetaRow(), false, false);
+          // add it to the set of servers with metadata
+          metadataServerSet
+              .add(new TServerInstance(tab.tablet_location, 
Long.valueOf(tab.tablet_session, 16)));
         }
       }
-      List<TServerInstance> tserversList = new ArrayList<>(tserversSet);
+
+      // remove servers with metadata on them from the list of servers to be 
shutdown
+      tserverSet.removeAll(metadataServerSet);
+
+      assertEquals("Expecting two tServers in shutdown-list", 2, 
tserverSet.size());
+
+      List<TServerInstance> tserversList = new ArrayList<>(tserverSet);
       Collections.shuffle(tserversList, RANDOM);
 
       for (int i1 = 0; i1 < count; ++i1) {
@@ -168,14 +191,13 @@ public class SuspendedTabletsIT extends 
ConfigurableMacBase {
 
       String tableName = getUniqueNames(1)[0];
 
-      // Create a table with a bunch of splits
-      log.info("Creating table " + tableName);
-      ctx.tableOperations().create(tableName);
       SortedSet<Text> splitPoints = new TreeSet<>();
       for (int i = 1; i < TABLETS; ++i) {
         splitPoints.add(new Text("" + i));
       }
-      ctx.tableOperations().addSplits(tableName, splitPoints);
+      log.info("Creating table " + tableName);
+      NewTableConfiguration ntc = new 
NewTableConfiguration().withSplits(splitPoints);
+      ctx.tableOperations().create(tableName, ntc);
 
       // Wait for all of the tablets to hosted ...
       log.info("Waiting on hosting and balance");
@@ -188,8 +210,8 @@ public class SuspendedTabletsIT extends ConfigurableMacBase 
{
       // ... and balanced.
       ctx.instanceOperations().waitForBalance();
       do {
-        // Give at least another 5 seconds for migrations to finish up
-        Thread.sleep(5000);
+        // Give at least another 15 seconds for migrations to finish up
+        Thread.sleep(15000);
         ds = TabletLocations.retrieve(ctx, tableName);
       } while (ds.hostedCount != TABLETS);
 
@@ -203,29 +225,23 @@ public class SuspendedTabletsIT extends 
ConfigurableMacBase {
       log.info("Eliminating tablet servers");
       serverStopper.eliminateTabletServers(ctx, beforeDeathState, 2);
 
-      // Eventually some tablets will be suspended.
+      // All tablets should be either hosted or suspended.
       log.info("Waiting on suspended tablets");
-      ds = TabletLocations.retrieve(ctx, tableName);
-      // Until we can scan the metadata table, the manager probably can't 
either, so won't have been
-      // able to suspend the tablets.
-      // So we note the time that we were first able to successfully scan the 
metadata table.
-      long killTime = System.nanoTime();
-      while (ds.suspended.keySet().size() != 2) {
+      do {
         Thread.sleep(1000);
         ds = TabletLocations.retrieve(ctx, tableName);
-      }
+      } while (ds.suspended.keySet().size() != 2
+          && (ds.suspendedCount + ds.hostedCount) != TABLETS);
 
       SetMultimap<HostAndPort,KeyExtent> deadTabletsByServer = ds.suspended;
 
-      // By this point, all tablets should be either hosted or suspended. All 
suspended tablets
-      // should
-      // "belong" to the dead tablet servers, and should be in exactly the 
same place as before any
-      // tserver death.
+      // All suspended tablets should "belong" to the dead tablet servers, and 
should be in exactly
+      // the same place as before any tserver death.
       for (HostAndPort server : deadTabletsByServer.keySet()) {
-        assertEquals(deadTabletsByServer.get(server), 
beforeDeathState.hosted.get(server));
+        // Comparing pre-death, hosted tablets to suspended tablets on a server
+        assertEquals(beforeDeathState.hosted.get(server), 
deadTabletsByServer.get(server));
       }
       assertEquals(TABLETS, ds.hostedCount + ds.suspendedCount);
-
       // Restart the first tablet server, making sure it ends up on the same 
port
       HostAndPort restartedServer = 
deadTabletsByServer.keySet().iterator().next();
       log.info("Restarting " + restartedServer);
@@ -248,9 +264,6 @@ public class SuspendedTabletsIT extends ConfigurableMacBase 
{
         Thread.sleep(1000);
         ds = TabletLocations.retrieve(ctx, tableName);
       }
-
-      long recoverTime = System.nanoTime();
-      assertTrue(recoverTime - killTime >= 
NANOSECONDS.convert(SUSPEND_DURATION, SECONDS));
     }
   }
 

Reply via email to