This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 9bbf42f4dc Fix TabletGroupWatcher shutdown order (#3571)
9bbf42f4dc is described below
commit 9bbf42f4dc3ebcf21d70c837f577c60708d3d2de
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Mon Jul 10 11:40:12 2023 -0400
Fix TabletGroupWatcher shutdown order (#3571)
Modified TabletGroupWatcher such that when shutting
down all TabletServers it unloads user tables before the
metadata table and the metadata table before the root
table.
---
.../manager/state/LoggingTabletStateStore.java | 6 +
.../server/manager/state/MetaDataStateStore.java | 15 +-
.../server/manager/state/RootTabletStateStore.java | 5 +-
.../server/manager/state/TabletStateStore.java | 11 +-
.../server/manager/state/ZooTabletStateStore.java | 10 +-
.../manager/state/RootTabletStateStoreTest.java | 3 +-
.../accumulo/manager/TabletGroupWatcher.java | 49 +++++-
.../apache/accumulo/manager/state/TableCounts.java | 9 ++
.../apache/accumulo/manager/state/TableStats.java | 6 +
.../manager/tserverOps/ShutdownTServer.java | 3 +
.../org/apache/accumulo/tserver/TabletServer.java | 1 +
.../accumulo/tserver/UnloadTabletHandler.java | 1 +
.../org/apache/accumulo/tserver/tablet/Tablet.java | 1 +
.../test/functional/ManagerAssignmentIT.java | 166 +++++++++++++++++++++
test/src/main/resources/log4j2-test.properties | 3 +
15 files changed, 276 insertions(+), 13 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
index 6ef48fdc07..84c62254ca 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.logging.TabletLogger;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletLocationState;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.hadoop.fs.Path;
@@ -40,6 +41,11 @@ class LoggingTabletStateStore implements TabletStateStore {
this.wrapped = tss;
}
+ @Override
+ public DataLevel getLevel() {
+ return wrapped.getLevel();
+ }
+
@Override
public String name() {
return wrapped.name();
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
index bb27b32cda..8c7bc888eb 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
@@ -27,6 +27,7 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletLocationState;
import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
@@ -40,16 +41,24 @@ class MetaDataStateStore implements TabletStateStore {
protected final CurrentState state;
private final String targetTableName;
private final Ample ample;
+ private final DataLevel level;
- protected MetaDataStateStore(ClientContext context, CurrentState state,
String targetTableName) {
+ protected MetaDataStateStore(DataLevel level, ClientContext context,
CurrentState state,
+ String targetTableName) {
+ this.level = level;
this.context = context;
this.state = state;
this.ample = context.getAmple();
this.targetTableName = targetTableName;
}
- MetaDataStateStore(ClientContext context, CurrentState state) {
- this(context, state, MetadataTable.NAME);
+ MetaDataStateStore(DataLevel level, ClientContext context, CurrentState
state) {
+ this(level, context, state, MetadataTable.NAME);
+ }
+
+ @Override
+ public DataLevel getLevel() {
+ return level;
}
@Override
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java
index 9acd1cc48a..98123e1c2f 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java
@@ -21,12 +21,13 @@ package org.apache.accumulo.server.manager.state;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.TabletLocationState;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
class RootTabletStateStore extends MetaDataStateStore {
- RootTabletStateStore(ClientContext context, CurrentState state) {
- super(context, state, RootTable.NAME);
+ RootTabletStateStore(DataLevel level, ClientContext context, CurrentState
state) {
+ super(level, context, state, RootTable.NAME);
}
@Override
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
index c96912ee58..05072fd1b0 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
@@ -36,6 +36,11 @@ import org.apache.hadoop.fs.Path;
*/
public interface TabletStateStore extends Iterable<TabletLocationState> {
+ /**
+ * Get the level for this state store
+ */
+ DataLevel getLevel();
+
/**
* Identifying name for this tablet state store.
*/
@@ -112,13 +117,13 @@ public interface TabletStateStore extends
Iterable<TabletLocationState> {
TabletStateStore tss;
switch (level) {
case ROOT:
- tss = new ZooTabletStateStore(context);
+ tss = new ZooTabletStateStore(level, context);
break;
case METADATA:
- tss = new RootTabletStateStore(context, state);
+ tss = new RootTabletStateStore(level, context, state);
break;
case USER:
- tss = new MetaDataStateStore(context, state);
+ tss = new MetaDataStateStore(level, context, state);
break;
default:
throw new IllegalArgumentException("Unknown level " + level);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
index ae7643d32d..398e67c112 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
@@ -29,6 +29,7 @@ import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletLocationState;
import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.Ample.ReadConsistency;
import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
@@ -45,12 +46,19 @@ class ZooTabletStateStore implements TabletStateStore {
private static final Logger log =
LoggerFactory.getLogger(ZooTabletStateStore.class);
private final Ample ample;
private final ClientContext context;
+ private final DataLevel level;
- ZooTabletStateStore(ClientContext context) {
+ ZooTabletStateStore(DataLevel level, ClientContext context) {
+ this.level = level;
this.context = context;
this.ample = context.getAmple();
}
+ @Override
+ public DataLevel getLevel() {
+ return level;
+ }
+
@Override
public ClosableIterator<TabletLocationState> iterator() {
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
index a6fdda4a0b..68dd6c5f86 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
@@ -37,6 +37,7 @@ import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletLocationState;
import
org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException;
import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
@@ -93,7 +94,7 @@ public class RootTabletStateStoreTest {
ServerContext context = MockServerContext.get();
expect(context.getAmple()).andReturn(new TestAmple()).anyTimes();
EasyMock.replay(context);
- ZooTabletStateStore tstore = new ZooTabletStateStore(context);
+ ZooTabletStateStore tstore = new ZooTabletStateStore(DataLevel.ROOT,
context);
KeyExtent root = RootTable.EXTENT;
String sessionId = "this is my unique session data";
TServerInstance server =
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 1b7f21d7b1..482bbd2f44 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -65,6 +65,7 @@ import org.apache.accumulo.core.metadata.TabletLocationState;
import
org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException;
import org.apache.accumulo.core.metadata.TabletState;
import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily;
@@ -257,12 +258,47 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
if (state == TabletState.ASSIGNED) {
goal = TabletGoalState.HOSTED;
}
+ if (Manager.log.isTraceEnabled()) {
+ Manager.log.trace(
+ "[{}] Shutting down all Tservers: {}, dependentCount: {}
Extent: {}, state: {}, goal: {}",
+ store.name(),
manager.serversToShutdown.equals(currentTServers.keySet()),
+ dependentWatcher == null ? "null" :
dependentWatcher.assignedOrHosted(), tls.extent,
+ state, goal);
+ }
// if we are shutting down all the tabletservers, we have to do it
in order
if ((goal == TabletGoalState.SUSPENDED && state ==
TabletState.HOSTED)
&& manager.serversToShutdown.equals(currentTServers.keySet())) {
- if (dependentWatcher != null &&
dependentWatcher.assignedOrHosted() > 0) {
- goal = TabletGoalState.HOSTED;
+ if (dependentWatcher != null) {
+ // If the dependentWatcher is for the user tables, check to see
+ // that user tables exist.
+ DataLevel dependentLevel = dependentWatcher.store.getLevel();
+ boolean userTablesExist = true;
+ switch (dependentLevel) {
+ case USER:
+ Set<TableId> onlineTables = manager.onlineTables();
+ onlineTables.remove(RootTable.ID);
+ onlineTables.remove(MetadataTable.ID);
+ userTablesExist = !onlineTables.isEmpty();
+ break;
+ case METADATA:
+ case ROOT:
+ default:
+ break;
+ }
+ // If the stats object in the dependentWatcher is empty, then it
+ // currently does not have data about what is hosted or not. In
+ // that case host these tablets until the dependent watcher can
+ // gather some data.
+ final Map<TableId,TableCounts> stats =
dependentWatcher.getStats();
+ if (dependentLevel == DataLevel.USER) {
+ if (userTablesExist
+ && (stats == null || stats.isEmpty() ||
assignedOrHosted(stats) > 0)) {
+ goal = TabletGoalState.HOSTED;
+ }
+ } else if (stats == null || stats.isEmpty() ||
assignedOrHosted(stats) > 0) {
+ goal = TabletGoalState.HOSTED;
+ }
}
}
@@ -308,6 +344,8 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
TServerConnection client =
manager.tserverSet.getConnection(location.getServerInstance());
if (client != null) {
+ Manager.log.trace("[{}] Requesting TabletServer {} unload {}
{}", store.name(),
+ location.getServerInstance(), tls.extent,
goal.howUnload());
client.unloadTablet(manager.managerLock, tls.extent,
goal.howUnload(),
manager.getSteadyTime());
unloaded++;
@@ -327,6 +365,7 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
// provide stats after flushing changes to avoid race conditions w/
delete table
stats.end(managerState);
+ Manager.log.trace("[{}] End stats collection: {}", store.name(),
stats);
// Report changes
for (TabletState state : TabletState.values()) {
@@ -507,8 +546,12 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
}
private int assignedOrHosted() {
+ return assignedOrHosted(stats.getLast());
+ }
+
+ private int assignedOrHosted(Map<TableId,TableCounts> last) {
int result = 0;
- for (TableCounts counts : stats.getLast().values()) {
+ for (TableCounts counts : last.values()) {
result += counts.assigned() + counts.hosted();
}
return result;
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableCounts.java
b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableCounts.java
index 5a9b98d471..0f1b4ca6fa 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableCounts.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableCounts.java
@@ -42,4 +42,13 @@ public class TableCounts {
public int suspended() {
return counts[TabletState.SUSPENDED.ordinal()];
}
+
+ @Override
+ public String toString() {
+ return new StringBuilder().append("unassigned:
").append(unassigned()).append(", assigned: ")
+ .append(assigned()).append(", assignedToDeadServers:
").append(assignedToDeadServers())
+ .append(", hosted: ").append(hosted()).append(", suspended:
").append(suspended())
+ .toString();
+ }
+
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java
b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java
index 89997e12b0..3e2380a57c 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java
@@ -79,4 +79,10 @@ public class TableStats {
public synchronized long lastScanFinished() {
return endScan;
}
+
+ @Override
+ public String toString() {
+ return new StringBuilder().append("last:
").append(last.toString()).toString();
+ }
+
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java
index 0cf05ac0c0..7c50d8236f 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java
@@ -70,6 +70,9 @@ public class ShutdownTServer extends ManagerRepo {
connection.halt(manager.getManagerLock());
log.info("tablet server asked to halt {}", server);
return 0;
+ } else {
+ log.info("tablet server {} still has tablets for tables: {}",
server,
+ (status.tableMap == null) ? "null" : status.tableMap.keySet());
}
} catch (TTransportException ex) {
// expected
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 3fd3dfb5e7..74176696e2 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -401,6 +401,7 @@ public class TabletServer extends AbstractServer implements
TabletHostingServer
}
void requestStop() {
+ log.info("Stop requested.");
serverStopRequested = true;
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
index 0b3c611a05..1eaeb5acf3 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
@@ -55,6 +55,7 @@ class UnloadTabletHandler implements Runnable {
public void run() {
Tablet t = null;
+ log.info("Tablet unload for extent {} requested.", extent);
synchronized (server.unopenedTablets) {
if (server.unopenedTablets.contains(extent)) {
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 1b818d8759..8097c3660b 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -894,6 +894,7 @@ public class Tablet extends TabletBase {
public void close(boolean saveState) throws IOException {
initiateClose(saveState);
completeClose(saveState, true);
+ log.info("Tablet {} closed.", this.extent);
}
void initiateClose(boolean saveState) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
index c3fc72d242..04dc8490d2 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
@@ -21,22 +21,43 @@ package org.apache.accumulo.test.functional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.fail;
import java.time.Duration;
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.admin.Locations;
import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.TabletLocationState;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
+import org.apache.accumulo.core.trace.TraceUtil;
+import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterControl;
import org.apache.accumulo.server.manager.state.MetaDataTableScanner;
+import org.apache.accumulo.test.util.Wait;
import org.junit.jupiter.api.Test;
public class ManagerAssignmentIT extends AccumuloClusterHarness {
@@ -91,6 +112,151 @@ public class ManagerAssignmentIT extends
AccumuloClusterHarness {
}
}
+ @Test
+ public void testShutdownOnlyTServerWithUserTable() throws Exception {
+
+ // 2 TabletServers started for this test, shut them down so we only have 1.
+ getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ ((MiniAccumuloClusterControl)
getClusterControl()).start(ServerType.TABLET_SERVER,
+ Collections.emptyMap(), 1);
+
+ String tableName = getUniqueNames(1)[0];
+
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+
+ Wait.waitFor(() -> client.instanceOperations().getTabletServers().size()
== 1);
+
+ client.tableOperations().create(tableName);
+
+ // wait for everything to be hosted and balanced
+ client.instanceOperations().waitForBalance();
+
+ try (var writer = client.createBatchWriter(tableName)) {
+ for (int i = 0; i < 1000000; i++) {
+ Mutation m = new Mutation(String.format("%08d", i));
+ m.put("", "", "");
+ writer.addMutation(m);
+ }
+ }
+ client.tableOperations().flush(tableName, null, null, true);
+
+ final CountDownLatch latch = new CountDownLatch(10);
+
+ Runnable task = new Runnable() {
+ @Override
+ public void run() {
+ while (true) {
+ try (var scanner = new
IsolatedScanner(client.createScanner(tableName))) {
+ // TODO maybe do not close scanner? The following limit was
placed on the stream to
+ // avoid reading all the data possibly leaving a scan session
active on the tserver
+ int count = 0;
+ for (Entry<Key,Value> e : scanner) {
+ count++;
+ // let the test thread know that this thread has read some data
+ if (count == 1_000) {
+ latch.countDown();
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ break;
+ }
+ }
+ }
+ };
+
+ ExecutorService service = Executors.newFixedThreadPool(10);
+ for (int i = 0; i < 10; i++) {
+ service.execute(task);
+ }
+
+ // Wait until all threads are reading some data
+ latch.await();
+
+ // getClusterControl().stopAllServers(ServerType.TABLET_SERVER)
+ // could potentially send a kill -9 to the process. Shut the tablet
+ // servers down in a more graceful way.
+
+ Locations locs = client.tableOperations().locate(tableName,
+ Collections.singletonList(TabletsSection.getRange()));
+ locs.groupByTablet().keySet().stream().map(tid ->
locs.getTabletLocation(tid))
+ .forEach(location -> {
+ HostAndPort address = HostAndPort.fromString(location);
+ String addressWithSession = address.toString();
+ var zLockPath =
ServiceLock.path(getCluster().getServerContext().getZooKeeperRoot()
+ + Constants.ZTSERVERS + "/" + address.toString());
+ long sessionId =
+
ServiceLock.getSessionId(getCluster().getServerContext().getZooCache(),
zLockPath);
+ if (sessionId != 0) {
+ addressWithSession = address.toString() + "[" +
Long.toHexString(sessionId) + "]";
+ }
+
+ final String finalAddress = addressWithSession;
+ System.out.println("Attempting to shutdown TabletServer at: " +
address.toString());
+ try {
+ ThriftClientTypes.MANAGER.executeVoid((ClientContext) client,
+ c -> c.shutdownTabletServer(TraceUtil.traceInfo(),
+ getCluster().getServerContext().rpcCreds(),
finalAddress, false));
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ fail("Error shutting down TabletServer", e);
+ }
+
+ });
+
+ Wait.waitFor(() -> client.instanceOperations().getTabletServers().size()
== 0);
+
+ }
+ }
+
+ @Test
+ public void testShutdownOnlyTServerWithoutUserTable() throws Exception {
+
+ // 2 TabletServers started for this test, shut them down so we only have 1.
+ getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ ((MiniAccumuloClusterControl)
getClusterControl()).start(ServerType.TABLET_SERVER,
+ Collections.emptyMap(), 1);
+
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+
+ Wait.waitFor(() -> client.instanceOperations().getTabletServers().size()
== 1);
+
+ client.instanceOperations().waitForBalance();
+
+ // getClusterControl().stopAllServers(ServerType.TABLET_SERVER)
+ // could potentially send a kill -9 to the process. Shut the tablet
+ // servers down in a more graceful way.
+
+ Locations locs = client.tableOperations().locate(RootTable.NAME,
+ Collections.singletonList(TabletsSection.getRange()));
+ locs.groupByTablet().keySet().stream().map(tid ->
locs.getTabletLocation(tid))
+ .forEach(location -> {
+ HostAndPort address = HostAndPort.fromString(location);
+ String addressWithSession = address.toString();
+ var zLockPath =
ServiceLock.path(getCluster().getServerContext().getZooKeeperRoot()
+ + Constants.ZTSERVERS + "/" + address.toString());
+ long sessionId =
+
ServiceLock.getSessionId(getCluster().getServerContext().getZooCache(),
zLockPath);
+ if (sessionId != 0) {
+ addressWithSession = address.toString() + "[" +
Long.toHexString(sessionId) + "]";
+ }
+
+ final String finalAddress = addressWithSession;
+ System.out.println("Attempting to shutdown TabletServer at: " +
address.toString());
+ try {
+ ThriftClientTypes.MANAGER.executeVoid((ClientContext) client,
+ c -> c.shutdownTabletServer(TraceUtil.traceInfo(),
+ getCluster().getServerContext().rpcCreds(),
finalAddress, false));
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ fail("Error shutting down TabletServer", e);
+ }
+
+ });
+
+ Wait.waitFor(() -> client.instanceOperations().getTabletServers().size()
== 0);
+
+ }
+ }
+
private TabletLocationState getTabletLocationState(AccumuloClient c, String
tableId) {
try (MetaDataTableScanner s = new MetaDataTableScanner((ClientContext) c,
new Range(TabletsSection.encodeRow(TableId.of(tableId), null)),
MetadataTable.NAME)) {
diff --git a/test/src/main/resources/log4j2-test.properties
b/test/src/main/resources/log4j2-test.properties
index cf09c1b90a..f3a23b7474 100644
--- a/test/src/main/resources/log4j2-test.properties
+++ b/test/src/main/resources/log4j2-test.properties
@@ -149,6 +149,9 @@ logger.37.level = warn
logger.38.name = org.apache.hadoop.fs.TrashPolicyDefault
logger.38.level = debug
+logger.39.name = org.apache.accumulo.manager.Manager
+logger.39.level = trace
+
property.metricsFilename = ./target/test-metrics
# appender.metrics.type = Console
