This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push: new 9bbf42f4dc Fix TabletGroupWatcher shutdown order (#3571) 9bbf42f4dc is described below commit 9bbf42f4dc3ebcf21d70c837f577c60708d3d2de Author: Dave Marion <dlmar...@apache.org> AuthorDate: Mon Jul 10 11:40:12 2023 -0400 Fix TabletGroupWatcher shutdown order (#3571) Modified TabletGroupWatcher such that when shutting down all TabletServers it unloads user tables before the metadata table and the metadata table before the root table. --- .../manager/state/LoggingTabletStateStore.java | 6 + .../server/manager/state/MetaDataStateStore.java | 15 +- .../server/manager/state/RootTabletStateStore.java | 5 +- .../server/manager/state/TabletStateStore.java | 11 +- .../server/manager/state/ZooTabletStateStore.java | 10 +- .../manager/state/RootTabletStateStoreTest.java | 3 +- .../accumulo/manager/TabletGroupWatcher.java | 49 +++++- .../apache/accumulo/manager/state/TableCounts.java | 9 ++ .../apache/accumulo/manager/state/TableStats.java | 6 + .../manager/tserverOps/ShutdownTServer.java | 3 + .../org/apache/accumulo/tserver/TabletServer.java | 1 + .../accumulo/tserver/UnloadTabletHandler.java | 1 + .../org/apache/accumulo/tserver/tablet/Tablet.java | 1 + .../test/functional/ManagerAssignmentIT.java | 166 +++++++++++++++++++++ test/src/main/resources/log4j2-test.properties | 3 + 15 files changed, 276 insertions(+), 13 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java index 6ef48fdc07..84c62254ca 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java @@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.logging.TabletLogger; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletLocationState; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.util.HostAndPort; import org.apache.hadoop.fs.Path; @@ -40,6 +41,11 @@ class LoggingTabletStateStore implements TabletStateStore { this.wrapped = tss; } + @Override + public DataLevel getLevel() { + return wrapped.getLevel(); + } + @Override public String name() { return wrapped.name(); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java index bb27b32cda..8c7bc888eb 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletLocationState; import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; @@ -40,16 +41,24 @@ class MetaDataStateStore implements TabletStateStore { protected final CurrentState state; private final String targetTableName; private final Ample ample; + private final DataLevel level; - protected MetaDataStateStore(ClientContext context, CurrentState state, String targetTableName) { + protected MetaDataStateStore(DataLevel level, ClientContext context, CurrentState state, + String targetTableName) { + this.level = level; this.context = context; this.state = state; this.ample = context.getAmple(); this.targetTableName = targetTableName; } - MetaDataStateStore(ClientContext context, CurrentState state) { - this(context, state, MetadataTable.NAME); + MetaDataStateStore(DataLevel level, ClientContext context, CurrentState state) { + this(level, context, state, MetadataTable.NAME); + } + + @Override + public DataLevel getLevel() { + return level; } @Override diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java index 9acd1cc48a..98123e1c2f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java @@ -21,12 +21,13 @@ package org.apache.accumulo.server.manager.state; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TabletLocationState; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; class RootTabletStateStore extends MetaDataStateStore { - RootTabletStateStore(ClientContext context, CurrentState state) { - super(context, state, RootTable.NAME); + RootTabletStateStore(DataLevel level, ClientContext context, CurrentState state) { + super(level, context, state, RootTable.NAME); } @Override diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java index c96912ee58..05072fd1b0 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java @@ -36,6 +36,11 @@ import org.apache.hadoop.fs.Path; */ public interface TabletStateStore extends Iterable<TabletLocationState> { + /** + * Get the level for this state store + */ + DataLevel getLevel(); + /** * Identifying name for this tablet state store. */ @@ -112,13 +117,13 @@ public interface TabletStateStore extends Iterable<TabletLocationState> { TabletStateStore tss; switch (level) { case ROOT: - tss = new ZooTabletStateStore(context); + tss = new ZooTabletStateStore(level, context); break; case METADATA: - tss = new RootTabletStateStore(context, state); + tss = new RootTabletStateStore(level, context, state); break; case USER: - tss = new MetaDataStateStore(context, state); + tss = new MetaDataStateStore(level, context, state); break; default: throw new IllegalArgumentException("Unknown level " + level); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java index ae7643d32d..398e67c112 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java @@ -29,6 +29,7 @@ import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletLocationState; import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.Ample.ReadConsistency; import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator; import org.apache.accumulo.core.metadata.schema.TabletMetadata; @@ -45,12 +46,19 @@ class ZooTabletStateStore implements TabletStateStore { private static final Logger log = LoggerFactory.getLogger(ZooTabletStateStore.class); private final Ample ample; private final ClientContext context; + private final DataLevel level; - ZooTabletStateStore(ClientContext context) { + ZooTabletStateStore(DataLevel level, ClientContext context) { + this.level = level; this.context = context; this.ample = context.getAmple(); } + @Override + public DataLevel getLevel() { + return level; + } + @Override public ClosableIterator<TabletLocationState> iterator() { diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java index a6fdda4a0b..68dd6c5f86 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletLocationState; import org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException; import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; @@ -93,7 +94,7 @@ public class RootTabletStateStoreTest { ServerContext context = MockServerContext.get(); expect(context.getAmple()).andReturn(new TestAmple()).anyTimes(); EasyMock.replay(context); - ZooTabletStateStore tstore = new ZooTabletStateStore(context); + ZooTabletStateStore tstore = new ZooTabletStateStore(DataLevel.ROOT, context); KeyExtent root = RootTable.EXTENT; String sessionId = "this is my unique session data"; TServerInstance server = diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index 1b7f21d7b1..482bbd2f44 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -65,6 +65,7 @@ import org.apache.accumulo.core.metadata.TabletLocationState; import org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException; import org.apache.accumulo.core.metadata.TabletState; import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; @@ -257,12 +258,47 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread { if (state == TabletState.ASSIGNED) { goal = TabletGoalState.HOSTED; } + if (Manager.log.isTraceEnabled()) { + Manager.log.trace( + "[{}] Shutting down all Tservers: {}, dependentCount: {} Extent: {}, state: {}, goal: {}", + store.name(), manager.serversToShutdown.equals(currentTServers.keySet()), + dependentWatcher == null ? "null" : dependentWatcher.assignedOrHosted(), tls.extent, + state, goal); + } // if we are shutting down all the tabletservers, we have to do it in order if ((goal == TabletGoalState.SUSPENDED && state == TabletState.HOSTED) && manager.serversToShutdown.equals(currentTServers.keySet())) { - if (dependentWatcher != null && dependentWatcher.assignedOrHosted() > 0) { - goal = TabletGoalState.HOSTED; + if (dependentWatcher != null) { + // If the dependentWatcher is for the user tables, check to see + // that user tables exist. + DataLevel dependentLevel = dependentWatcher.store.getLevel(); + boolean userTablesExist = true; + switch (dependentLevel) { + case USER: + Set<TableId> onlineTables = manager.onlineTables(); + onlineTables.remove(RootTable.ID); + onlineTables.remove(MetadataTable.ID); + userTablesExist = !onlineTables.isEmpty(); + break; + case METADATA: + case ROOT: + default: + break; + } + // If the stats object in the dependentWatcher is empty, then it + // currently does not have data about what is hosted or not. In + // that case host these tablets until the dependent watcher can + // gather some data. + final Map<TableId,TableCounts> stats = dependentWatcher.getStats(); + if (dependentLevel == DataLevel.USER) { + if (userTablesExist + && (stats == null || stats.isEmpty() || assignedOrHosted(stats) > 0)) { + goal = TabletGoalState.HOSTED; + } + } else if (stats == null || stats.isEmpty() || assignedOrHosted(stats) > 0) { + goal = TabletGoalState.HOSTED; + } } } @@ -308,6 +344,8 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread { TServerConnection client = manager.tserverSet.getConnection(location.getServerInstance()); if (client != null) { + Manager.log.trace("[{}] Requesting TabletServer {} unload {} {}", store.name(), + location.getServerInstance(), tls.extent, goal.howUnload()); client.unloadTablet(manager.managerLock, tls.extent, goal.howUnload(), manager.getSteadyTime()); unloaded++; @@ -327,6 +365,7 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread { // provide stats after flushing changes to avoid race conditions w/ delete table stats.end(managerState); + Manager.log.trace("[{}] End stats collection: {}", store.name(), stats); // Report changes for (TabletState state : TabletState.values()) { @@ -507,8 +546,12 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread { } private int assignedOrHosted() { + return assignedOrHosted(stats.getLast()); + } + + private int assignedOrHosted(Map<TableId,TableCounts> last) { int result = 0; - for (TableCounts counts : stats.getLast().values()) { + for (TableCounts counts : last.values()) { result += counts.assigned() + counts.hosted(); } return result; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableCounts.java b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableCounts.java index 5a9b98d471..0f1b4ca6fa 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableCounts.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableCounts.java @@ -42,4 +42,13 @@ public class TableCounts { public int suspended() { return counts[TabletState.SUSPENDED.ordinal()]; } + + @Override + public String toString() { + return new StringBuilder().append("unassigned: ").append(unassigned()).append(", assigned: ") + .append(assigned()).append(", assignedToDeadServers: ").append(assignedToDeadServers()) + .append(", hosted: ").append(hosted()).append(", suspended: ").append(suspended()) + .toString(); + } + } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java index 89997e12b0..3e2380a57c 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java @@ -79,4 +79,10 @@ public class TableStats { public synchronized long lastScanFinished() { return endScan; } + + @Override + public String toString() { + return new StringBuilder().append("last: ").append(last.toString()).toString(); + } + } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java index 0cf05ac0c0..7c50d8236f 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java @@ -70,6 +70,9 @@ public class ShutdownTServer extends ManagerRepo { connection.halt(manager.getManagerLock()); log.info("tablet server asked to halt {}", server); return 0; + } else { + log.info("tablet server {} still has tablets for tables: {}", server, + (status.tableMap == null) ? "null" : status.tableMap.keySet()); } } catch (TTransportException ex) { // expected diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index 3fd3dfb5e7..74176696e2 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -401,6 +401,7 @@ public class TabletServer extends AbstractServer implements TabletHostingServer } void requestStop() { + log.info("Stop requested."); serverStopRequested = true; } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java index 0b3c611a05..1eaeb5acf3 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java @@ -55,6 +55,7 @@ class UnloadTabletHandler implements Runnable { public void run() { Tablet t = null; + log.info("Tablet unload for extent {} requested.", extent); synchronized (server.unopenedTablets) { if (server.unopenedTablets.contains(extent)) { diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 1b818d8759..8097c3660b 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@ -894,6 +894,7 @@ public class Tablet extends TabletBase { public void close(boolean saveState) throws IOException { initiateClose(saveState); completeClose(saveState, true); + log.info("Tablet {} closed.", this.extent); } void initiateClose(boolean saveState) { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java index c3fc72d242..04dc8490d2 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java @@ -21,22 +21,43 @@ package org.apache.accumulo.test.functional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.fail; import java.time.Duration; +import java.util.Collections; +import java.util.Map.Entry; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.IsolatedScanner; +import org.apache.accumulo.core.client.admin.Locations; import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.fate.zookeeper.ServiceLock; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TabletLocationState; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.core.util.HostAndPort; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.harness.AccumuloClusterHarness; +import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterControl; import org.apache.accumulo.server.manager.state.MetaDataTableScanner; +import org.apache.accumulo.test.util.Wait; import org.junit.jupiter.api.Test; public class ManagerAssignmentIT extends AccumuloClusterHarness { @@ -91,6 +112,151 @@ public class ManagerAssignmentIT extends AccumuloClusterHarness { } } + @Test + public void testShutdownOnlyTServerWithUserTable() throws Exception { + + // 2 TabletServers started for this test, shut them down so we only have 1. + getClusterControl().stopAllServers(ServerType.TABLET_SERVER); + ((MiniAccumuloClusterControl) getClusterControl()).start(ServerType.TABLET_SERVER, + Collections.emptyMap(), 1); + + String tableName = getUniqueNames(1)[0]; + + try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { + + Wait.waitFor(() -> client.instanceOperations().getTabletServers().size() == 1); + + client.tableOperations().create(tableName); + + // wait for everything to be hosted and balanced + client.instanceOperations().waitForBalance(); + + try (var writer = client.createBatchWriter(tableName)) { + for (int i = 0; i < 1000000; i++) { + Mutation m = new Mutation(String.format("%08d", i)); + m.put("", "", ""); + writer.addMutation(m); + } + } + client.tableOperations().flush(tableName, null, null, true); + + final CountDownLatch latch = new CountDownLatch(10); + + Runnable task = new Runnable() { + @Override + public void run() { + while (true) { + try (var scanner = new IsolatedScanner(client.createScanner(tableName))) { + // TODO maybe do not close scanner? The following limit was placed on the stream to + // avoid reading all the data possibly leaving a scan session active on the tserver + int count = 0; + for (Entry<Key,Value> e : scanner) { + count++; + // let the test thread know that this thread has read some data + if (count == 1_000) { + latch.countDown(); + } + } + } catch (Exception e) { + e.printStackTrace(); + break; + } + } + } + }; + + ExecutorService service = Executors.newFixedThreadPool(10); + for (int i = 0; i < 10; i++) { + service.execute(task); + } + + // Wait until all threads are reading some data + latch.await(); + + // getClusterControl().stopAllServers(ServerType.TABLET_SERVER) + // could potentially send a kill -9 to the process. Shut the tablet + // servers down in a more graceful way. + + Locations locs = client.tableOperations().locate(tableName, + Collections.singletonList(TabletsSection.getRange())); + locs.groupByTablet().keySet().stream().map(tid -> locs.getTabletLocation(tid)) + .forEach(location -> { + HostAndPort address = HostAndPort.fromString(location); + String addressWithSession = address.toString(); + var zLockPath = ServiceLock.path(getCluster().getServerContext().getZooKeeperRoot() + + Constants.ZTSERVERS + "/" + address.toString()); + long sessionId = + ServiceLock.getSessionId(getCluster().getServerContext().getZooCache(), zLockPath); + if (sessionId != 0) { + addressWithSession = address.toString() + "[" + Long.toHexString(sessionId) + "]"; + } + + final String finalAddress = addressWithSession; + System.out.println("Attempting to shutdown TabletServer at: " + address.toString()); + try { + ThriftClientTypes.MANAGER.executeVoid((ClientContext) client, + c -> c.shutdownTabletServer(TraceUtil.traceInfo(), + getCluster().getServerContext().rpcCreds(), finalAddress, false)); + } catch (AccumuloException | AccumuloSecurityException e) { + fail("Error shutting down TabletServer", e); + } + + }); + + Wait.waitFor(() -> client.instanceOperations().getTabletServers().size() == 0); + + } + } + + @Test + public void testShutdownOnlyTServerWithoutUserTable() throws Exception { + + // 2 TabletServers started for this test, shut them down so we only have 1. + getClusterControl().stopAllServers(ServerType.TABLET_SERVER); + ((MiniAccumuloClusterControl) getClusterControl()).start(ServerType.TABLET_SERVER, + Collections.emptyMap(), 1); + + try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { + + Wait.waitFor(() -> client.instanceOperations().getTabletServers().size() == 1); + + client.instanceOperations().waitForBalance(); + + // getClusterControl().stopAllServers(ServerType.TABLET_SERVER) + // could potentially send a kill -9 to the process. Shut the tablet + // servers down in a more graceful way. + + Locations locs = client.tableOperations().locate(RootTable.NAME, + Collections.singletonList(TabletsSection.getRange())); + locs.groupByTablet().keySet().stream().map(tid -> locs.getTabletLocation(tid)) + .forEach(location -> { + HostAndPort address = HostAndPort.fromString(location); + String addressWithSession = address.toString(); + var zLockPath = ServiceLock.path(getCluster().getServerContext().getZooKeeperRoot() + + Constants.ZTSERVERS + "/" + address.toString()); + long sessionId = + ServiceLock.getSessionId(getCluster().getServerContext().getZooCache(), zLockPath); + if (sessionId != 0) { + addressWithSession = address.toString() + "[" + Long.toHexString(sessionId) + "]"; + } + + final String finalAddress = addressWithSession; + System.out.println("Attempting to shutdown TabletServer at: " + address.toString()); + try { + ThriftClientTypes.MANAGER.executeVoid((ClientContext) client, + c -> c.shutdownTabletServer(TraceUtil.traceInfo(), + getCluster().getServerContext().rpcCreds(), finalAddress, false)); + } catch (AccumuloException | AccumuloSecurityException e) { + fail("Error shutting down TabletServer", e); + } + + }); + + Wait.waitFor(() -> client.instanceOperations().getTabletServers().size() == 0); + + } + } + private TabletLocationState getTabletLocationState(AccumuloClient c, String tableId) { try (MetaDataTableScanner s = new MetaDataTableScanner((ClientContext) c, new Range(TabletsSection.encodeRow(TableId.of(tableId), null)), MetadataTable.NAME)) { diff --git a/test/src/main/resources/log4j2-test.properties b/test/src/main/resources/log4j2-test.properties index cf09c1b90a..f3a23b7474 100644 --- a/test/src/main/resources/log4j2-test.properties +++ b/test/src/main/resources/log4j2-test.properties @@ -149,6 +149,9 @@ logger.37.level = warn logger.38.name = org.apache.hadoop.fs.TrashPolicyDefault logger.38.level = debug +logger.39.name = org.apache.accumulo.manager.Manager +logger.39.level = trace + property.metricsFilename = ./target/test-metrics # appender.metrics.type = Console