This is an automated email from the ASF dual-hosted git repository.
awong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 9d01e10 KUDU-2915: add tool to unregister a tablet server
9d01e10 is described below
commit 9d01e1046249a815f26c7b5ebb1ceb2b67f72b9e
Author: zhangyifan27 <chinazhangyi...@163.com>
AuthorDate: Wed Dec 29 15:38:40 2021 +0800
KUDU-2915: add tool to unregister a tablet server
Add a 'kudu tserver unregister' tool to unregister a tserver from the
master. This tool will be useful when we want to decommission a tserver
without restarting masters.
This tool unregisters the dead tserver from master's in-memory map and
removes its persisted state from catalog table by default. It's also
possible to unregister a tserver which is not presumed dead by adding
'-force_unregister_live_tserver', or keep tserver's persisted state
by adding '-remove_tserver_state=false'.
Change-Id: If1f5c2979a8d14428f4bcc8e850c57ce228c793a
Reviewed-on: http://gerrit.cloudera.org:8080/18124
Reviewed-by: Alexey Serbin <aser...@cloudera.com>
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Kudu Jenkins
---
src/kudu/master/catalog_manager.cc | 1 +
src/kudu/master/master.proto | 20 +++++
src/kudu/master/master_service.cc | 16 ++++
src/kudu/master/master_service.h | 6 ++
src/kudu/master/ts_manager.cc | 15 ++++
src/kudu/master/ts_manager.h | 3 +
src/kudu/tools/kudu-tool-test.cc | 137 ++++++++++++++++++++++++++++++++++
src/kudu/tools/tool_action_common.cc | 62 +++++++++++++++
src/kudu/tools/tool_action_common.h | 5 ++
src/kudu/tools/tool_action_master.cc | 63 ----------------
src/kudu/tools/tool_action_tserver.cc | 54 ++++++++++++++
11 files changed, 319 insertions(+), 63 deletions(-)
diff --git a/src/kudu/master/catalog_manager.cc
b/src/kudu/master/catalog_manager.cc
index 94fb6ac..0a2a830 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -6231,6 +6231,7 @@ bool
CatalogManager::ScopedLeaderSharedLock::CheckIsInitializedAndIsLeaderOrResp
INITTED_OR_RESPOND(ConnectToMasterResponsePB);
INITTED_OR_RESPOND(GetMasterRegistrationResponsePB);
+INITTED_OR_RESPOND(UnregisterTServerResponsePB);
INITTED_OR_RESPOND(TSHeartbeatResponsePB);
INITTED_AND_LEADER_OR_RESPOND(AddMasterResponsePB);
INITTED_AND_LEADER_OR_RESPOND(AlterTableResponsePB);
diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto
index b9ded52..2967f24 100644
--- a/src/kudu/master/master.proto
+++ b/src/kudu/master/master.proto
@@ -1070,6 +1070,22 @@ message RefreshAuthzCacheResponsePB {
optional MasterErrorPB error = 1;
}
+// RemoveTabletServerRequest/Response: remove a tablet server from master's
+// in-memory map and persisted catalog.
+message UnregisterTServerRequestPB {
+ // The tserver UUID to be unregistered.
+ optional string uuid = 1;
+
+ // Whether to return an error in case the tserver is not presumed to be dead,
+ // per --tserver_unresponsive_timeout_ms.
+ // Disable this by default to make sure the tserver has been brought down.
+ optional bool force_unregister_live_tserver = 2 [default = false];
+}
+
+message UnregisterTServerResponsePB {
+ optional MasterErrorPB error = 1;
+}
+
enum MasterFeatures {
UNKNOWN_FEATURE = 0;
// The master supports creating tables with non-covering range partitions.
@@ -1184,6 +1200,10 @@ service MasterService {
option (kudu.rpc.authz_method) = "AuthorizeSuperUser";
}
+ rpc UnregisterTServer(UnregisterTServerRequestPB) returns
(UnregisterTServerResponsePB) {
+ option (kudu.rpc.authz_method) = "AuthorizeSuperUser";
+ }
+
// Master->Master RPCs
// ------------------------------------------------------------
diff --git a/src/kudu/master/master_service.cc
b/src/kudu/master/master_service.cc
index 71d84f2..0d0305d 100644
--- a/src/kudu/master/master_service.cc
+++ b/src/kudu/master/master_service.cc
@@ -331,6 +331,22 @@ void MasterServiceImpl::RemoveMaster(const
RemoveMasterRequestPB* req,
// See completion_cb in CatalogManager::InitiateMasterChangeConfig().
}
+void MasterServiceImpl::UnregisterTServer(const UnregisterTServerRequestPB*
req,
+ UnregisterTServerResponsePB* resp,
+ rpc::RpcContext* rpc) {
+ const auto& ts_uuid = req->uuid();
+ bool force_unregister_live_tserver = req->force_unregister_live_tserver();
+
+ Status s = server_->ts_manager()->UnregisterTServer(ts_uuid,
force_unregister_live_tserver);
+ if (PREDICT_FALSE(!s.ok() && !s.IsNotFound())) {
+ // Ignore the NotFound error to make this RPC retriable and effectively
idempotent.
+ StatusToPB(s, resp->mutable_error()->mutable_status());
+ resp->mutable_error()->set_code(MasterErrorPB::UNKNOWN_ERROR);
+ }
+
+ rpc->RespondSuccess();
+}
+
void MasterServiceImpl::TSHeartbeat(const TSHeartbeatRequestPB* req,
TSHeartbeatResponsePB* resp,
rpc::RpcContext* rpc) {
diff --git a/src/kudu/master/master_service.h b/src/kudu/master/master_service.h
index 011a4bb..f86a225 100644
--- a/src/kudu/master/master_service.h
+++ b/src/kudu/master/master_service.h
@@ -75,6 +75,8 @@ class RefreshAuthzCacheRequestPB;
class RefreshAuthzCacheResponsePB;
class RemoveMasterRequestPB;
class RemoveMasterResponsePB;
+class UnregisterTServerRequestPB;
+class UnregisterTServerResponsePB;
class ReplaceTabletRequestPB;
class ReplaceTabletResponsePB;
class TSHeartbeatRequestPB;
@@ -114,6 +116,10 @@ class MasterServiceImpl : public MasterServiceIf {
void RemoveMaster(const RemoveMasterRequestPB* req,
RemoveMasterResponsePB* resp, rpc::RpcContext* rpc)
override;
+ void UnregisterTServer(const UnregisterTServerRequestPB* req,
+ UnregisterTServerResponsePB* resp,
+ rpc::RpcContext* rpc) override;
+
void Ping(const PingRequestPB* req,
PingResponsePB* resp,
rpc::RpcContext* rpc) override;
diff --git a/src/kudu/master/ts_manager.cc b/src/kudu/master/ts_manager.cc
index 928e352..78e9930 100644
--- a/src/kudu/master/ts_manager.cc
+++ b/src/kudu/master/ts_manager.cc
@@ -316,6 +316,21 @@ void TSManager::SetAllTServersNeedFullTabletReports() {
}
}
+Status TSManager::UnregisterTServer(const std::string& ts_uuid,
+ bool force_unregister_live_tserver) {
+ lock_guard<rw_spinlock> l(lock_);
+ shared_ptr<TSDescriptor> ts_desc;
+ if (!FindCopy(servers_by_id_, ts_uuid, &ts_desc)) {
+ return Status::NotFound(Substitute("Requested tserver $0 has not been
registered", ts_uuid));
+ }
+
+ if (!force_unregister_live_tserver && !ts_desc->PresumedDead()) {
+ return Status::IllegalState(Substitute("TServer $0 is not presumed dead.",
ts_uuid));
+ }
+ servers_by_id_.erase(ts_uuid);
+ return Status::OK();
+}
+
int TSManager::ClusterSkew() const {
int min_count = std::numeric_limits<int>::max();
int max_count = 0;
diff --git a/src/kudu/master/ts_manager.h b/src/kudu/master/ts_manager.h
index 464dfa6..9b9fe53 100644
--- a/src/kudu/master/ts_manager.h
+++ b/src/kudu/master/ts_manager.h
@@ -125,6 +125,9 @@ class TSManager {
// Resets the tserver states and reloads them from disk.
Status ReloadTServerStates(SysCatalogTable* sys_catalog);
+ // Remove the tserver from 'servers_by_id_'.
+ Status UnregisterTServer(const std::string& ts_uuid, bool
force_unregister_live_tserver);
+
private:
friend class TServerStateLoader;
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index c34bdd0..69b5471 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -143,6 +143,8 @@ DECLARE_bool(hive_metastore_sasl_enabled);
DECLARE_bool(show_values);
DECLARE_bool(show_attributes);
DECLARE_int32(catalog_manager_inject_latency_load_ca_info_ms);
+DECLARE_int32(heartbeat_interval_ms);
+DECLARE_int32(tserver_unresponsive_timeout_ms);
DECLARE_int32(rpc_negotiation_inject_delay_ms);
DECLARE_string(block_manager);
DECLARE_string(hive_metastore_uris);
@@ -7447,6 +7449,141 @@ TEST_F(ToolTest, TestNonDefaultPrincipal) {
"--sasl_protocol_name=oryx",
HostPort::ToCommaSeparatedString(cluster_->master_rpc_addrs())}));
}
+class UnregisterTServerTest : public ToolTest, public
::testing::WithParamInterface<bool> {
+ public:
+ void StartCluster() {
+ // Test on a multi-master cluster.
+ InternalMiniClusterOptions opts;
+ opts.num_masters = 3;
+ StartMiniCluster(std::move(opts));
+ }
+
+ string GetMasterAddrsStr() {
+ vector<string> master_addrs;
+ for (const auto& hp : mini_cluster_->master_rpc_addrs()) {
+ master_addrs.emplace_back(hp.ToString());
+ }
+ return JoinStrings(master_addrs, ",");
+ }
+};
+
+INSTANTIATE_TEST_SUITE_P(, UnregisterTServerTest, ::testing::Bool());
+
+TEST_P(UnregisterTServerTest, TestUnregisterTServer) {
+ bool remove_tserver_state = GetParam();
+
+ // Set a short timeout that masters consider a tserver dead.
+ FLAGS_tserver_unresponsive_timeout_ms = 3000;
+ NO_FATALS(StartCluster());
+ const string master_addrs_str = GetMasterAddrsStr();
+ MiniTabletServer* ts = mini_cluster_->mini_tablet_server(0);
+ const string ts_uuid = ts->uuid();
+ const string ts_hostport = ts->bound_rpc_addr().ToString();
+
+ // Enter maintenance mode on the tserver and shut it down.
+ ASSERT_OK(RunKuduTool({"tserver", "state", "enter_maintenance",
master_addrs_str, ts_uuid}));
+ ts->Shutdown();
+
+ {
+ string out;
+ string err;
+ // Getting an error when running ksck and the output contains the dead
tserver.
+ Status s =
+ RunActionStdoutStderrString(Substitute("cluster ksck $0",
master_addrs_str), &out, &err);
+ ASSERT_TRUE(s.IsRuntimeError());
+ ASSERT_STR_CONTAINS(out, Substitute("$0 | $1 | UNAVAILABLE", ts_uuid,
ts_hostport));
+ }
+ // Wait the tserver become dead.
+ ASSERT_EVENTUALLY(
+ [&] { ASSERT_EQ(0,
mini_cluster_->mini_master(0)->master()->ts_manager()->GetLiveCount()); });
+
+ // Unregister the tserver.
+ ASSERT_OK(RunKuduTool({"tserver",
+ "unregister",
+ master_addrs_str,
+ ts_uuid,
+ Substitute("-remove_tserver_state=$0",
remove_tserver_state)}));
+ {
+ // Run ksck and get no error.
+ string out;
+ NO_FATALS(RunActionStdoutString(Substitute("cluster ksck $0",
master_addrs_str), &out));
+ if (remove_tserver_state) {
+ // Both the persisted state and registration of the tserver was removed.
+ ASSERT_STR_NOT_CONTAINS(out, Substitute(" $0 | MAINTENANCE_MODE",
ts_uuid));
+ ASSERT_STR_NOT_CONTAINS(out, ts_uuid);
+ } else {
+ // Only the registration of the tserver was removed.
+ ASSERT_STR_CONTAINS(out, Substitute(" $0 | MAINTENANCE_MODE", ts_uuid));
+ ASSERT_STR_NOT_CONTAINS(out, Substitute("$0 | $1 | UNAVAILABLE",
ts_uuid, ts_hostport));
+ }
+ }
+
+ // Restart the tserver and re-register it on masters.
+ ts->Start();
+ {
+ string out;
+ ASSERT_EVENTUALLY([&]() {
+ NO_FATALS(RunActionStdoutString(Substitute("cluster ksck $0",
master_addrs_str), &out));
+ });
+ if (remove_tserver_state) {
+ // The tserver came back as a brand new tserver.
+ ASSERT_STR_NOT_CONTAINS(out, Substitute(" $0 | MAINTENANCE_MODE",
ts_uuid));
+ ASSERT_STR_CONTAINS(out, ts_uuid);
+ } else {
+ // The tserver got its original maintenance state.
+ ASSERT_STR_CONTAINS(out, Substitute(" $0 | MAINTENANCE_MODE", ts_uuid));
+ ASSERT_STR_CONTAINS(out, ts_uuid);
+ }
+ }
+}
+
+TEST_F(UnregisterTServerTest, TestUnregisterTServerNotPresumedDead) {
+ // Reduce the TS<->Master heartbeat interval to speed up testing.
+ FLAGS_heartbeat_interval_ms = 100;
+ NO_FATALS(StartCluster());
+ const string master_addrs_str = GetMasterAddrsStr();
+ MiniTabletServer* ts = mini_cluster_->mini_tablet_server(0);
+ const string ts_uuid = ts->uuid();
+ const string ts_hostport = ts->bound_rpc_addr().ToString();
+
+ // Shut down the tserver.
+ ts->Shutdown();
+ // Get an error because the tserver is not presumed dead by masters.
+ {
+ string out;
+ string err;
+ Status s = RunActionStdoutStderrString(
+ Substitute("tserver unregister $0 $1", master_addrs_str, ts_uuid),
&out, &err);
+ ASSERT_TRUE(s.IsRuntimeError());
+ ASSERT_STR_CONTAINS(err, ts_uuid);
+ }
+ // The ksck output contains the dead tserver.
+ {
+ string out;
+ string err;
+ Status s =
+ RunActionStdoutStderrString(Substitute("cluster ksck $0",
master_addrs_str), &out, &err);
+ ASSERT_TRUE(s.IsRuntimeError());
+ ASSERT_STR_CONTAINS(out, Substitute("$0 | $1 | UNAVAILABLE", ts_uuid,
ts_hostport));
+ }
+
+ // We could force unregister the tserver.
+ ASSERT_OK(RunKuduTool(
+ {"tserver", "unregister", master_addrs_str, ts_uuid,
"-force_unregister_live_tserver"}));
+ {
+ string out;
+ NO_FATALS(RunActionStdoutString(Substitute("cluster ksck $0",
master_addrs_str), &out));
+ ASSERT_STR_NOT_CONTAINS(out, ts_uuid);
+ }
+
+ // After several hearbeat intervals, the tserver still does not appear in
ksck output.
+ SleepFor(MonoDelta::FromMilliseconds(3 * FLAGS_heartbeat_interval_ms));
+ {
+ string out;
+ NO_FATALS(RunActionStdoutString(Substitute("cluster ksck $0",
master_addrs_str), &out));
+ ASSERT_STR_NOT_CONTAINS(out, ts_uuid);
+ }
+}
} // namespace tools
} // namespace kudu
diff --git a/src/kudu/tools/tool_action_common.cc
b/src/kudu/tools/tool_action_common.cc
index a8cfb03..a89e0b4 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ b/src/kudu/tools/tool_action_common.cc
@@ -23,8 +23,10 @@
#include <iomanip>
#include <iostream>
#include <iterator>
+#include <map>
#include <memory>
#include <numeric>
+#include <set>
#include <stack>
#include <string>
#include <unordered_map>
@@ -60,6 +62,7 @@
#include "kudu/gutil/strings/substitute.h"
#include "kudu/gutil/strings/util.h"
#include "kudu/master/master.h"
+#include "kudu/master/master.pb.h"
#include "kudu/master/master.proxy.h" // IWYU pragma: keep
#include "kudu/rpc/messenger.h"
#include "kudu/rpc/response_callback.h"
@@ -196,6 +199,8 @@ using kudu::consensus::ReplicateMsg;
using kudu::log::LogEntryPB;
using kudu::log::LogEntryReader;
using kudu::log::ReadableLogSegment;
+using kudu::master::ConnectToMasterRequestPB;
+using kudu::master::ConnectToMasterResponsePB;
using kudu::master::MasterServiceProxy;
using kudu::pb_util::SecureDebugString;
using kudu::pb_util::SecureShortDebugString;
@@ -222,7 +227,9 @@ using kudu::tserver::TabletServerServiceProxy; // NOLINT
using kudu::tserver::WriteRequestPB;
using std::cout;
using std::endl;
+using std::map;
using std::ostream;
+using std::set;
using std::setfill;
using std::setw;
using std::shared_ptr;
@@ -744,6 +751,61 @@ Status MasterAddressesToSet(
return Status::OK();
}
+Status VerifyMasterAddressList(const vector<string>& master_addresses) {
+ map<string, set<string>> addresses_per_master;
+ for (const auto& address : master_addresses) {
+ unique_ptr<MasterServiceProxy> proxy;
+ RETURN_NOT_OK(BuildProxy(address, master::Master::kDefaultPort, &proxy));
+
+ RpcController ctl;
+ ctl.set_timeout(MonoDelta::FromMilliseconds(FLAGS_timeout_ms));
+ ConnectToMasterRequestPB req;
+ ConnectToMasterResponsePB resp;
+ RETURN_NOT_OK(proxy->ConnectToMaster(req, &resp, &ctl));
+ const auto& resp_master_addrs = resp.master_addrs();
+ if (resp_master_addrs.size() != master_addresses.size()) {
+ const auto addresses_provided = JoinStrings(master_addresses, ",");
+ const auto addresses_cluster_config =
+ JoinMapped(resp_master_addrs,
+ [](const HostPortPB& pb) { return Substitute("$0:$1",
pb.host(), pb.port()); },
+ ",");
+ return Status::InvalidArgument(
+ Substitute("list of master addresses provided ($0) "
+ "does not match the actual cluster configuration ($1) ",
+ addresses_provided,
+ addresses_cluster_config));
+ }
+ set<string> addr_set;
+ for (const auto& hp : resp_master_addrs) {
+ addr_set.emplace(Substitute("$0:$1", hp.host(), hp.port()));
+ }
+ addresses_per_master.emplace(address, std::move(addr_set));
+ }
+
+ bool mismatch = false;
+ if (addresses_per_master.size() > 1) {
+ const auto it_0 = addresses_per_master.cbegin();
+ auto it_1 = addresses_per_master.begin();
+ ++it_1;
+ for (auto it = it_1; it != addresses_per_master.end(); ++it) {
+ if (it->second != it_0->second) {
+ mismatch = true;
+ break;
+ }
+ }
+ }
+
+ if (mismatch) {
+ string err_msg = Substitute("specified: ($0);",
JoinStrings(master_addresses, ","));
+ for (const auto& e : addresses_per_master) {
+ err_msg += Substitute(" from master $0: ($1);", e.first,
JoinStrings(e.second, ","));
+ }
+ return Status::ConfigurationError(Substitute("master address lists
mismatch: $0", err_msg));
+ }
+
+ return Status::OK();
+}
+
Status PrintServerStatus(const string& address, uint16_t default_port) {
ServerStatusPB status;
RETURN_NOT_OK(GetServerStatus(address, default_port, &status));
diff --git a/src/kudu/tools/tool_action_common.h
b/src/kudu/tools/tool_action_common.h
index 562d426..d89b55c 100644
--- a/src/kudu/tools/tool_action_common.h
+++ b/src/kudu/tools/tool_action_common.h
@@ -236,6 +236,11 @@ Status MasterAddressesToSet(
const std::string& master_addresses_arg,
kudu::UnorderedHostPortSet* res);
+// Make sure the list of master addresses specified in 'master_addresses'
+// corresponds to the actual list of masters addresses in the cluster,
+// as reported in ConnectToMasterResponsePB::master_addrs.
+Status VerifyMasterAddressList(const std::vector<std::string>&
master_addresses);
+
// A table of data to present to the user.
//
// Supports formatting based on the --format flag.
diff --git a/src/kudu/tools/tool_action_master.cc
b/src/kudu/tools/tool_action_master.cc
index 4764be1..48cb94a 100644
--- a/src/kudu/tools/tool_action_master.cc
+++ b/src/kudu/tools/tool_action_master.cc
@@ -89,8 +89,6 @@ DEFINE_string(kudu_abs_path, "", "Absolute file path of the
'kudu' executable us
using kudu::master::AddMasterRequestPB;
using kudu::master::AddMasterResponsePB;
-using kudu::master::ConnectToMasterRequestPB;
-using kudu::master::ConnectToMasterResponsePB;
using kudu::master::ListMastersRequestPB;
using kudu::master::ListMastersResponsePB;
using kudu::master::Master;
@@ -605,67 +603,6 @@ Status MasterDumpMemTrackers(const RunnerContext& context)
{
return DumpMemTrackers(address, Master::kDefaultPort);
}
-// Make sure the list of master addresses specified in 'master_addresses'
-// corresponds to the actual list of masters addresses in the cluster,
-// as reported in ConnectToMasterResponsePB::master_addrs.
-Status VerifyMasterAddressList(const vector<string>& master_addresses) {
- map<string, set<string>> addresses_per_master;
- for (const auto& address : master_addresses) {
- unique_ptr<MasterServiceProxy> proxy;
- RETURN_NOT_OK(BuildProxy(address, Master::kDefaultPort, &proxy));
-
- RpcController ctl;
- ctl.set_timeout(MonoDelta::FromMilliseconds(FLAGS_timeout_ms));
- ConnectToMasterRequestPB req;
- ConnectToMasterResponsePB resp;
- RETURN_NOT_OK(proxy->ConnectToMaster(req, &resp, &ctl));
- const auto& resp_master_addrs = resp.master_addrs();
- if (resp_master_addrs.size() != master_addresses.size()) {
- const auto addresses_provided = JoinStrings(master_addresses, ",");
- const auto addresses_cluster_config = JoinMapped(
- resp_master_addrs,
- [](const HostPortPB& pb) {
- return Substitute("$0:$1", pb.host(), pb.port());
- }, ",");
- return Status::InvalidArgument(Substitute(
- "list of master addresses provided ($0) "
- "does not match the actual cluster configuration ($1) ",
- addresses_provided, addresses_cluster_config));
- }
- set<string> addr_set;
- for (const auto& hp : resp_master_addrs) {
- addr_set.emplace(Substitute("$0:$1", hp.host(), hp.port()));
- }
- addresses_per_master.emplace(address, std::move(addr_set));
- }
-
- bool mismatch = false;
- if (addresses_per_master.size() > 1) {
- const auto it_0 = addresses_per_master.cbegin();
- auto it_1 = addresses_per_master.begin();
- ++it_1;
- for (auto it = it_1; it != addresses_per_master.end(); ++it) {
- if (it->second != it_0->second) {
- mismatch = true;
- break;
- }
- }
- }
-
- if (mismatch) {
- string err_msg = Substitute("specified: ($0);",
- JoinStrings(master_addresses, ","));
- for (const auto& e : addresses_per_master) {
- err_msg += Substitute(" from master $0: ($1);",
- e.first, JoinStrings(e.second, ","));
- }
- return Status::ConfigurationError(
- Substitute("master address lists mismatch: $0", err_msg));
- }
-
- return Status::OK();
-}
-
Status PrintRebuildReport(const RebuildReport& rebuild_report) {
cout << "Rebuild Report" << endl;
cout << "Tablet Servers" << endl;
diff --git a/src/kudu/tools/tool_action_tserver.cc
b/src/kudu/tools/tool_action_tserver.cc
index 1b3ab8e..7947395 100644
--- a/src/kudu/tools/tool_action_tserver.cc
+++ b/src/kudu/tools/tool_action_tserver.cc
@@ -34,6 +34,7 @@
#include "kudu/gutil/strings/numbers.h"
#include "kudu/gutil/strings/split.h"
#include "kudu/gutil/strings/substitute.h"
+#include "kudu/master/master.h"
#include "kudu/master/master.pb.h"
#include "kudu/master/master.proxy.h"
#include "kudu/rpc/response_callback.h"
@@ -57,6 +58,14 @@ DEFINE_bool(error_if_not_fully_quiesced, false, "If true,
the command to start "
"quiescing will return an error if the tserver is not fully quiesced, i.e.
"
"there are still tablet leaders or active scanners on it.");
+DEFINE_bool(force_unregister_live_tserver, false,
+ "If true, force the unregistration of the tserver even if it is
not presumed dead "
+ "by the master. Make sure the tserver has been shut down before
setting this true.");
+DEFINE_bool(remove_tserver_state, true,
+ "If false, remove the tserver from the master's in-memory map but
keep its persisted "
+ "state (if any). If the same tserver re-registers on the master it
will get its "
+ "original state");
+
DECLARE_string(columns);
using std::cout;
@@ -72,6 +81,8 @@ using master::ChangeTServerStateResponsePB;
using master::ListTabletServersRequestPB;
using master::ListTabletServersResponsePB;
using master::MasterServiceProxy;
+using master::UnregisterTServerRequestPB;
+using master::UnregisterTServerResponsePB;
using master::TServerStateChangePB;
using rpc::RpcController;
using tserver::QuiesceTabletServerRequestPB;
@@ -296,6 +307,39 @@ Status QuiescingStatus(const RunnerContext& context) {
return table.PrintTo(cout);
}
+Status UnregisterTServer(const RunnerContext& context) {
+ const auto& ts_uuid = FindOrDie(context.required_args, kTServerIdArg);
+ vector<string> master_addresses;
+ RETURN_NOT_OK(ParseMasterAddresses(context, &master_addresses));
+ RETURN_NOT_OK(VerifyMasterAddressList(master_addresses));
+ if (FLAGS_remove_tserver_state) {
+ // We don't care about FLAGS_allow_missing_tserver because it doesn't
+ // make sense for ExitMaintenance.
+ RETURN_NOT_OK(ExitMaintenance(context));
+ }
+
+ string err_str;
+ for (const auto& address : master_addresses) {
+ unique_ptr<MasterServiceProxy> proxy;
+ RETURN_NOT_OK(BuildProxy(address, master::Master::kDefaultPort, &proxy));
+ UnregisterTServerRequestPB req;
+ req.set_uuid(ts_uuid);
+ req.set_force_unregister_live_tserver(FLAGS_force_unregister_live_tserver);
+ UnregisterTServerResponsePB resp;
+ RpcController rpc;
+ Status s = proxy->UnregisterTServer(req, &resp, &rpc);
+ if (!s.ok() || resp.has_error()) {
+ err_str += Substitute(" Unable to unregister the tserver from master $0,
status: $1",
+ address,
+ StatusFromPB(resp.error().status()).ToString());
+ }
+ }
+ if (err_str.empty()) {
+ return Status::OK();
+ }
+ return Status::RemoteError(err_str);
+}
+
} // anonymous namespace
unique_ptr<Mode> BuildTServerMode() {
@@ -412,6 +456,15 @@ unique_ptr<Mode> BuildTServerMode() {
.AddAction(std::move(exit_maintenance))
.Build();
+ unique_ptr<Action> unregister_tserver =
+ ClusterActionBuilder("unregister", &UnregisterTServer)
+ .Description(
+ "Unregister a tablet server from the master's in-memory state
and system catalog.")
+ .AddRequiredParameter({kTServerIdArg, kTServerIdDesc})
+ .AddOptionalParameter("force_unregister_live_tserver")
+ .AddOptionalParameter("remove_tserver_state")
+ .Build();
+
return ModeBuilder("tserver")
.Description("Operate on a Kudu Tablet Server")
.AddAction(std::move(dump_memtrackers))
@@ -421,6 +474,7 @@ unique_ptr<Mode> BuildTServerMode() {
.AddAction(std::move(status))
.AddAction(std::move(timestamp))
.AddAction(std::move(list_tservers))
+ .AddAction(std::move(unregister_tserver))
.AddMode(std::move(quiesce))
.AddMode(std::move(state))
.Build();
