This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 10167d19e feat(dns_resolver): add metrics for dns_resolver (#1873)
10167d19e is described below
commit 10167d19ed27beb9cd153633e70c81ed02102c1f
Author: Yingchun Lai <[email protected]>
AuthorDate: Tue Jan 30 15:26:23 2024 +0800
feat(dns_resolver): add metrics for dns_resolver (#1873)
The DNS resolve function maybe slow, and the memory size of dns_resolver is
not limited currently, this patch adds 3 useful metrics to improve
observability:
- dns_resolver_cache_size
- dns_resolver_resolve_duration_ns
- dns_resolver_resolve_by_dns_duration_ns
---
src/runtime/rpc/dns_resolver.cpp | 73 ++++++++++++++++++++++++++++-----------
src/runtime/rpc/dns_resolver.h | 21 +++++++----
src/runtime/rpc/rpc_host_port.cpp | 3 ++
3 files changed, 71 insertions(+), 26 deletions(-)
diff --git a/src/runtime/rpc/dns_resolver.cpp b/src/runtime/rpc/dns_resolver.cpp
index 8ccbdac56..06bf4aca1 100644
--- a/src/runtime/rpc/dns_resolver.cpp
+++ b/src/runtime/rpc/dns_resolver.cpp
@@ -19,27 +19,52 @@
#include <algorithm>
#include <memory>
+#include <set>
#include <utility>
+#include "absl/strings/string_view.h"
+#include "fmt/core.h"
#include "fmt/format.h"
#include "runtime/rpc/dns_resolver.h"
#include "runtime/rpc/group_address.h"
#include "runtime/rpc/group_host_port.h"
+#include "utils/autoref_ptr.h"
#include "utils/fmt_logging.h"
+METRIC_DEFINE_gauge_int64(server,
+ dns_resolver_cache_size,
+ dsn::metric_unit::kKeys,
+ "The size of the host_port to rpc_address resolve
results cache");
+
+METRIC_DEFINE_percentile_int64(
+ server,
+ dns_resolver_resolve_duration_ns,
+ dsn::metric_unit::kNanoSeconds,
+ "The duration of resolving a host port, may either get from cache or
resolve by DNS lookup");
+
+METRIC_DEFINE_percentile_int64(server,
+ dns_resolver_resolve_by_dns_duration_ns,
+ dsn::metric_unit::kNanoSeconds,
+ "The duration of resolving a host port by DNS
lookup");
namespace dsn {
-void dns_resolver::add_item(const host_port &hp, const rpc_address &addr)
+dns_resolver::dns_resolver()
+ : METRIC_VAR_INIT_server(dns_resolver_cache_size),
+ METRIC_VAR_INIT_server(dns_resolver_resolve_duration_ns),
+ METRIC_VAR_INIT_server(dns_resolver_resolve_by_dns_duration_ns)
{
- utils::auto_write_lock l(_lock);
- _dsn_cache.insert(std::make_pair(hp, addr));
+#ifndef MOCK_TEST
+ static int only_one_instance = 0;
+ only_one_instance++;
+ CHECK_EQ_MSG(1, only_one_instance, "dns_resolver should only created
once!");
+#endif
}
bool dns_resolver::get_cached_addresses(const host_port &hp,
std::vector<rpc_address> &addresses)
{
utils::auto_read_lock l(_lock);
- const auto &found = _dsn_cache.find(hp);
- if (found == _dsn_cache.end()) {
+ const auto &found = _dns_cache.find(hp);
+ if (found == _dns_cache.end()) {
return false;
}
@@ -55,19 +80,26 @@ error_s dns_resolver::resolve_addresses(const host_port
&hp, std::vector<rpc_add
}
std::vector<rpc_address> resolved_addresses;
- RETURN_NOT_OK(hp.resolve_addresses(resolved_addresses));
+ {
+ METRIC_VAR_AUTO_LATENCY(dns_resolver_resolve_by_dns_duration_ns);
+ RETURN_NOT_OK(hp.resolve_addresses(resolved_addresses));
+ }
{
- utils::auto_write_lock l(_lock);
if (resolved_addresses.size() > 1) {
- LOG_DEBUG(
- "host_port '{}' resolves to {} different addresses {}, using
the first one {}.",
- hp,
- resolved_addresses.size(),
- fmt::join(resolved_addresses, ","),
- resolved_addresses[0]);
+ LOG_DEBUG("host_port '{}' resolves to {} different addresses {},
only the first one {} "
+ "will be cached.",
+ hp,
+ resolved_addresses.size(),
+ fmt::join(resolved_addresses, ","),
+ resolved_addresses[0]);
+ }
+
+ utils::auto_write_lock l(_lock);
+ const auto it = _dns_cache.insert(std::make_pair(hp,
resolved_addresses[0]));
+ if (it.second) {
+ METRIC_VAR_INCREMENT(dns_resolver_cache_size);
}
- _dsn_cache.insert(std::make_pair(hp, resolved_addresses[0]));
}
addresses = std::move(resolved_addresses);
@@ -76,18 +108,19 @@ error_s dns_resolver::resolve_addresses(const host_port
&hp, std::vector<rpc_add
rpc_address dns_resolver::resolve_address(const host_port &hp)
{
+ METRIC_VAR_AUTO_LATENCY(dns_resolver_resolve_duration_ns);
switch (hp.type()) {
case HOST_TYPE_GROUP: {
rpc_address addr;
- auto group_address = hp.group_host_port();
- addr.assign_group(group_address->name());
+ auto hp_group = hp.group_host_port();
+ addr.assign_group(hp_group->name());
- for (const auto &hp : group_address->members()) {
+ for (const auto &hp : hp_group->members()) {
CHECK_TRUE(addr.group_address()->add(resolve_address(hp)));
}
addr.group_address()->set_update_leader_automatically(
- group_address->is_update_leader_automatically());
-
addr.group_address()->set_leader(resolve_address(group_address->leader()));
+ hp_group->is_update_leader_automatically());
+ addr.group_address()->set_leader(resolve_address(hp_group->leader()));
return addr;
}
case HOST_TYPE_IPV4: {
@@ -104,7 +137,7 @@ rpc_address dns_resolver::resolve_address(const host_port
&hp)
return addresses[0];
}
default:
- return rpc_address();
+ return {};
}
}
diff --git a/src/runtime/rpc/dns_resolver.h b/src/runtime/rpc/dns_resolver.h
index 7cf47895c..aadb3287f 100644
--- a/src/runtime/rpc/dns_resolver.h
+++ b/src/runtime/rpc/dns_resolver.h
@@ -25,17 +25,22 @@
#include "runtime/rpc/rpc_address.h"
#include "runtime/rpc/rpc_host_port.h"
#include "utils/errors.h"
+#include "utils/metrics.h"
#include "utils/synchronize.h"
namespace dsn {
// This class provide a way to resolve host_port to rpc_address.
+// Now each host_post will be resolved just once, and then cached the first
rpc_address result in
+// the resolved result list.
+// If some host_port's rpc_address changes, you need to restart the Pegasus
process to make it take
+// effect.
+// TODO(yingchun): Now the cache is unlimited, the cache size may be huge.
Implement an expiration
+// mechanism to limit the cache size and make it possible to update the
resolve result.
class dns_resolver
{
public:
- explicit dns_resolver() = default;
-
- void add_item(const host_port &hp, const rpc_address &addr);
+ explicit dns_resolver();
// Resolve this host_port to an unique rpc_address.
rpc_address resolve_address(const host_port &hp);
@@ -45,10 +50,14 @@ private:
error_s resolve_addresses(const host_port &hp, std::vector<rpc_address>
&addresses);
- error_s do_resolution(const host_port &hp, std::vector<rpc_address>
&addresses);
-
mutable utils::rw_lock_nr _lock;
- std::unordered_map<host_port, rpc_address> _dsn_cache;
+ // Cache the host_port resolve results, the cached rpc_address is the
first one in the resolved
+ // list.
+ std::unordered_map<host_port, rpc_address> _dns_cache;
+
+ METRIC_VAR_DECLARE_gauge_int64(dns_resolver_cache_size);
+ METRIC_VAR_DECLARE_percentile_int64(dns_resolver_resolve_duration_ns);
+
METRIC_VAR_DECLARE_percentile_int64(dns_resolver_resolve_by_dns_duration_ns);
};
} // namespace dsn
diff --git a/src/runtime/rpc/rpc_host_port.cpp
b/src/runtime/rpc/rpc_host_port.cpp
index 2da05e7b8..8be334fd2 100644
--- a/src/runtime/rpc/rpc_host_port.cpp
+++ b/src/runtime/rpc/rpc_host_port.cpp
@@ -180,6 +180,9 @@ error_s
host_port::resolve_addresses(std::vector<rpc_address> &addresses) const
return error_s::make(dsn::ERR_INVALID_STATE, "invalid host_port type:
HOST_TYPE_GROUP");
case HOST_TYPE_IPV4:
break;
+ default:
+ CHECK(false, "");
+ __builtin_unreachable();
}
rpc_address rpc_addr;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
