--- 00-README.conf | 14 +++ src/base/Makefile.am | 1 + src/base/statistics.h | 88 +++++++++++++ src/mds/Makefile.am | 8 +- src/mds/mds_dt_tipc.c | 3 + src/mds/mds_tipc_recvq_stats.cc | 29 +++++ src/mds/mds_tipc_recvq_stats.h | 32 +++++ src/mds/mds_tipc_recvq_stats_impl.cc | 178 +++++++++++++++++++++++++++ src/mds/mds_tipc_recvq_stats_impl.h | 39 ++++++ 9 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 src/base/statistics.h create mode 100644 src/mds/mds_tipc_recvq_stats.cc create mode 100644 src/mds/mds_tipc_recvq_stats.h create mode 100644 src/mds/mds_tipc_recvq_stats_impl.cc create mode 100644 src/mds/mds_tipc_recvq_stats_impl.h
diff --git a/00-README.conf b/00-README.conf index 8f20e5209..da1825f06 100644 --- a/00-README.conf +++ b/00-README.conf @@ -737,3 +737,17 @@ initiate a 'self-fencing' by rebooting the node, if it determines the node should no longer be active according to the consensus service, to prevent a split-brain situation. +TIPC receive queue utilization +============================== + +If setting the environment variable MDS_RECVQ_STATS_LOG_FREQ_SEC in a service config +file enables TIPC receive queue utilisation statistics. The argument is how often the +statistics will be written to syslog. + +Example amfd.conf: + +export MDS_RECVQ_STATS_LOG_FREQ_SEC=5 + +then every 5 seconds a log record is written: + +May 20 12:23:30 SC-1 local0.notice osafamfd[545]: NO TIPC receive queue utilization (in %): min: 3.86 max: 4.38 mean: 4.15 std dev: 0.18 diff --git a/src/base/Makefile.am b/src/base/Makefile.am index ce93562e5..025fb86a2 100644 --- a/src/base/Makefile.am +++ b/src/base/Makefile.am @@ -157,6 +157,7 @@ noinst_HEADERS += \ src/base/saf_error.h \ src/base/saf_mem.h \ src/base/sprr_dl_api.h \ + src/base/statistics.h \ src/base/string_parse.h \ src/base/sysf_exc_scr.h \ src/base/sysf_ipc.h \ diff --git a/src/base/statistics.h b/src/base/statistics.h new file mode 100644 index 000000000..9ce980fc1 --- /dev/null +++ b/src/base/statistics.h @@ -0,0 +1,88 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2019 The OpenSAF Foundation + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef STATISTICS_H_ +#define STATISTICS_H_ + +#include <cmath> + +namespace base { + +class Statistics { + public: + void clear() { + n_ = 0; + } + + void push(double x) { + n_++; + + // See Knuth, Art Of Computer Programming, Volume 2. The Seminumerical Algorithms, 4.2.2. Accuracy of Floating Point Arithmetic, + // using the recurrence formulas: + // M1 = x1, Mk = Mk-1 + (xk - Mk-1) / k (15) + // S1 = 0, Sk = Sk-1 + (xk - Mk-1) * (xk - Mk) (16) + // for 2 <= k <= n, sqrt(Sn/(n-1) + if (n_ == 1) { + prev_m_ = current_m_ = x; + prev_s_ = 0; + min_ = x; + max_ = x; + } else { + current_m_ = prev_m_ + (x - prev_m_) / n_; + current_s_ = prev_s_ + (x - prev_m_) * (x - current_m_); + + if (x > max_) max_ = x; + if (x < min_) min_ = x; + prev_m_ = current_m_; + prev_s_ = current_s_; + } + } + + double mean() const { + return (n_ > 0) ? current_m_ : 0; + } + + double variance() const { + return (n_ > 1) ? current_s_ / (n_ - 1) : 0; + } + + double std_dev() const { + return sqrt(variance()); + } + + double min() const { + return min_; + } + double max() const { + return max_; + } + + private: + int n_{0}; + double prev_m_{0}; + double current_m_{0}; + double prev_s_{0}; + double current_s_{0}; + double min_{0}; + double max_{0}; +}; + +} // namespace base + +#endif // STATISTICS_H_ + diff --git a/src/mds/Makefile.am b/src/mds/Makefile.am index 3724d2ea8..2d7b652e9 100644 --- a/src/mds/Makefile.am +++ b/src/mds/Makefile.am @@ -46,8 +46,12 @@ lib_libopensaf_core_la_SOURCES += \ src/mds/ncs_vda.c if ENABLE_TIPC_TRANSPORT -noinst_HEADERS += src/mds/mds_dt_tipc.h -lib_libopensaf_core_la_SOURCES += src/mds/mds_dt_tipc.c +noinst_HEADERS += src/mds/mds_dt_tipc.h \ + src/mds/mds_tipc_recvq_stats.h \ + src/mds/mds_tipc_recvq_stats_impl.h +lib_libopensaf_core_la_SOURCES += src/mds/mds_dt_tipc.c \ + src/mds/mds_tipc_recvq_stats.cc \ + src/mds/mds_tipc_recvq_stats_impl.cc endif if ENABLE_TESTS diff --git a/src/mds/mds_dt_tipc.c b/src/mds/mds_dt_tipc.c index d8f8c783e..7ddc9bf92 100644 --- a/src/mds/mds_dt_tipc.c +++ b/src/mds/mds_dt_tipc.c @@ -47,6 +47,7 @@ #include "mds_dt_tipc.h" #include "mds_dt_tcp_disc.h" #include "mds_core.h" +#include "mds_tipc_recvq_stats.h" #include "base/osaf_utility.h" #include "base/osaf_poll.h" @@ -366,6 +367,8 @@ uint32_t mdtm_tipc_init(NODE_ID nodeid, uint32_t *mds_tipc_ref) "MDTM: Successfully set TIPC_DEST_DROPPABLE to zero"); } + mds_tipc_recvq_stats(tipc_cb.BSRsock); + return NCSCC_RC_SUCCESS; } diff --git a/src/mds/mds_tipc_recvq_stats.cc b/src/mds/mds_tipc_recvq_stats.cc new file mode 100644 index 000000000..6e66b5358 --- /dev/null +++ b/src/mds/mds_tipc_recvq_stats.cc @@ -0,0 +1,29 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2019 The OpenSAF Foundation + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "mds_tipc_recvq_stats.h" +#include "mds_tipc_recvq_stats_impl.h" + + +void mds_tipc_recvq_stats(int sd) { + static TipcRecvqStatsImpl tipc_recvq_stats; + + if (tipc_recvq_stats.init(sd) == 0) { + tipc_recvq_stats.start(); + } +} diff --git a/src/mds/mds_tipc_recvq_stats.h b/src/mds/mds_tipc_recvq_stats.h new file mode 100644 index 000000000..c70834203 --- /dev/null +++ b/src/mds/mds_tipc_recvq_stats.h @@ -0,0 +1,32 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2019 The OpenSAF Foundation + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef MDS_TIPC_RECVQ_STATS_H_ +#define MDS_TIPC_RECVQ_STATS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void mds_tipc_recvq_stats(int sd); + +#ifdef __cplusplus +} +#endif + +#endif // MDS_TIPC_RECVQ_STATS_H_ diff --git a/src/mds/mds_tipc_recvq_stats_impl.cc b/src/mds/mds_tipc_recvq_stats_impl.cc new file mode 100644 index 000000000..df86dc85e --- /dev/null +++ b/src/mds/mds_tipc_recvq_stats_impl.cc @@ -0,0 +1,178 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2019 The OpenSAF Foundation + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "mds_tipc_recvq_stats_impl.h" +#include <errno.h> +#include <linux/tipc.h> +#include <poll.h> +#include <sys/socket.h> +#include <sys/timerfd.h> +#include <sys/types.h> +#include <unistd.h> +#include <cinttypes> +#include <cstdlib> +#include <cstring> +#include <cstdint> +#include <thread> +#include "base/osaf_time.h" +#include "base/statistics.h" +#include "base/logtrace.h" + +void TipcRecvqStatsImpl::start() { + std::thread(&TipcRecvqStatsImpl::tipc_recvq_stats_bg, this).detach(); +} + +int TipcRecvqStatsImpl::init(int sd) { + int optval{0}; + socklen_t optlen = sizeof(optval); + long log_freq_sec{0}; + + sd_ = sd; + recvq_size_ = 0; + char *val; + + if ((val = getenv("MDS_RECVQ_STATS_LOG_FREQ_SEC")) != NULL) { + log_freq_sec = strtol(val, NULL, 0); + log_freq_ = log_freq_sec * 10; + if (log_freq_sec < 1) { + LOG_NO("MDS_RECVQ_STATS_LOG_FREQ_SEC value is set too low: %ld seconds", log_freq_sec); + return -1; + } + } else { + return -1; + } + + if (getsockopt(sd_, SOL_SOCKET, SO_RCVBUF, &optval, &optlen) < 0) { + LOG_NO("TIPC getsockopt failed: %s", strerror(errno)); + return -1; + } + + recvq_size_ = optval; + + return get_tipc_recvq_used(&optval); +} + +int TipcRecvqStatsImpl::get_tipc_recvq_used(int *optval) { +#ifndef TIPC_SOCK_RECVQ_USED +#define TIPC_SOCK_RECVQ_USED 137 +#endif + + socklen_t optlen = sizeof(optval); + + if (getsockopt(sd_, SOL_TIPC, TIPC_SOCK_RECVQ_USED, optval, &optlen) == 0) { + return 0; + } else { + LOG_NO("TIPC getsockopt failed: %s", strerror(errno)); + return -1; + } +} + +int TipcRecvqStatsImpl::create_timer() { + if ((timer_fd_ = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC)) < 0) { + LOG_NO("timerfd_create failed: %s", strerror(errno)); + return -1; + } + return 0; +} + +int TipcRecvqStatsImpl::start_timer() { + uint64_t interval_msec{100}; + struct itimerspec spec; + spec.it_interval.tv_sec = interval_msec / kMillisPerSec; + spec.it_interval.tv_nsec = (interval_msec % kMillisPerSec) * (kNanosPerSec / kMillisPerSec); + spec.it_value.tv_sec = spec.it_interval.tv_sec; + spec.it_value.tv_nsec = spec.it_interval.tv_nsec; + + if (timerfd_settime(timer_fd_, 0, &spec, NULL) < 0) { + LOG_NO("timerfd_settime failed: %s", strerror(errno)); + return -1; + } + return 0; +} + +int TipcRecvqStatsImpl::stop_timer() { + struct itimerspec spec; + + spec.it_interval.tv_sec = 0; + spec.it_interval.tv_nsec = 0; + spec.it_value.tv_sec = 0; + spec.it_value.tv_nsec = 0; + + if (timerfd_settime(timer_fd_, 0, &spec, NULL) < 0) { + LOG_NO("timerfd_settime failed: %s", strerror(errno)); + return -1; + } + return 0; +} + +void TipcRecvqStatsImpl::tipc_recvq_stats_bg() { + base::Statistics stats; + int optval; + int ticks{0}; + + enum { + FD_TMR = 0, + NUM_FDS + }; + + create_timer(); + + struct pollfd fds[NUM_FDS]; + fds[FD_TMR].fd = timer_fd_; + fds[FD_TMR].events = POLLIN; + + start_timer(); + + LOG_NO("TIPC receive queue statistics started"); + + while (true) { + int rc = poll(fds, NUM_FDS, -1); + if (rc == -1) { + if (errno == EINTR) continue; + LOG_IN("poll failed: %s", strerror(errno)); + break; + } + if (rc > 0) { + if (fds[FD_TMR].revents == POLLIN) { + uint64_t expirations = 0; + if (read(timer_fd_, &expirations, 8) != 8) { + LOG_NO("error reading timerfd value: %s", strerror(errno)); + return; + } else { + if (expirations != 1) { + LOG_NO("timerfd expired %" PRIu64 " times", expirations); + } + } + if (get_tipc_recvq_used(&optval) == 0) { + ++ticks; + stats.push((optval / recvq_size_) * 100.0); + if (ticks >= log_freq_) { + LOG_NO("TIPC receive queue utilization (in %%): min: %2.2f max: %2.2f mean: %2.2f std dev: %2.2f", + stats.min(), stats.max(), stats.mean(), stats.std_dev()); + ticks = 0; + stats.clear(); + } + } else { + break; + } + } + } + } + return; +} + diff --git a/src/mds/mds_tipc_recvq_stats_impl.h b/src/mds/mds_tipc_recvq_stats_impl.h new file mode 100644 index 000000000..fefe2751d --- /dev/null +++ b/src/mds/mds_tipc_recvq_stats_impl.h @@ -0,0 +1,39 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2019 The OpenSAF Foundation + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef MDS_TIPC_RECVQ_STATS_IMPL_ +#define MDS_TIPC_RECVQ_STATS_IMPL_ + +class TipcRecvqStatsImpl { + public: + int init(int sd); + void start(); + private: + int get_tipc_recvq_used(int *optval); + void tipc_recvq_stats_bg(); + int create_timer(); + int start_timer(); + int stop_timer(); + + int sd_{-1}; + double recvq_size_{0}; + int timer_fd_{-1}; + long log_freq_{0}; +}; + +#endif // MDS_TIPC_RECVQ_STATS_IMPL_ -- 2.17.1 _______________________________________________ Opensaf-devel mailing list Opensaf-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/opensaf-devel