---
00-README.conf | 14 +++
src/base/Makefile.am | 1 +
src/base/statistics.h | 88 +++++++++++++
src/mds/Makefile.am | 8 +-
src/mds/mds_dt_tipc.c | 3 +
src/mds/mds_tipc_recvq_stats.cc | 29 +++++
src/mds/mds_tipc_recvq_stats.h | 32 +++++
src/mds/mds_tipc_recvq_stats_impl.cc | 178 +++++++++++++++++++++++++++
src/mds/mds_tipc_recvq_stats_impl.h | 39 ++++++
9 files changed, 390 insertions(+), 2 deletions(-)
create mode 100644 src/base/statistics.h
create mode 100644 src/mds/mds_tipc_recvq_stats.cc
create mode 100644 src/mds/mds_tipc_recvq_stats.h
create mode 100644 src/mds/mds_tipc_recvq_stats_impl.cc
create mode 100644 src/mds/mds_tipc_recvq_stats_impl.h
diff --git a/00-README.conf b/00-README.conf
index 8f20e5209..da1825f06 100644
--- a/00-README.conf
+++ b/00-README.conf
@@ -737,3 +737,17 @@ initiate a 'self-fencing' by rebooting the node, if it
determines the node
should no longer be active according to the consensus service, to prevent
a split-brain situation.
+TIPC receive queue utilization
+==============================
+
+If setting the environment variable MDS_RECVQ_STATS_LOG_FREQ_SEC in a service
config
+file enables TIPC receive queue utilisation statistics. The argument is how
often the
+statistics will be written to syslog.
+
+Example amfd.conf:
+
+export MDS_RECVQ_STATS_LOG_FREQ_SEC=5
+
+then every 5 seconds a log record is written:
+
+May 20 12:23:30 SC-1 local0.notice osafamfd[545]: NO TIPC receive queue
utilization (in %): min: 3.86 max: 4.38 mean: 4.15 std dev: 0.18
diff --git a/src/base/Makefile.am b/src/base/Makefile.am
index ce93562e5..025fb86a2 100644
--- a/src/base/Makefile.am
+++ b/src/base/Makefile.am
@@ -157,6 +157,7 @@ noinst_HEADERS += \
src/base/saf_error.h \
src/base/saf_mem.h \
src/base/sprr_dl_api.h \
+ src/base/statistics.h \
src/base/string_parse.h \
src/base/sysf_exc_scr.h \
src/base/sysf_ipc.h \
diff --git a/src/base/statistics.h b/src/base/statistics.h
new file mode 100644
index 000000000..9ce980fc1
--- /dev/null
+++ b/src/base/statistics.h
@@ -0,0 +1,88 @@
+/* -*- OpenSAF -*-
+ *
+ * (C) Copyright 2019 The OpenSAF Foundation
+ * Copyright Ericsson AB 2019 - All Rights Reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed
+ * under the GNU Lesser General Public License Version 2.1, February 1999.
+ * The complete license can be accessed from the following location:
+ * http://opensource.org/licenses/lgpl-license.php
+ * See the Copying file included with the OpenSAF distribution for full
+ * licensing terms.
+ *
+ * Author(s): Ericsson AB
+ *
+ */
+
+#ifndef STATISTICS_H_
+#define STATISTICS_H_
+
+#include <cmath>
+
+namespace base {
+
+class Statistics {
+ public:
+ void clear() {
+ n_ = 0;
+ }
+
+ void push(double x) {
+ n_++;
+
+ // See Knuth, Art Of Computer Programming, Volume 2. The Seminumerical
Algorithms, 4.2.2. Accuracy of Floating Point Arithmetic,
+ // using the recurrence formulas:
+ // M1 = x1, Mk = Mk-1 + (xk - Mk-1) / k (15)
+ // S1 = 0, Sk = Sk-1 + (xk - Mk-1) * (xk - Mk) (16)
+ // for 2 <= k <= n, sqrt(Sn/(n-1)
+ if (n_ == 1) {
+ prev_m_ = current_m_ = x;
+ prev_s_ = 0;
+ min_ = x;
+ max_ = x;
+ } else {
+ current_m_ = prev_m_ + (x - prev_m_) / n_;
+ current_s_ = prev_s_ + (x - prev_m_) * (x - current_m_);
+
+ if (x > max_) max_ = x;
+ if (x < min_) min_ = x;
+ prev_m_ = current_m_;
+ prev_s_ = current_s_;
+ }
+ }
+
+ double mean() const {
+ return (n_ > 0) ? current_m_ : 0;
+ }
+
+ double variance() const {
+ return (n_ > 1) ? current_s_ / (n_ - 1) : 0;
+ }
+
+ double std_dev() const {
+ return sqrt(variance());
+ }
+
+ double min() const {
+ return min_;
+ }
+ double max() const {
+ return max_;
+ }
+
+ private:
+ int n_{0};
+ double prev_m_{0};
+ double current_m_{0};
+ double prev_s_{0};
+ double current_s_{0};
+ double min_{0};
+ double max_{0};
+};
+
+} // namespace base
+
+#endif // STATISTICS_H_
+
diff --git a/src/mds/Makefile.am b/src/mds/Makefile.am
index 3724d2ea8..2d7b652e9 100644
--- a/src/mds/Makefile.am
+++ b/src/mds/Makefile.am
@@ -46,8 +46,12 @@ lib_libopensaf_core_la_SOURCES += \
src/mds/ncs_vda.c
if ENABLE_TIPC_TRANSPORT
-noinst_HEADERS += src/mds/mds_dt_tipc.h
-lib_libopensaf_core_la_SOURCES += src/mds/mds_dt_tipc.c
+noinst_HEADERS += src/mds/mds_dt_tipc.h \
+ src/mds/mds_tipc_recvq_stats.h \
+ src/mds/mds_tipc_recvq_stats_impl.h
+lib_libopensaf_core_la_SOURCES += src/mds/mds_dt_tipc.c \
+ src/mds/mds_tipc_recvq_stats.cc \
+ src/mds/mds_tipc_recvq_stats_impl.cc
endif
if ENABLE_TESTS
diff --git a/src/mds/mds_dt_tipc.c b/src/mds/mds_dt_tipc.c
index d8f8c783e..7ddc9bf92 100644
--- a/src/mds/mds_dt_tipc.c
+++ b/src/mds/mds_dt_tipc.c
@@ -47,6 +47,7 @@
#include "mds_dt_tipc.h"
#include "mds_dt_tcp_disc.h"
#include "mds_core.h"
+#include "mds_tipc_recvq_stats.h"
#include "base/osaf_utility.h"
#include "base/osaf_poll.h"
@@ -366,6 +367,8 @@ uint32_t mdtm_tipc_init(NODE_ID nodeid, uint32_t *mds_tipc_ref)
"MDTM: Successfully set TIPC_DEST_DROPPABLE to zero");
}
+ mds_tipc_recvq_stats(tipc_cb.BSRsock);
+
return NCSCC_RC_SUCCESS;
}
diff --git a/src/mds/mds_tipc_recvq_stats.cc b/src/mds/mds_tipc_recvq_stats.cc
new file mode 100644
index 000000000..6e66b5358
--- /dev/null
+++ b/src/mds/mds_tipc_recvq_stats.cc
@@ -0,0 +1,29 @@
+/* -*- OpenSAF -*-
+ *
+ * (C) Copyright 2019 The OpenSAF Foundation
+ * Copyright Ericsson AB 2019 - All Rights Reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed
+ * under the GNU Lesser General Public License Version 2.1, February 1999.
+ * The complete license can be accessed from the following location:
+ * http://opensource.org/licenses/lgpl-license.php
+ * See the Copying file included with the OpenSAF distribution for full
+ * licensing terms.
+ *
+ * Author(s): Ericsson AB
+ *
+ */
+
+#include "mds_tipc_recvq_stats.h"
+#include "mds_tipc_recvq_stats_impl.h"
+
+
+void mds_tipc_recvq_stats(int sd) {
+ static TipcRecvqStatsImpl tipc_recvq_stats;
+
+ if (tipc_recvq_stats.init(sd) == 0) {
+ tipc_recvq_stats.start();
+ }
+}
diff --git a/src/mds/mds_tipc_recvq_stats.h b/src/mds/mds_tipc_recvq_stats.h
new file mode 100644
index 000000000..c70834203
--- /dev/null
+++ b/src/mds/mds_tipc_recvq_stats.h
@@ -0,0 +1,32 @@
+/* -*- OpenSAF -*-
+ *
+ * (C) Copyright 2019 The OpenSAF Foundation
+ * Copyright Ericsson AB 2019 - All Rights Reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed
+ * under the GNU Lesser General Public License Version 2.1, February 1999.
+ * The complete license can be accessed from the following location:
+ * http://opensource.org/licenses/lgpl-license.php
+ * See the Copying file included with the OpenSAF distribution for full
+ * licensing terms.
+ *
+ * Author(s): Ericsson AB
+ *
+ */
+
+#ifndef MDS_TIPC_RECVQ_STATS_H_
+#define MDS_TIPC_RECVQ_STATS_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void mds_tipc_recvq_stats(int sd);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // MDS_TIPC_RECVQ_STATS_H_
diff --git a/src/mds/mds_tipc_recvq_stats_impl.cc
b/src/mds/mds_tipc_recvq_stats_impl.cc
new file mode 100644
index 000000000..df86dc85e
--- /dev/null
+++ b/src/mds/mds_tipc_recvq_stats_impl.cc
@@ -0,0 +1,178 @@
+/* -*- OpenSAF -*-
+ *
+ * (C) Copyright 2019 The OpenSAF Foundation
+ * Copyright Ericsson AB 2019 - All Rights Reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed
+ * under the GNU Lesser General Public License Version 2.1, February 1999.
+ * The complete license can be accessed from the following location:
+ * http://opensource.org/licenses/lgpl-license.php
+ * See the Copying file included with the OpenSAF distribution for full
+ * licensing terms.
+ *
+ * Author(s): Ericsson AB
+ *
+ */
+
+#include "mds_tipc_recvq_stats_impl.h"
+#include <errno.h>
+#include <linux/tipc.h>
+#include <poll.h>
+#include <sys/socket.h>
+#include <sys/timerfd.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <cinttypes>
+#include <cstdlib>
+#include <cstring>
+#include <cstdint>
+#include <thread>
+#include "base/osaf_time.h"
+#include "base/statistics.h"
+#include "base/logtrace.h"
+
+void TipcRecvqStatsImpl::start() {
+ std::thread(&TipcRecvqStatsImpl::tipc_recvq_stats_bg, this).detach();
+}
+
+int TipcRecvqStatsImpl::init(int sd) {
+ int optval{0};
+ socklen_t optlen = sizeof(optval);
+ long log_freq_sec{0};
+
+ sd_ = sd;
+ recvq_size_ = 0;
+ char *val;
+
+ if ((val = getenv("MDS_RECVQ_STATS_LOG_FREQ_SEC")) != NULL) {
+ log_freq_sec = strtol(val, NULL, 0);
+ log_freq_ = log_freq_sec * 10;
+ if (log_freq_sec < 1) {
+ LOG_NO("MDS_RECVQ_STATS_LOG_FREQ_SEC value is set too low: %ld seconds",
log_freq_sec);
+ return -1;
+ }
+ } else {
+ return -1;
+ }
+
+ if (getsockopt(sd_, SOL_SOCKET, SO_RCVBUF, &optval, &optlen) < 0) {
+ LOG_NO("TIPC getsockopt failed: %s", strerror(errno));
+ return -1;
+ }
+
+ recvq_size_ = optval;
+
+ return get_tipc_recvq_used(&optval);
+}
+
+int TipcRecvqStatsImpl::get_tipc_recvq_used(int *optval) {
+#ifndef TIPC_SOCK_RECVQ_USED
+#define TIPC_SOCK_RECVQ_USED 137
+#endif
+
+ socklen_t optlen = sizeof(optval);
+
+ if (getsockopt(sd_, SOL_TIPC, TIPC_SOCK_RECVQ_USED, optval, &optlen) == 0) {
+ return 0;
+ } else {
+ LOG_NO("TIPC getsockopt failed: %s", strerror(errno));
+ return -1;
+ }
+}
+
+int TipcRecvqStatsImpl::create_timer() {
+ if ((timer_fd_ = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC))
< 0) {
+ LOG_NO("timerfd_create failed: %s", strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+int TipcRecvqStatsImpl::start_timer() {
+ uint64_t interval_msec{100};
+ struct itimerspec spec;
+ spec.it_interval.tv_sec = interval_msec / kMillisPerSec;
+ spec.it_interval.tv_nsec = (interval_msec % kMillisPerSec) * (kNanosPerSec
/ kMillisPerSec);
+ spec.it_value.tv_sec = spec.it_interval.tv_sec;
+ spec.it_value.tv_nsec = spec.it_interval.tv_nsec;
+
+ if (timerfd_settime(timer_fd_, 0, &spec, NULL) < 0) {
+ LOG_NO("timerfd_settime failed: %s", strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+int TipcRecvqStatsImpl::stop_timer() {
+ struct itimerspec spec;
+
+ spec.it_interval.tv_sec = 0;
+ spec.it_interval.tv_nsec = 0;
+ spec.it_value.tv_sec = 0;
+ spec.it_value.tv_nsec = 0;
+
+ if (timerfd_settime(timer_fd_, 0, &spec, NULL) < 0) {
+ LOG_NO("timerfd_settime failed: %s", strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+void TipcRecvqStatsImpl::tipc_recvq_stats_bg() {
+ base::Statistics stats;
+ int optval;
+ int ticks{0};
+
+ enum {
+ FD_TMR = 0,
+ NUM_FDS
+ };
+
+ create_timer();
+
+ struct pollfd fds[NUM_FDS];
+ fds[FD_TMR].fd = timer_fd_;
+ fds[FD_TMR].events = POLLIN;
+
+ start_timer();
+
+ LOG_NO("TIPC receive queue statistics started");
+
+ while (true) {
+ int rc = poll(fds, NUM_FDS, -1);
+ if (rc == -1) {
+ if (errno == EINTR) continue;
+ LOG_IN("poll failed: %s", strerror(errno));
+ break;
+ }
+ if (rc > 0) {
+ if (fds[FD_TMR].revents == POLLIN) {
+ uint64_t expirations = 0;
+ if (read(timer_fd_, &expirations, 8) != 8) {
+ LOG_NO("error reading timerfd value: %s", strerror(errno));
+ return;
+ } else {
+ if (expirations != 1) {
+ LOG_NO("timerfd expired %" PRIu64 " times", expirations);
+ }
+ }
+ if (get_tipc_recvq_used(&optval) == 0) {
+ ++ticks;
+ stats.push((optval / recvq_size_) * 100.0);
+ if (ticks >= log_freq_) {
+ LOG_NO("TIPC receive queue utilization (in %%): min: %2.2f max: %2.2f
mean: %2.2f std dev: %2.2f",
+ stats.min(), stats.max(), stats.mean(), stats.std_dev());
+ ticks = 0;
+ stats.clear();
+ }
+ } else {
+ break;
+ }
+ }
+ }
+ }
+ return;
+}
+
diff --git a/src/mds/mds_tipc_recvq_stats_impl.h
b/src/mds/mds_tipc_recvq_stats_impl.h
new file mode 100644
index 000000000..fefe2751d
--- /dev/null
+++ b/src/mds/mds_tipc_recvq_stats_impl.h
@@ -0,0 +1,39 @@
+/* -*- OpenSAF -*-
+ *
+ * (C) Copyright 2019 The OpenSAF Foundation
+ * Copyright Ericsson AB 2019 - All Rights Reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed
+ * under the GNU Lesser General Public License Version 2.1, February 1999.
+ * The complete license can be accessed from the following location:
+ * http://opensource.org/licenses/lgpl-license.php
+ * See the Copying file included with the OpenSAF distribution for full
+ * licensing terms.
+ *
+ * Author(s): Ericsson AB
+ *
+ */
+
+#ifndef MDS_TIPC_RECVQ_STATS_IMPL_
+#define MDS_TIPC_RECVQ_STATS_IMPL_
+
+class TipcRecvqStatsImpl {
+ public:
+ int init(int sd);
+ void start();
+ private:
+ int get_tipc_recvq_used(int *optval);
+ void tipc_recvq_stats_bg();
+ int create_timer();
+ int start_timer();
+ int stop_timer();
+
+ int sd_{-1};
+ double recvq_size_{0};
+ int timer_fd_{-1};
+ long log_freq_{0};
+};
+
+#endif // MDS_TIPC_RECVQ_STATS_IMPL_
