--- src/fm/Makefile.am | 6 +- src/fm/fmd/fm_amf.cc | 14 + src/fm/fmd/tipc_server.cc | 93 ++++++ src/fm/fmd/tipc_server.h | 45 +++ tools/devel/fenced/Makefile | 63 ++++ tools/devel/fenced/README_TOOLS | 15 + tools/devel/fenced/command.cc | 134 ++++++++ tools/devel/fenced/command.h | 43 +++ tools/devel/fenced/cpp_macros.h | 33 ++ tools/devel/fenced/fenced.conf | 17 + tools/devel/fenced/fenced_main.cc | 179 +++++++++++ tools/devel/fenced/node_state_file.cc | 87 ++++++ tools/devel/fenced/node_state_file.h | 41 +++ tools/devel/fenced/node_state_hdlr.cc | 54 ++++ tools/devel/fenced/node_state_hdlr.h | 45 +++ tools/devel/fenced/node_state_hdlr_factory.cc | 66 ++++ tools/devel/fenced/node_state_hdlr_factory.h | 35 +++ tools/devel/fenced/node_state_hdlr_pl.cc | 292 ++++++++++++++++++ tools/devel/fenced/node_state_hdlr_pl.h | 60 ++++ tools/devel/fenced/node_state_hdlr_sc.cc | 42 +++ tools/devel/fenced/node_state_hdlr_sc.h | 41 +++ tools/devel/fenced/osaffenced.service | 14 + tools/devel/fenced/service.cc | 53 ++++ tools/devel/fenced/service.h | 42 +++ tools/devel/fenced/timer.cc | 62 ++++ tools/devel/fenced/timer.h | 53 ++++ tools/devel/fenced/watchdog.cc | 37 +++ tools/devel/fenced/watchdog.h | 39 +++ 28 files changed, 1703 insertions(+), 2 deletions(-) create mode 100644 src/fm/fmd/tipc_server.cc create mode 100644 src/fm/fmd/tipc_server.h create mode 100755 tools/devel/fenced/Makefile create mode 100644 tools/devel/fenced/README_TOOLS create mode 100644 tools/devel/fenced/command.cc create mode 100644 tools/devel/fenced/command.h create mode 100644 tools/devel/fenced/cpp_macros.h create mode 100644 tools/devel/fenced/fenced.conf create mode 100644 tools/devel/fenced/fenced_main.cc create mode 100644 tools/devel/fenced/node_state_file.cc create mode 100644 tools/devel/fenced/node_state_file.h create mode 100644 tools/devel/fenced/node_state_hdlr.cc create mode 100644 tools/devel/fenced/node_state_hdlr.h create mode 100644 tools/devel/fenced/node_state_hdlr_factory.cc create mode 100644 tools/devel/fenced/node_state_hdlr_factory.h create mode 100644 tools/devel/fenced/node_state_hdlr_pl.cc create mode 100644 tools/devel/fenced/node_state_hdlr_pl.h create mode 100644 tools/devel/fenced/node_state_hdlr_sc.cc create mode 100644 tools/devel/fenced/node_state_hdlr_sc.h create mode 100644 tools/devel/fenced/osaffenced.service create mode 100644 tools/devel/fenced/service.cc create mode 100644 tools/devel/fenced/service.h create mode 100644 tools/devel/fenced/timer.cc create mode 100644 tools/devel/fenced/timer.h create mode 100644 tools/devel/fenced/watchdog.cc create mode 100644 tools/devel/fenced/watchdog.h
diff --git a/src/fm/Makefile.am b/src/fm/Makefile.am index 0f254b94f..325847ae9 100644 --- a/src/fm/Makefile.am +++ b/src/fm/Makefile.am @@ -20,7 +20,8 @@ noinst_HEADERS += \ src/fm/fmd/fm_cb.h \ src/fm/fmd/fm_evt.h \ src/fm/fmd/fm_mds.h \ - src/fm/fmd/fm_mem.h + src/fm/fmd/fm_mem.h \ + src/fm/fmd/tipc_server.h osaf_execbin_PROGRAMS += bin/osaffmd nodist_pkgclccli_SCRIPTS += \ @@ -44,7 +45,8 @@ bin_osaffmd_SOURCES = \ src/fm/fmd/fm_amf.cc \ src/fm/fmd/fm_main.cc \ src/fm/fmd/fm_mds.cc \ - src/fm/fmd/fm_rda.cc + src/fm/fmd/fm_rda.cc \ + src/fm/fmd/tipc_server.cc bin_osaffmd_LDADD = \ lib/libSaAmf.la \ diff --git a/src/fm/fmd/fm_amf.cc b/src/fm/fmd/fm_amf.cc index e99f3ba7e..8cf284f97 100644 --- a/src/fm/fmd/fm_amf.cc +++ b/src/fm/fmd/fm_amf.cc @@ -34,6 +34,12 @@ ******************************************************************************/ #include "fm.h" +#include "tipc_server.h" + +namespace { +TIPCServer tipc_srv; +} + extern uint32_t gl_fm_hdl; uint32_t fm_amf_init(FM_AMF_CB *fm_amf_cb); @@ -151,6 +157,11 @@ void fm_saf_CSI_set_callback(SaInvocationT invocation, const SaNameT *compName, } else { fm_cb->amf_state = new_haState; fm_cb->csi_assigned = true; + if (new_haState == SA_AMF_HA_ACTIVE) { + tipc_srv.publish(); + } else { + tipc_srv.unpublish(); + } } error = saAmfResponse(fm_amf_cb->amf_hdl, invocation, error); } @@ -300,6 +311,9 @@ uint32_t fm_amf_init(FM_AMF_CB *fm_amf_cb) { SaNameT sname; uint32_t rc = NCSCC_RC_SUCCESS; TRACE_ENTER(); + + tipc_srv.init(); + memset(&amfCallbacks, 0, sizeof(SaAmfCallbacksT)); if (fm_amf_cb->nid_started && amf_comp_name_get_set_from_file("FM_COMP_NAME_FILE", &sname) != diff --git a/src/fm/fmd/tipc_server.cc b/src/fm/fmd/tipc_server.cc new file mode 100644 index 000000000..4586934d9 --- /dev/null +++ b/src/fm/fmd/tipc_server.cc @@ -0,0 +1,93 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include <cerrno> +#include <cstring> +#include <syslog.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <unistd.h> +#include "tipc_server.h" + +TIPCServer::TIPCServer() { +} + +TIPCServer::~TIPCServer() { + close(sd_); +} + +int TIPCServer::init() { + if ((sd_ = socket(AF_TIPC, SOCK_RDM, 0)) < 0) { + syslog(LOG_ERR, "Failed to create TIPC socket for fenced svc: %s", strerror(errno)); + return -1; + } + get_node_id(); + tipc_bind(FMD_FENCED_SVC_TYPE, TIPC_CLUSTER_SCOPE); + return 0; +} + +void TIPCServer::get_node_id() { + struct sockaddr_tipc addr; + socklen_t sz = sizeof(addr); + + memset(&addr, 0, sizeof(addr)); + node_id_ = 0; + + if (getsockname(sd_, (struct sockaddr *)&addr, &sz) < 0) { + syslog(LOG_ERR, "Failed to get socket name: %s", strerror(errno)); + } + + node_id_ = addr.addr.id.node; +} + +int TIPCServer::tipc_bind(uint32_t service_type, signed char scope) { + struct sockaddr_tipc server_addr; + + server_addr.family = AF_TIPC; + server_addr.addrtype = TIPC_ADDR_NAMESEQ; + server_addr.addr.nameseq.type = service_type; + server_addr.scope = scope; + + server_addr.addr.nameseq.lower = node_id_; + server_addr.addr.nameseq.upper = node_id_; + + if (bind(sd_, (struct sockaddr *)&server_addr, sizeof(server_addr)) != 0) { + syslog(LOG_ERR, "Failed to bind TIPC port for fenced svc: %s", strerror(errno)); + return -1; + } else { + syslog(LOG_NOTICE, "Fenced svc %s port {%u,%u,%u} scope: %d", (scope < 0) ? "unbound" : "bound", + server_addr.addr.nameseq.type, server_addr.addr.nameseq.lower, + server_addr.addr.nameseq.upper, scope); + } + return 0; +} + +void TIPCServer::publish() { + if (!published_) { + if (tipc_bind(FMD_FENCED_SVC_TYPE_ACTIVE, TIPC_CLUSTER_SCOPE) == 0) { + published_ = true; + } + } +} + +void TIPCServer::unpublish() { + if (published_) { + if (tipc_bind(FMD_FENCED_SVC_TYPE_ACTIVE, -TIPC_CLUSTER_SCOPE) == 0) { + published_ = false; + } + } +} diff --git a/src/fm/fmd/tipc_server.h b/src/fm/fmd/tipc_server.h new file mode 100644 index 000000000..ef9781682 --- /dev/null +++ b/src/fm/fmd/tipc_server.h @@ -0,0 +1,45 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef TIPC_SERVER_H_ +#define TIPC_SERVER_H_ + +#include <linux/tipc.h> +#include <cinttypes> +#include "base/macros.h" + +class TIPCServer { + public: + const uint32_t FMD_FENCED_SVC_TYPE = 19888; + const uint32_t FMD_FENCED_SVC_TYPE_ACTIVE = 19889; + + TIPCServer(); + ~TIPCServer(); + int init(); + void publish(); + void unpublish(); + protected: + private: + void get_node_id(); + int tipc_bind(uint32_t service_type, signed char scope); + bool published_ {false}; + uint32_t node_id_; + int sd_; + DELETE_COPY_AND_MOVE_OPERATORS(TIPCServer); +}; + +#endif // TIPC_CLIENT_H_ diff --git a/tools/devel/fenced/Makefile b/tools/devel/fenced/Makefile new file mode 100755 index 000000000..a2761e46c --- /dev/null +++ b/tools/devel/fenced/Makefile @@ -0,0 +1,63 @@ +# -*- OpenSAF -*- +# +# Copyright Ericsson AB 2019 - All Rights Reserved. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed +# under the GNU Lesser General Public License Version 2.1, February 1999. +# The complete license can be accessed from the following location: +# http://opensource.org/licenses/lgpl-license.php +# See the Copying file included with the OpenSAF distribution for full +# licensing terms. +# +# Author(s): Ericsson AB +# +SYSTEMD_DIR = /lib/systemd/system +OSAF_DIR = /etc/opensaf +INST_DIR = /usr/local/lib/opensaf +CONFIG_FILE=fenced.conf +SERVICE_FILE = osaffenced.service + +PGM = osaffenced +OBJS = command.o fenced_main.o node_state_file.o node_state_hdlr.o node_state_hdlr_factory.o node_state_hdlr_pl.o node_state_hdlr_sc.o service.o timer.o watchdog.o + +INCLUDES = -I. + +#--------------------------------------------------------- +# Compiler & linker flags +#--------------------------------------------------------- +INCLUDES = -I. +CXXFLAGS = -g -Wall -Werror -std=c++11 $(INCLUDES) +LDFLAGS = -lsystemd -lpthread + +#--------------------------------------------------------- +# Explicit targets +#--------------------------------------------------------- +all: $(PGM) + +$(PGM): $(OBJS) + $(CXX) -o $@ $(OBJS) $(LDFLAGS) + +$(OBJECTS): %.o: + $(CXX) -c $< -o $@ + +.PHONY: install +install: osaffenced + cp $< $(INST_DIR) + cp $(CONFIG_FILE) $(OSAF_DIR) + cp $(SERVICE_FILE) $(SYSTEMD_DIR) + systemctl enable $(SERVICE_FILE) + systemctl start $(SERVICE_FILE) + +.PHONY: uninstall +uninstall: osaffenced + systemctl stop $(SERVICE_FILE) + systemctl disable $(SERVICE_FILE) + rm $(INST_DIR)/$< + rm $(OSAF_DIR)/$(CONFIG_FILE) + rm $(SYSTEMD_DIR)/$(SERVICE_FILE) + +.PHONY: clean +clean: + -rm -f *.o *~ $(PGM) diff --git a/tools/devel/fenced/README_TOOLS b/tools/devel/fenced/README_TOOLS new file mode 100644 index 000000000..4bb40234f --- /dev/null +++ b/tools/devel/fenced/README_TOOLS @@ -0,0 +1,15 @@ +Google style guide: +cpplint.py --filter="-build/include,-build/header_guard,-whitespace/line_length" *.cc *.h + +Memory and thread checker: +valgrind --leak-check=full --log-file=/tmp/osaffenced.valgrind ./osaffenced +valgrind --tool=helgrind --log-file=/tmp/osaffenced.valgrind ./osaffenced + +Static lint checker: +CodeChecker log -b "make" -o compilation.json +CodeChecker analyze compilation.json -o ~/.reports +CodeChecker parse ~/.reports -e html -o ~/reports_html +firefox ~/reports_html/index.html + +McCabe Cyclomatic Complexity, top 10 most complex function: +find . -name "*.cc" | xargs pmccabe | sort -nr | head -10 \ No newline at end of file diff --git a/tools/devel/fenced/command.cc b/tools/devel/fenced/command.cc new file mode 100644 index 000000000..38f0fd84e --- /dev/null +++ b/tools/devel/fenced/command.cc @@ -0,0 +1,134 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +/* + for more info, see: + man sd_bus_message_append + + some busctl commands: + busctl introspect org.freedesktop.systemd1 /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager + busctl get-property org.freedesktop.systemd1 /org/freedesktop/systemd1/unit/opensafd_2eservice org.freedesktop.systemd1.Unit ActiveState + busctl call org.freedesktop.systemd1 /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager StartUnit "ss" "opensafd.service" "replace" +*/ + +#include "command.h" + +#include <syslog.h> +#include <cstdio> +#include <sstream> +#include <string> + +namespace { +const char SERVICE_NAME[] = "org.freedesktop.systemd1"; +const char OBJECT_PATH[] = "/org/freedesktop/systemd1"; +const char INTERFACE[] = "org.freedesktop.systemd1.Manager"; +const char JOB_MODE[] = "replace"; +} + +Command::Command() { +} + +Command::~Command() { + sd_bus_unref(bus_); +} + +int32_t Command::sd_start_stop_cmd(const std::string& method_name, + const std::string& service) { + sd_bus_error error = SD_BUS_ERROR_NULL; + sd_bus_message *msg {NULL}; + const char *path {NULL}; + + std::string no_esc_service = service + ".service"; + // connect to the system bus + int rc = sd_bus_default_system(&bus_); + if (rc < 0) { + syslog(LOG_WARNING, "sd_bus_default_system failed: %s", strerror(-rc)); + goto leave; + } + + // Issue the method call and store the response message + rc = sd_bus_call_method(bus_, + SERVICE_NAME, + OBJECT_PATH, + INTERFACE, + method_name.c_str(), + &error, + &msg, + "ss", + no_esc_service.c_str(), + JOB_MODE); + if (rc < 0) { + syslog(LOG_WARNING, "sd_bus_call_method failed: %s", strerror(-rc)); + goto done; + } + + // Parse the response message + rc = sd_bus_message_read(msg, "o", &path); + if (rc < 0) { + syslog(LOG_WARNING, "sd_bus_message_read failed: %s", strerror(-rc)); + goto done; + } + syslog(LOG_INFO, "fenced:: %s", path); + sd_bus_error_free(&error); +done: + sd_bus_message_unref(msg); + sd_bus_unref(bus_); +leave: + return rc; +} + +bool Command::is_active(const std::string& service) { + sd_bus_error error {SD_BUS_ERROR_NULL}; + char *buf {NULL}; + bool is_active {false}; + + // escaped service name + std::string esc_service = service + "_2eservice"; + std::string object_path = OBJECT_PATH; + object_path += "/unit/" + esc_service; + + // Connect to the system bus + int rc = sd_bus_default_system(&bus_); + if (rc < 0) { + syslog(LOG_WARNING, "sd_bus_default_system failed: %s", strerror(-rc)); + goto leave; + } + + // Get service active property + rc = sd_bus_get_property_string(bus_, + SERVICE_NAME, + object_path.c_str(), + "org.freedesktop.systemd1.Unit", + "ActiveState", + &error, + &buf); + if (rc < 0) { + syslog(LOG_WARNING, "sd_bus_get_property failed: %s", strerror(-rc)); + goto done; + } + + if (strncmp(buf, "active", 6) == 0) { + is_active = true; + } + + free(buf); + sd_bus_error_free(&error); + done: + sd_bus_unref(bus_); + leave: + return is_active; +} diff --git a/tools/devel/fenced/command.h b/tools/devel/fenced/command.h new file mode 100644 index 000000000..4716e0a81 --- /dev/null +++ b/tools/devel/fenced/command.h @@ -0,0 +1,43 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef COMMAND_H_ +#define COMMAND_H_ + +#include <systemd/sd-bus.h> +#include <cstdint> +#include <string> + +#include "cpp_macros.h" + +class Command { + public: + Command(); + ~Command(); + int32_t start(const std::string& service) {return sd_start_stop_cmd("StartUnit", service);} + int32_t stop(const std::string& service) {return sd_start_stop_cmd("StopUnit", service);} + bool is_active(const std::string& service); + + protected: + private: + int32_t sd_start_stop_cmd(const std::string& method_name, + const std::string& service); + sd_bus *bus_{NULL}; + DELETE_COPY_AND_MOVE_OPERATORS(Command); +}; + +#endif // COMMAND_H_ diff --git a/tools/devel/fenced/cpp_macros.h b/tools/devel/fenced/cpp_macros.h new file mode 100644 index 000000000..9831ee407 --- /dev/null +++ b/tools/devel/fenced/cpp_macros.h @@ -0,0 +1,33 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef CPP_MACROS_H_ +#define CPP_MACROS_H_ + +#define USE_DEFAULT_COPY_AND_MOVE_OPERATORS(className) \ + className(className&&) = default; \ + className(const className&) = default; \ + className& operator=(className&&) = default; \ + className& operator=(const className&) = default + +#define DELETE_COPY_AND_MOVE_OPERATORS(className) \ + className(className&&) = delete; \ + className(const className&) = delete; \ + className& operator=(className&&) = delete; \ + className& operator=(const className&) = delete + +#endif // CPP_MACROS_H_ diff --git a/tools/devel/fenced/fenced.conf b/tools/devel/fenced/fenced.conf new file mode 100644 index 000000000..220d9d42d --- /dev/null +++ b/tools/devel/fenced/fenced.conf @@ -0,0 +1,17 @@ +# systemd services managed by fenced. Separate service names by whitespace, e.g. "opensafd" +SERVICES_TO_FENCE="opensafd" + +# Directory where files "node_is_not_headless" and "node_is_headless" are created/deleted +NODE_STATE_FILE_DIR="/tmp" +ISOLATION_STATUS_CHECK_TIME_SEC=3 + +# "payload" +NODE_TYPE="payload" + +### --- Payload node specific configuration +# TIPC portname +FMD_FENCED_SVC_TYPE=19888; +FMD_FENCED_SVC_TYPE_ACTIVE=19889; + +AWAIT_ACTIVE_TIME_SEC=10 +### --- End Payload node specific configuration diff --git a/tools/devel/fenced/fenced_main.cc b/tools/devel/fenced/fenced_main.cc new file mode 100644 index 000000000..7598d05c0 --- /dev/null +++ b/tools/devel/fenced/fenced_main.cc @@ -0,0 +1,179 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include <poll.h> +#include <sched.h> +#include <signal.h> +#include <syslog.h> +#include <sys/socket.h> +#include <cerrno> +#include <cinttypes> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <string> + +#include "command.h" +#include "service.h" +#include "timer.h" +#include "node_state_hdlr_factory.h" +#include "watchdog.h" +#include "node_state_file.h" + +static NodeStateFile node_state_file; +static NodeStateHdlr *node_state_hdlr {nullptr}; +static Timer tmr; +static Command cmd; +static Service svc; +static Watchdog wd; + +static void sigterm_handler(int signum, siginfo_t *info, void *ptr) { + (void) signum; + (void) info; + (void) ptr; + + exit(0); +} + +static void set_rt_prio() { + struct sched_param param {0}; + int policy {SCHED_RR}; + param.sched_priority = sched_get_priority_min(policy); + + if (sched_setscheduler(0, policy, ¶m) < 0) { + syslog(LOG_ERR, "failed to set scheduling class: %s", strerror(errno)); + } +} + +// +int main(int argc, char* argv[]) { + (void) argc; + + NodeStateHdlr::NodeIsolationState isolated {NodeStateHdlr::NodeIsolationState::kUndefined}; + NodeStateHdlr::NodeIsolationState current_isolation_state {NodeStateHdlr::NodeIsolationState::kUndefined}; + struct sigaction act; + + enum { + FD_TMRWD = 0, + FD_EVT, + NUM_FDS + }; + struct pollfd fds[NUM_FDS]; + + openlog(basename(argv[0]), LOG_PID, LOG_LOCAL0); + + sigemptyset(&act.sa_mask); + act.sa_sigaction = sigterm_handler; + act.sa_flags = SA_SIGINFO | SA_RESETHAND; + if (sigaction(SIGTERM, &act, NULL) < 0) { + syslog(LOG_ERR, "sigaction TERM failed: %s", strerror(errno)); + } + + set_rt_prio(); + + svc.init(); + + node_state_hdlr = NodeStateHdlrFactory::instance().create(); + + if (node_state_hdlr->init() < 0) { + exit(-1); + } + + node_state_file.init(); + + // use systemd watchdog time value divided by two to kick the watchdog + int timer_fd = tmr.start(wd.interval_usec() / 2, Timer::Type::kPeriodic); + syslog(LOG_NOTICE, "systemd watchdog interval us %ld", wd.interval_usec() / 2); + + fds[FD_EVT].fd = node_state_hdlr->get_event_fd(); + fds[FD_EVT].events = POLLIN; + + fds[FD_TMRWD].fd = timer_fd; + fds[FD_TMRWD].events = POLLIN; + + while (true) { + int rc = poll(fds, NUM_FDS, -1); + if (rc == -1) { + if (errno == EINTR) continue; + syslog(LOG_ERR, "poll failed: %s", strerror(errno)); + break; + } + if (rc > 0) { + if (fds[FD_EVT].revents == POLLIN) { + for (;;) { + if (recv(fds[FD_EVT].fd, &isolated, sizeof(isolated), + MSG_DONTWAIT) < 0) { + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + break; + syslog(LOG_ERR, "recv failed: %s", strerror(errno)); + goto done; + } + } + if (isolated != current_isolation_state) { + current_isolation_state = isolated; + node_state_file.update(isolated); + if (isolated == NodeStateHdlr::NodeIsolationState::kIsolated) { + for (auto &s : svc.services_) { + if (cmd.is_active(s.first)) { + syslog(LOG_NOTICE, "Node is isolated - %s will be stopped", s.first.c_str()); + s.second.fenced_stopped = true; + cmd.stop(s.first); + } + } + } else { + for (auto &s : svc.services_) { + if (s.second.fenced_stopped && !cmd.is_active(s.first)) { + syslog(LOG_NOTICE, "Node is not isolated - %s will be started", s.first.c_str()); + s.second.fenced_stopped = false; + cmd.start(s.first); + } + } + } + } + } + // timerfd + if (fds[FD_TMRWD].revents == POLLIN) { + uint64_t expirations = 0; + if (read(tmr.get_fd(), &expirations, 8) != 8) { + syslog(LOG_WARNING, "error reading timerfd value"); + } else { + if (expirations != 1) { + syslog(LOG_NOTICE, "timerfd expired %" PRIu64 " times", expirations); + } + } + + if (current_isolation_state == NodeStateHdlr::NodeIsolationState::kIsolated) { + for (auto &s : svc.services_) { + if (cmd.is_active(s.first)) { + syslog(LOG_NOTICE, "%s is active, node is isolated - %s will be stopped", + s.first.c_str(), s.first.c_str()); + s.second.fenced_stopped = true; + cmd.stop(s.first); + } + } + } + // kick systemd watchdog + wd.kick(); + } + } + } +done: + closelog(); + return -1; +} diff --git a/tools/devel/fenced/node_state_file.cc b/tools/devel/fenced/node_state_file.cc new file mode 100644 index 000000000..aeddd1a9f --- /dev/null +++ b/tools/devel/fenced/node_state_file.cc @@ -0,0 +1,87 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "node_state_file.h" + +#include <syslog.h> +#include <unistd.h> +#include <cstdio> +#include <fstream> + +NodeStateFile::NodeStateFile() { +} + +NodeStateFile::~NodeStateFile() { + unlink(node_is_isolated_.c_str()); + unlink(node_is_not_isolated_.c_str()); +} + +bool NodeStateFile::exists(const std::string& file) { + return (access(file.c_str(), F_OK ) != -1); +} + +int NodeStateFile::create(const std::string file) { + std::ofstream outfile(file); + if (outfile.good()) { + syslog(LOG_NOTICE, "file \"%s\" created", file.c_str()); + return -1; + } else { + syslog(LOG_ERR, "failed to create file \"%s\"", file.c_str()); + return 0; + } +} + +void NodeStateFile::update(NodeStateHdlr::NodeIsolationState isolated) { + if (isolated == NodeStateHdlr::NodeIsolationState::kIsolated) { + if (exists(node_is_not_isolated_)) { + if (std::rename(node_is_not_isolated_.c_str(), + node_is_isolated_.c_str()) != 0) { + syslog(LOG_ERR, "failed to rename \"%s\" to \"%s\"", + node_is_not_isolated_.c_str(), node_is_isolated_.c_str()); + } else { + syslog(LOG_NOTICE, "file \"%s\" renamed to \"%s\"", + node_is_not_isolated_.c_str(), node_is_isolated_.c_str()); + } + } else if (!exists(node_is_isolated_)) { + create(node_is_isolated_); + } + } else { + if (exists(node_is_isolated_)) { + if (std::rename(node_is_isolated_.c_str(), node_is_not_isolated_.c_str()) != 0) { + syslog(LOG_ERR, "failed to rename \"%s\" to \"%s\"", + node_is_isolated_.c_str(), node_is_not_isolated_.c_str()); + } else { + syslog(LOG_NOTICE, "file \"%s\" renamed to \"%s\"", node_is_isolated_.c_str(), node_is_not_isolated_.c_str()); + } + } else if (!exists(node_is_not_isolated_)) { + create(node_is_not_isolated_); + } + } +} + +void NodeStateFile::init() { + char *p = getenv("NODE_STATE_FILE_DIR"); + std::string str; + + if (p == NULL) { + syslog(LOG_ERR, "no \"NODE_STATE_FILE_DIR\" has been configured"); + return; + } + str = p; + node_is_isolated_ = str + "/" + "osaf_node_is_isolated"; + node_is_not_isolated_ = str + "/" + "osaf_node_is_not_isolated"; +} diff --git a/tools/devel/fenced/node_state_file.h b/tools/devel/fenced/node_state_file.h new file mode 100644 index 000000000..1270094aa --- /dev/null +++ b/tools/devel/fenced/node_state_file.h @@ -0,0 +1,41 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef NODE_STATE_FILE_H_ +#define NODE_STATE_FILE_H_ + +#include <string> + +#include "cpp_macros.h" +#include "node_state_hdlr.h" + +class NodeStateFile { + public: + NodeStateFile(); + ~NodeStateFile(); + void update(NodeStateHdlr::NodeIsolationState isolated); + void init(); + private: + bool exists(const std::string& file); + int create(const std::string file); + + std::string node_is_isolated_; + std::string node_is_not_isolated_; + DELETE_COPY_AND_MOVE_OPERATORS(NodeStateFile); +}; + +#endif // NODE_STATE_FILE_H_ diff --git a/tools/devel/fenced/node_state_hdlr.cc b/tools/devel/fenced/node_state_hdlr.cc new file mode 100644 index 000000000..fb90ff481 --- /dev/null +++ b/tools/devel/fenced/node_state_hdlr.cc @@ -0,0 +1,54 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "node_state_hdlr.h" + +#include <sys/socket.h> +#include <syslog.h> +#include <unistd.h> +#include <cstring> +#include <cerrno> + +NodeStateHdlr::NodeStateHdlr() { + int sockflags = SOCK_DGRAM | SOCK_NONBLOCK | SOCK_CLOEXEC; + int sfd[2]; + if (socketpair(AF_UNIX, sockflags, 0, sfd) != 0) { + syslog(LOG_ERR, "failed to create socketpair: %s", strerror(errno)); + read_fd_ = -1; + write_fd_ = -1; + } else { + read_fd_ = sfd[0]; + write_fd_ = sfd[1]; + } +} + +NodeStateHdlr::~NodeStateHdlr() { + close(read_fd_); + close(write_fd_); +} + +void NodeStateHdlr::send_update() { + int rc {0}; + do { + rc = send(write_fd_, &isolated_, sizeof(isolated_), MSG_DONTWAIT | MSG_NOSIGNAL); + } while (rc == -1 && errno == EINTR); + + if (rc != sizeof(isolated_)) { + syslog(LOG_ERR, "failed to send: %s", strerror(errno)); + } +} + diff --git a/tools/devel/fenced/node_state_hdlr.h b/tools/devel/fenced/node_state_hdlr.h new file mode 100644 index 000000000..38254cd13 --- /dev/null +++ b/tools/devel/fenced/node_state_hdlr.h @@ -0,0 +1,45 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef NODE_STATE_HDLR_H_ +#define NODE_STATE_HDLR_H_ + +#include <cinttypes> + +#include "cpp_macros.h" + +class NodeStateHdlr { + public: + enum class AwaitActiveTmrStatus {kNotRunning, kRunning, kExpired}; + enum class NodeIsolationState {kUndefined, kIsolated, kNotIsolated}; + + NodeStateHdlr(); + virtual ~NodeStateHdlr(); + virtual int init() = 0; + int get_event_fd() const {return read_fd_;} + + protected: + const uint32_t DFLT_ISOLATION_STATUS_CHECK_TIME_SEC = 3; + void send_update(); + NodeIsolationState isolated_ {NodeIsolationState::kIsolated}; + private: + int write_fd_; + int read_fd_; + DELETE_COPY_AND_MOVE_OPERATORS(NodeStateHdlr); +}; + +#endif // NODE_STATE_HDLR_ diff --git a/tools/devel/fenced/node_state_hdlr_factory.cc b/tools/devel/fenced/node_state_hdlr_factory.cc new file mode 100644 index 000000000..a9b89e85c --- /dev/null +++ b/tools/devel/fenced/node_state_hdlr_factory.cc @@ -0,0 +1,66 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "node_state_hdlr_factory.h" + +#include <syslog.h> +#include <algorithm> +#include <string> + +#include "node_state_hdlr_sc.h" +#include "node_state_hdlr_pl.h" + +// case insensitive string compare +static bool str_equal(std::string &str1, std::string &str2) { + return ((str1.size() == str2.size()) && + std::equal(str1.begin(), str1.end(), str2.begin(), [] (char &c1, char &c2) { + return (c1 == c2 || std::toupper(c1) == std::toupper(c2)); + })); +} + +NodeStateHdlrFactory::NodeStateHdlrFactory() { + char *p = getenv("NODE_TYPE"); + std::string str; + std::string ctl {"controller"}; + is_controller_ = false; + + if (p == NULL) { + syslog(LOG_INFO, "no \"NODE_TYPE\" has been configured, defaults to payload node"); + } else { + str = p; + if (str_equal(str, ctl)) { + is_controller_ = true; + } + } +} + +NodeStateHdlrFactory::~NodeStateHdlrFactory() { +} + +NodeStateHdlrFactory& NodeStateHdlrFactory::instance() { + // in C++11 this method is thread safe, see §6.7 [stmt.dcl] p4 + static NodeStateHdlrFactory n; + return n; +} + +NodeStateHdlr* NodeStateHdlrFactory::create() { + if (is_controller_) { + return new NodeStateHdlrSc; + } else { + return new NodeStateHdlrPl; + } +} diff --git a/tools/devel/fenced/node_state_hdlr_factory.h b/tools/devel/fenced/node_state_hdlr_factory.h new file mode 100644 index 000000000..9ed8f4c31 --- /dev/null +++ b/tools/devel/fenced/node_state_hdlr_factory.h @@ -0,0 +1,35 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef NODE_STATE_HDLR_FACTORY_H_ +#define NODE_STATE_HDLR_FACTORY_H_ + +#include "node_state_hdlr.h" + +class NodeStateHdlrFactory { + public: + NodeStateHdlr *create(); + static NodeStateHdlrFactory& instance(); + private: + bool is_controller_ {false}; + NodeStateHdlrFactory(); + ~NodeStateHdlrFactory(); + + DELETE_COPY_AND_MOVE_OPERATORS(NodeStateHdlrFactory); +}; + +#endif // NODE_STATE_HDLR_FACTORY_H_ diff --git a/tools/devel/fenced/node_state_hdlr_pl.cc b/tools/devel/fenced/node_state_hdlr_pl.cc new file mode 100644 index 000000000..ffb2cc652 --- /dev/null +++ b/tools/devel/fenced/node_state_hdlr_pl.cc @@ -0,0 +1,292 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "node_state_hdlr_pl.h" + +#include <arpa/inet.h> +#include <poll.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <syslog.h> +#include <cinttypes> +#include <cstring> +#include <thread> +#include <utility> + +NodeStateHdlrPl::NodeStateHdlrPl() { + syslog(LOG_INFO, "NodeStatHdlrPl instantiated"); +} + +NodeStateHdlrPl::~NodeStateHdlrPl() { + close(sd_); + + for (auto &kv : node_map_) { + delete kv.second; + } +} + +int NodeStateHdlrPl::init() { + sockaddr_tipc topsrv {}; + tipc_subscr subscr {}; + char *val {nullptr}; + uint32_t isolation_status_check_time_sec {DFLT_ISOLATION_STATUS_CHECK_TIME_SEC}; + uint32_t await_active_time_sec {DFLT_AWAIT_ACTIVE_TIME_SEC}; + + if ((val = getenv("FMD_FENCED_SVC_TYPE")) != NULL) { + fmd_fenced_svc_type_ = strtol(val, nullptr, 0); + } + + if ((val = getenv("FMD_FENCED_SVC_TYPE_ACTIVE")) != NULL) { + fmd_fenced_svc_type_active_ = strtol(val, nullptr, 0); + } + + if ((val = getenv("ISOLATION_STATUS_CHECK_TIME_SEC")) != NULL) { + isolation_status_check_time_sec = strtol(val, nullptr, 0); + } + + if ((val = getenv("AWAIT_ACTIVE_TIME_SEC")) != NULL) { + await_active_time_sec = strtol(val, nullptr, 0); + } + + // seconds to microseconds + isolation_status_check_time_us_ = isolation_status_check_time_sec * kMicrosPerSec; + await_active_time_us_ = await_active_time_sec * kMicrosPerSec; + + // connect to topology server + memset(&topsrv, 0, sizeof(topsrv)); + topsrv.family = AF_TIPC; + topsrv.addrtype = TIPC_ADDR_NAME; + topsrv.addr.name.name.type = TIPC_TOP_SRV; + topsrv.addr.name.name.instance = TIPC_TOP_SRV; + + sd_ = socket(AF_TIPC, SOCK_SEQPACKET, 0); + + if (connect(sd_, reinterpret_cast<sockaddr *>(&topsrv), sizeof(topsrv)) < 0) { + syslog(LOG_ERR, "failed to connect to topology server: %s", strerror(errno)); + return -1; + } + + // subscribe to FMD + subscr.seq.type = htonl(fmd_fenced_svc_type_); + subscr.seq.lower = htonl(0); + subscr.seq.upper = htonl(~0); + subscr.timeout = htonl(TIPC_WAIT_FOREVER); + subscr.filter = htonl(TIPC_SUB_SERVICE); + subscr.usr_handle[0] = static_cast<char>(2); + + if (send(sd_, &subscr, sizeof(subscr), 0) != sizeof(subscr)) { + syslog(LOG_ERR, "failed to subscribe to service event: %s", strerror(errno)); + return -1; + } + + // subscribe to Active FMD + subscr.seq.type = htonl(fmd_fenced_svc_type_active_); + subscr.seq.lower = htonl(0); + subscr.seq.upper = htonl(~0); + subscr.timeout = htonl(TIPC_WAIT_FOREVER); + subscr.filter = htonl(TIPC_SUB_SERVICE); + subscr.usr_handle[0] = static_cast<char>(2); + + if (send(sd_, &subscr, sizeof(subscr), 0) != sizeof(subscr)) { + syslog(LOG_ERR, "failed to subscribe to service event: %s", strerror(errno)); + return -1; + } + + // subscribe to node events + subscr.seq.type = htonl(0); + subscr.seq.lower = htonl(0); + subscr.seq.upper = htonl(~0); + subscr.timeout = htonl(TIPC_WAIT_FOREVER); + subscr.filter = htonl(TIPC_SUB_PORTS); + subscr.usr_handle[0] = static_cast<char>(3); + + if (send(sd_, &subscr, sizeof(subscr), 0) != sizeof(subscr)) { + syslog(LOG_ERR, "failed to subscribe to node events: %s", strerror(errno)); + return -1; + } + + // create receive thread + std::thread t(&NodeStateHdlrPl::receive, this); + t.detach(); + return 0; +} + +void NodeStateHdlrPl::check_isolation() { + int no_of_active = 0; + int no_of_sc = 0; + + for (auto &kv : node_map_) { + if (kv.second->fmdIsActive) { + no_of_active++; + } + if (kv.second->isController && kv.second->nodeIsUp) { + no_of_sc++; + } + } + + if (no_of_sc == 0) { + if (await_act_tmr_status_ == NodeStateHdlrPl::AwaitActiveTmrStatus::kRunning) { + await_act_tmr_.stop(); + } + await_act_tmr_status_ = NodeStateHdlrPl::AwaitActiveTmrStatus::kNotRunning; + isolated_ = NodeIsolationState::kIsolated; + goto notify; + } + + if (no_of_active == 0) { + if (await_act_tmr_status_ == NodeStateHdlrPl::AwaitActiveTmrStatus::kNotRunning) { + await_act_tmr_status_ = NodeStateHdlrPl::AwaitActiveTmrStatus::kRunning; + await_act_tmr_.start(await_active_time_us_, Timer::Type::kOneShot); + syslog(LOG_NOTICE, "await active timer started"); + goto done; + } else if (await_act_tmr_status_ == NodeStateHdlrPl::AwaitActiveTmrStatus::kRunning) { + goto done; + } else { /* timer expired */ + isolated_ = NodeIsolationState::kIsolated; + await_act_tmr_status_ = NodeStateHdlrPl::AwaitActiveTmrStatus::kNotRunning; + syslog(LOG_NOTICE, "no active controller detected"); + } + } else { + if (await_act_tmr_status_ == NodeStateHdlrPl::AwaitActiveTmrStatus::kRunning) { + await_act_tmr_.stop(); + } + await_act_tmr_status_ = NodeStateHdlrPl::AwaitActiveTmrStatus::kNotRunning; + + if (no_of_active == 1) { + isolated_ = NodeIsolationState::kNotIsolated; + syslog(LOG_NOTICE, "one active controller detected"); + } else if (no_of_active == 2) { + isolated_ = NodeIsolationState::kIsolated; + syslog(LOG_NOTICE, "two active controllers detected, split brain"); + } else { + isolated_ = NodeIsolationState::kIsolated; + syslog(LOG_NOTICE, "%d active controllers detected, split brain", no_of_active); + } + } +notify: + send_update(); +done: + return; +} + +void NodeStateHdlrPl::handle_event(const tipc_event &evt) { + NodeState* node_state; + + auto map = node_map_.find(ntohl(evt.port.node)); + + if (map == node_map_.end()) { + node_state = new NodeState; + node_map_.insert(std::make_pair(ntohl(evt.port.node), node_state)); + } else { + node_state = map->second; + } + + if (evt.event == htonl(TIPC_PUBLISHED)) { + syslog(LOG_NOTICE, "TIPC Published: svc %d port: %d", ntohl(evt.s.seq.type), ntohl(evt.port.node)); + if (ntohl(evt.s.seq.type) == fmd_fenced_svc_type_) { + node_state->isController = true; + } else if (ntohl(evt.s.seq.type) == fmd_fenced_svc_type_active_) { + node_state->fmdIsActive = true; + } else if (ntohl(evt.s.seq.type) == 0) { + node_state->nodeIsUp = true; + } + } else if (evt.event == htonl(TIPC_WITHDRAWN)) { + syslog(LOG_NOTICE, "TIPC Withdraw: svc %d port: %d", ntohl(evt.s.seq.type), ntohl(evt.port.node)); + if (ntohl(evt.s.seq.type) == fmd_fenced_svc_type_active_) { + node_state->fmdIsActive = false; + } else if (ntohl(evt.s.seq.type) == 0) { + node_state->nodeIsUp = false; + node_state->isController = false; + } + } +} + +// +void NodeStateHdlrPl::receive() { + Timer tmr; + tipc_event evt; + + enum { + FD_TMR = 0, + FD_TIPC, + FD_AWAIT_ACTIVE_TMR, + NUM_FDS + }; + struct pollfd fds[NUM_FDS]; + int await_act_tmr_fd = await_act_tmr_.get_fd(); + int timer_fd = tmr.start(isolation_status_check_time_us_, Timer::Type::kOneShot); + + fds[FD_TIPC].fd = sd_; + fds[FD_TIPC].events = POLLIN; + + fds[FD_TMR].fd = timer_fd; + fds[FD_TMR].events = POLLIN; + + fds[FD_AWAIT_ACTIVE_TMR].fd = await_act_tmr_.get_fd(); + fds[FD_AWAIT_ACTIVE_TMR].events = POLLIN; + + while (true) { + int rc = poll(fds, NUM_FDS, -1); + if (rc == -1) { + if (errno == EINTR) continue; + syslog(LOG_ERR, "poll failed: %s", strerror(errno)); + break; + } + if (rc > 0) { + if (fds[FD_TIPC].revents == POLLIN) { + for (;;) { + if (recv(sd_, &evt, sizeof(evt), + MSG_DONTWAIT) < 0) { + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + break; + syslog(LOG_ERR, "recv failed: %s", strerror(errno)); + return; + } + handle_event(evt); + } + } + if (fds[FD_TMR].revents == POLLIN) { + uint64_t expirations = 0; + if (read(timer_fd, &expirations, 8) != 8) { + syslog(LOG_WARNING, "error reading timerfd value"); + } else { + if (expirations != 1) { + syslog(LOG_NOTICE, "timerfd expired %" PRIu64 " times", expirations); + } + } + if (isolated_ == NodeIsolationState::kIsolated) { + send_update(); + } + } + if (fds[FD_AWAIT_ACTIVE_TMR].revents == POLLIN) { + uint64_t expirations = 0; + if (read(await_act_tmr_fd, &expirations, 8) != 8) { + syslog(LOG_WARNING, "error reading timerfd value"); + } else { + if (expirations != 1) { + syslog(LOG_NOTICE, "timerfd expired %" PRIu64 " times", expirations); + } + } + // await active timer expired, check if any active is detected + await_act_tmr_status_ = NodeStateHdlrPl::AwaitActiveTmrStatus::kExpired; + } + } + check_isolation(); + } +} diff --git a/tools/devel/fenced/node_state_hdlr_pl.h b/tools/devel/fenced/node_state_hdlr_pl.h new file mode 100644 index 000000000..a90496a5b --- /dev/null +++ b/tools/devel/fenced/node_state_hdlr_pl.h @@ -0,0 +1,60 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef NODE_STATE_HDLR_PL_H_ +#define NODE_STATE_HDLR_PL_H_ + +#include <linux/tipc.h> +#include <cstdint> +#include <map> +#include "cpp_macros.h" +#include "timer.h" +#include "node_state_hdlr.h" + +class NodeStateHdlrPl : public NodeStateHdlr { + public: + NodeStateHdlrPl(); + ~NodeStateHdlrPl(); + int init(); + + private: + const uint32_t DFLT_FMD_FENCED_SVC_TYPE = 19888; + const uint32_t DFLT_FMD_FENCED_SVC_TYPE_ACTIVE = 19889; + const uint32_t DFLT_AWAIT_ACTIVE_TIME_SEC = 10; + + struct NodeState { + bool fmdIsActive {false}; + bool isController {false}; + bool nodeIsUp {false}; + }; + using NodeId = uint32_t; + std::map<NodeId, NodeState*> node_map_; + void handle_event(const tipc_event &evt); + void receive(); + void check_isolation(); + int sd_; + int read_fd_; + uint32_t fmd_fenced_svc_type_ {DFLT_FMD_FENCED_SVC_TYPE}; + uint32_t fmd_fenced_svc_type_active_ {DFLT_FMD_FENCED_SVC_TYPE_ACTIVE}; + uint64_t isolation_status_check_time_us_; + uint64_t await_active_time_us_; + Timer await_act_tmr_; + AwaitActiveTmrStatus await_act_tmr_status_ {AwaitActiveTmrStatus::kNotRunning}; + DELETE_COPY_AND_MOVE_OPERATORS(NodeStateHdlrPl); +}; + +#endif // TIPC_CLIENT_H_ diff --git a/tools/devel/fenced/node_state_hdlr_sc.cc b/tools/devel/fenced/node_state_hdlr_sc.cc new file mode 100644 index 000000000..f1ea57aed --- /dev/null +++ b/tools/devel/fenced/node_state_hdlr_sc.cc @@ -0,0 +1,42 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "node_state_hdlr_sc.h" + +#include <netdb.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <poll.h> +#include <syslog.h> +#include <sys/types.h> +#include <cerrno> +#include <cinttypes> +#include <cstdlib> +#include <cstring> +#include <string> +#include <thread> + +NodeStateHdlrSc::NodeStateHdlrSc() { + syslog(LOG_WARNING, "NodeStatHdlrSc not supported"); +} + +NodeStateHdlrSc::~NodeStateHdlrSc() { +} + +int NodeStateHdlrSc::init() { + return 0; +} diff --git a/tools/devel/fenced/node_state_hdlr_sc.h b/tools/devel/fenced/node_state_hdlr_sc.h new file mode 100644 index 000000000..c598f5760 --- /dev/null +++ b/tools/devel/fenced/node_state_hdlr_sc.h @@ -0,0 +1,41 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef NODE_STATE_HDLR_SC_H_ +#define NODE_STATE_HDLR_SC_H_ + +#include <linux/tipc.h> +#include <netinet/in.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <atomic> +#include <cstdint> + +#include "cpp_macros.h" +#include "node_state_hdlr.h" +#include "timer.h" + +class NodeStateHdlrSc : public NodeStateHdlr { + public: + NodeStateHdlrSc(); + ~NodeStateHdlrSc(); + int init(); + private: + DELETE_COPY_AND_MOVE_OPERATORS(NodeStateHdlrSc); +}; + +#endif // NODE_STATE_HDLR_SC_H_ diff --git a/tools/devel/fenced/osaffenced.service b/tools/devel/fenced/osaffenced.service new file mode 100644 index 000000000..e8a58ea0a --- /dev/null +++ b/tools/devel/fenced/osaffenced.service @@ -0,0 +1,14 @@ +[Unit] +Description=OpenSAF Fencing daemon +After=remote-fs.target local-fs.target syslog.target network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/lib/opensaf/osaffenced +EnvironmentFile=/etc/opensaf/fenced.conf +WatchdogSec=10s +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/tools/devel/fenced/service.cc b/tools/devel/fenced/service.cc new file mode 100644 index 000000000..3d6d8f4f2 --- /dev/null +++ b/tools/devel/fenced/service.cc @@ -0,0 +1,53 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "service.h" + +#include <syslog.h> +#include <algorithm> +#include <cctype> +#include <iterator> +#include <sstream> +#include <vector> + +Service::Service() { +} + +Service::~Service() { +} + +void Service::init() { + std::string str; + char *p = getenv("SERVICES_TO_FENCE"); + + if (p == NULL) { + syslog(LOG_NOTICE, "no services to fence has been configured"); + return; + } else { + str = p; + } + + // services separated by white space + std::istringstream iss{str}; + std::vector<std::string> svc_to_fence(std::istream_iterator<std::string>{iss}, + std::istream_iterator<std::string>()); + + syslog(LOG_NOTICE, "the following services are managed by fenced: %s", str.c_str()); + for (auto it : svc_to_fence) { + services_[it] = {}; + } +} diff --git a/tools/devel/fenced/service.h b/tools/devel/fenced/service.h new file mode 100644 index 000000000..27437031b --- /dev/null +++ b/tools/devel/fenced/service.h @@ -0,0 +1,42 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef SERVICE_H_ +#define SERVICE_H_ + +#include <map> +#include <string> + +#include "cpp_macros.h" + +class Service { + public: + Service(); + ~Service(); + void init(); + + struct SvcInfo { + bool fenced_stopped {false}; + }; + + std::map<std::string, SvcInfo> services_; + + private: + DELETE_COPY_AND_MOVE_OPERATORS(Service); +}; + +#endif // COMMAND_H_ diff --git a/tools/devel/fenced/timer.cc b/tools/devel/fenced/timer.cc new file mode 100644 index 000000000..0f3e56649 --- /dev/null +++ b/tools/devel/fenced/timer.cc @@ -0,0 +1,62 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "timer.h" + +#include <syslog.h> +#include <cerrno> +#include <cstring> +#include <chrono> + +Timer::Timer() { + timer_fd_ = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC); +} + +int Timer::start(uint64_t interval_usec, Type type) { + struct itimerspec spec; + + if (type == Type::kPeriodic) { + spec.it_interval.tv_sec = interval_usec / kMicrosPerSec; + spec.it_interval.tv_nsec = (interval_usec % kMicrosPerSec) * (kNanosPerSec / kMicrosPerSec); + spec.it_value.tv_sec = interval_usec / kMicrosPerSec; + spec.it_value.tv_nsec = (interval_usec % kMicrosPerSec) * (kNanosPerSec / kMicrosPerSec); + } else { + spec.it_interval.tv_sec = 0; + spec.it_interval.tv_nsec = 0; + spec.it_value.tv_sec = interval_usec / kMicrosPerSec; + spec.it_value.tv_nsec = (interval_usec % kMicrosPerSec) * (kNanosPerSec / kMicrosPerSec); + } + + if (timerfd_settime(timer_fd_, 0, &spec, NULL) < 0) { + syslog(LOG_ERR, "timerfd_settime failed: %s", strerror(errno)); + } + return timer_fd_; +} + +void Timer::stop() { + struct itimerspec spec; + + spec.it_interval.tv_sec = 0; + spec.it_interval.tv_nsec = 0; + spec.it_value.tv_sec = 0; + spec.it_value.tv_nsec = 0; + + if (timerfd_settime(timer_fd_, 0, &spec, NULL) < 0) { + syslog(LOG_ERR, "timerfd_settime failed: %s", strerror(errno)); + } +} + diff --git a/tools/devel/fenced/timer.h b/tools/devel/fenced/timer.h new file mode 100644 index 000000000..0c384dd48 --- /dev/null +++ b/tools/devel/fenced/timer.h @@ -0,0 +1,53 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef TIMER_H_ +#define TIMER_H_ + +#include <sys/timerfd.h> +#include <unistd.h> +#include <cstdint> + +#include "cpp_macros.h" + +enum { + // Number of nanoseconds per second. + kNanosPerSec = 1000000000, + + // Number of microseconds per second. + kMicrosPerSec = 1000000, + + // Number of milliseconds per second. + kMillisPerSec = 1000 +}; + +class Timer { + public: + enum class Type {kOneShot, kPeriodic}; + + Timer(); + ~Timer() {close(timer_fd_);} + int start(uint64_t interval_usec, Type type); + void stop(); + int get_fd() const {return timer_fd_;} + protected: + private: + int timer_fd_; + DELETE_COPY_AND_MOVE_OPERATORS(Timer); +}; + +#endif // TIMER_H_ diff --git a/tools/devel/fenced/watchdog.cc b/tools/devel/fenced/watchdog.cc new file mode 100644 index 000000000..ebe8b769a --- /dev/null +++ b/tools/devel/fenced/watchdog.cc @@ -0,0 +1,37 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "watchdog.h" + +#include <syslog.h> +#include <systemd/sd-daemon.h> +#include <cstring> + +Watchdog::Watchdog() { + interval_usec_ = DFLT_WATCHDOG_TIMEOUT; + int rc = sd_watchdog_enabled(0, &interval_usec_); + if (rc < 0) { + syslog(LOG_WARNING, "sd_watchdog_enabled failed: %s", strerror(-rc)); + } +} + +void Watchdog::kick() { + int rc = sd_notify(0, "WATCHDOG=1"); + if (rc < 0) { + syslog(LOG_WARNING, "sd_notify failed: %s", strerror(-rc)); + } +} diff --git a/tools/devel/fenced/watchdog.h b/tools/devel/fenced/watchdog.h new file mode 100644 index 000000000..9c0505502 --- /dev/null +++ b/tools/devel/fenced/watchdog.h @@ -0,0 +1,39 @@ +/* -*- OpenSAF -*- + * + * Copyright Ericsson AB 2019 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef WATCHDOG_H_ +#define WATCHDOG_H_ + +#include <cstdint> + +#include "cpp_macros.h" +#include "timer.h" + +class Watchdog { + public: + const uint64_t DFLT_WATCHDOG_TIMEOUT = 10 * kMicrosPerSec; + Watchdog(); + ~Watchdog() {} + void kick(); + uint64_t interval_usec() const {return interval_usec_;} + protected: + private: + uint64_t interval_usec_; + DELETE_COPY_AND_MOVE_OPERATORS(Watchdog); +}; + +#endif // WATCHDOG_H_ -- 2.17.1 _______________________________________________ Opensaf-devel mailing list Opensaf-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/opensaf-devel