Add an external check which makes use of an external process to
check the status of a server.
---
v6
* Correct implementation and documentation of arguments to external-check
command so that they are consistent with both each other and ldirectord's
external check. The motivation being to allow the same scripts to be used
with both haproxy and ldirectord.
v5
* Rebase
v4
* Remove stray use of s->check in process_chk()
The check parameter should be used throughout process_chk()
* Layer 7 timeouts of agent checks should be ignored
* Ensure that argc is never used uninitialised in prepare_external_check()
v3
* Rebase: basically a rewrite of large sections of the code
* Merge with the following patches
+ "external-check: Actually execute command"
+ "Allow selection of of external-check in configuration file"
v2
* If the external command exits normally (WIFEXITED()) is true)
then set the check's code to the exit status (WEXITSTATUS())
of the process.
* Treat a timeout is a failure case rather than the test having passed
* Remove duplicate getnameinfo() call in start_checks()
* Remove duplicate assignment of sockaddr argument to getnameinfo(9
which caused the check port and check addr configuration of
a server to be ignored.
---
doc/configuration.txt | 26 ++++
include/types/checks.h | 4 +
include/types/proxy.h | 1 +
include/types/server.h | 26 +++-
src/cfgparse.c | 20 +++
src/checks.c | 339 +++++++++++++++++++++++++++++++++++++++++++++++--
6 files changed, 398 insertions(+), 18 deletions(-)
diff --git a/doc/configuration.txt b/doc/configuration.txt
index a82d48a..2d7af6d 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -3995,6 +3995,32 @@ option ldap-check
See also : "option httpchk"
+option external-check
+ Use external processes for server health checks
+ May be used in sections : defaults | frontend | listen | backend
+ yes | no | yes | yes
+ Arguments : external command to run
+
+ It is possible to test the health of a server using an external command.
+ This is achieved by running the command given as the argument to the
+ external-check option. The arguments to the command are:
+
+ proxy_address proxy_port server_address server_port
+
+ The proxy_address and proxy_port are derived from the first listener
+ that is either IPv4, IPv6 or a UNIX socket. It is an error for no such
+ listeners to exist. In the case of a UNIX socket listener the
+ proxy_address will be the path of the socket and the proxy_port will
+ be the string "NOT_USED".
+
+ If the command executed and exits with a zero status then the check is
+ considered to have passed, otherwise the check is considered to have
+ failed.
+
+ Example :
+ option external-check /bin/true
+
+
option log-health-checks
no option log-health-checks
Enable or disable logging of health checks
diff --git a/include/types/checks.h b/include/types/checks.h
index 09a4eee..731db37 100644
--- a/include/types/checks.h
+++ b/include/types/checks.h
@@ -44,6 +44,10 @@ enum {
HCHK_STATUS_L7OKCD, /* L7 check conditionally passed */
HCHK_STATUS_L7STS, /* L7 response error, for example HTTP
5xx */
+ HCHK_STATUS_PROCERR, /* External process check failure */
+ HCHK_STATUS_PROCTOUT, /* External process check timeout */
+ HCHK_STATUS_PROCOK, /* External process check passed */
+
HCHK_STATUS_SIZE
};
diff --git a/include/types/proxy.h b/include/types/proxy.h
index 66e5db7..e608b9f 100644
--- a/include/types/proxy.h
+++ b/include/types/proxy.h
@@ -152,6 +152,7 @@ enum {
#define PR_O2_LDAP_CHK 0x60000000 /* use LDAP check for server health */
#define PR_O2_SSL3_CHK 0x70000000 /* use SSLv3 CLIENT_HELLO packets for
server health */
#define PR_O2_LB_AGENT_CHK 0x80000000 /* use a TCP connection to obtain a
metric of server health */
+#define PR_O2_EXT_CHK 0x90000000 /* use external command for server
health */
/* unused: 0x90000000 to 0xF000000, reserved for health checks */
#define PR_O2_CHK_ANY 0xF0000000 /* Mask to cover any check */
/* end of proxy->options2 */
diff --git a/include/types/server.h b/include/types/server.h
index 24462eb..80db519 100644
--- a/include/types/server.h
+++ b/include/types/server.h
@@ -24,6 +24,7 @@
#include <netinet/in.h>
#include <arpa/inet.h>
+#include <stdbool.h>
#ifdef USE_OPENSSL
#include <openssl/ssl.h>
@@ -99,6 +100,14 @@
#define SRV_SSL_O_NO_TLS_TICKETS 0x0100 /* disable session resumption tickets
*/
#endif
+struct pid_list {
+ struct list list;
+ pid_t pid;
+ struct task *t;
+ int status;
+ bool exited;
+};
+
/* A tree occurrence is a descriptor of a place in a tree, with a pointer back
* to the server itself.
*/
@@ -109,17 +118,24 @@ struct tree_occ {
};
struct check {
- struct connection *conn; /* connection state for health
checks */
-
+ union {
+ struct {
+ struct connection *conn;/* connection state for health
checks */
+ struct buffer *bi, *bo; /* input and output buffers to
send/recv check */
+ int use_ssl; /* use SSL for health checks */
+ int send_proxy; /* send a PROXY protocol header
with checks */
+ };
+ struct {
+ char **argv; /* the arguments to use if
running a process-based check */
+ struct pid_list *curpid;/* entry in pid_list used for
current process-based test, or -1 if not in test */
+ };
+ };
short port; /* the port to use for the
health checks */
- struct buffer *bi, *bo; /* input and output buffers to
send/recv check */
struct task *task; /* the task associated to the
health check processing, NULL if disabled */
struct timeval start; /* last health check start time
*/
long duration; /* time in ms took to finish
last health check */
short status, code; /* check result, check code */
char desc[HCHK_DESC_LEN]; /* health check descritpion */
- int use_ssl; /* use SSL for health checks */
- int send_proxy; /* send a PROXY protocol header
with checks */
int inter, fastinter, downinter; /* checks: time in milliseconds
*/
int result; /* health-check result :
SRV_CHK_* */
int state; /* health-check result : CHK_*
*/
diff --git a/src/cfgparse.c b/src/cfgparse.c
index f67cb7b..79c0c9e 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -1630,6 +1630,10 @@ static int init_check(struct check *check, int type,
const char * file, int line
{
check->type = type;
+ if (check->type == PR_O2_EXT_CHK)
+ /* Nothing left to do for external checks */
+ return 0;
+
/* Allocate buffer for requests... */
if ((check->bi = calloc(sizeof(struct buffer) + global.tune.chksize,
sizeof(char))) == NULL) {
Alert("parsing [%s:%d] : out of memory while allocating check
buffer.\n", file, linenum);
@@ -3739,6 +3743,22 @@ stats_error_parsing:
memcpy(curproxy->check_req, DEF_LDAP_CHECK_REQ,
sizeof(DEF_LDAP_CHECK_REQ) - 1);
curproxy->check_len = sizeof(DEF_LDAP_CHECK_REQ) - 1;
}
+ else if (!strcmp(args[1], "external-check")) {
+ /* excute an external command to check servers' health
*/
+ free(curproxy->check_req);
+ curproxy->check_req = NULL;
+ curproxy->options2 &= ~PR_O2_CHK_ANY;
+ curproxy->options2 |= PR_O2_EXT_CHK;
+
+ if (!*(args[2])) {
+ Alert("parsing [%s:%d] : '%s' expects command
as argument.\n",
+ file, linenum, args[0]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+ curproxy->check_req = strdup(args[2]);
+ curproxy->check_len = strlen(curproxy->check_req);
+ }
else if (!strcmp(args[1], "forwardfor")) {
int cur_arg;
diff --git a/src/checks.c b/src/checks.c
index d2c43d2..f934336 100644
--- a/src/checks.c
+++ b/src/checks.c
@@ -22,9 +22,11 @@
#include <unistd.h>
#include <sys/socket.h>
#include <sys/types.h>
+#include <sys/wait.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
+#include <netdb.h>
#include <common/chunk.h>
#include <common/compat.h>
@@ -78,6 +80,10 @@ static const struct check_status
check_statuses[HCHK_STATUS_SIZE] = {
[HCHK_STATUS_L7OKD] = { SRV_CHK_PASSED, "L7OK",
"Layer7 check passed" },
[HCHK_STATUS_L7OKCD] = { SRV_CHK_PASSED | SRV_CHK_DISABLE, "L7OKC",
"Layer7 check conditionally passed" },
[HCHK_STATUS_L7STS] = { SRV_CHK_FAILED, "L7STS",
"Layer7 wrong status" },
+
+ [HCHK_STATUS_PROCERR] = { SRV_CHK_FAILED,
"PROCERR", "External check error" },
+ [HCHK_STATUS_PROCTOUT] = { SRV_CHK_FAILED,
"PROCTOUT", "External check timeout" },
+ [HCHK_STATUS_PROCOK] = { SRV_CHK_PASSED,
"PROCOK", "External check passed" },
};
static const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = {
/* 0: ignore, 1: error, 2: OK */
@@ -231,7 +237,7 @@ static void set_server_check_status(struct check *check,
short status, const cha
/* Failure to connect to the agent as a secondary check should not
* cause the server to be marked down. So only log status changes
* for HCHK_STATUS_* statuses */
- if (check == &s->agent && check->status < HCHK_STATUS_L7TOUT)
+ if (check == &s->agent && check->status <= HCHK_STATUS_L7TOUT)
return;
if (s->proxy->options2 & PR_O2_LOGHCHKS &&
@@ -1385,7 +1391,7 @@ static void process_result(struct check *check)
}
/*
- * establish a server health-check.
+ * establish a server health-check that makes use of a connection.
*
* It can return one of :
* - SN_ERR_NONE if everything's OK
@@ -1396,7 +1402,7 @@ static void process_result(struct check *check)
* - SN_ERR_INTERNAL for any other purely internal errors
* Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be
emitted.
*/
-static int establish_chk(struct task *t)
+static int establish_conn_chk(struct task *t)
{
struct check *check = t->context;
struct server *s = check->server;
@@ -1453,6 +1459,281 @@ static int establish_chk(struct task *t)
return ret;
}
+static struct list pid_list = LIST_HEAD_INIT(pid_list);
+static struct pool_head *pool2_pid_list;
+
+void block_sigchld(void)
+{
+ sigset_t set;
+ sigemptyset(&set);
+ sigaddset(&set, SIGCHLD);
+ assert(sigprocmask(SIG_SETMASK, &set, NULL) == 0);
+}
+
+void unblock_sigchld(void)
+{
+ sigset_t set;
+ sigemptyset(&set);
+ assert(sigprocmask(SIG_SETMASK, &set, NULL) == 0);
+}
+
+/* Call with SIGCHLD blocked */
+static struct pid_list *pid_list_add(pid_t pid, struct task *t)
+{
+ struct pid_list *elem;
+ struct check *check = t->context;
+
+ elem = pool_alloc2(pool2_pid_list);
+ if (!elem)
+ return NULL;
+ elem->pid = pid;
+ elem->t = t;
+ elem->exited = 0;
+ check->curpid = elem;
+ LIST_INIT(&elem->list);
+ LIST_ADD(&pid_list, &elem->list);
+ return elem;
+}
+
+/* Blocks blocks and then unblocks SIGCHLD */
+static void pid_list_del(struct pid_list *elem)
+{
+ struct check *check;
+
+ if (!elem)
+ return;
+
+ block_sigchld();
+ LIST_DEL(&elem->list);
+ unblock_sigchld();
+ if (!elem->exited)
+ kill(elem->pid, SIGTERM);
+
+ check = elem->t->context;
+ check->curpid = NULL;
+ pool_free2(pool2_pid_list, elem);
+}
+
+/* Called from inside SIGCHLD handler, SIGCHLD is blocked */
+static void pid_list_expire(pid_t pid, int status)
+{
+ struct pid_list *elem;
+
+ list_for_each_entry(elem, &pid_list, list) {
+ if (elem->pid == pid) {
+ elem->t->expire = now_ms;
+ elem->status = status;
+ elem->exited = 1;
+ return;
+ }
+ }
+}
+
+static void sigchld_handler(int signal)
+{
+ pid_t pid;
+ int status;
+ while ((pid = waitpid(0, &status, WNOHANG)) > 0)
+ pid_list_expire(pid, status);
+}
+
+static int init_pid_list(void) {
+ struct sigaction action = {
+ .sa_handler = sigchld_handler,
+ .sa_flags = SA_NOCLDSTOP
+ };
+
+ if (pool2_pid_list != NULL)
+ /* Nothing to do */
+ return 0;
+
+ if (sigaction(SIGCHLD, &action, NULL)) {
+ Alert("Failed to set signal handler for external health checks:
%s. Aborting.\n",
+ strerror(errno));
+ return 1;
+ }
+
+ pool2_pid_list = create_pool("pid_list", sizeof(struct pid_list),
MEM_F_SHARED);
+ if (pool2_pid_list == NULL) {
+ Alert("Failed to allocate memory pool for external health
checks: %s. Aborting.\n",
+ strerror(errno));
+ return 1;
+ }
+
+ return 0;
+}
+
+
+static int prepare_external_check(struct check *check)
+{
+ char host[NI_MAXHOST] = {};
+ char serv[NI_MAXSERV] = {};
+ struct server *s = check->server;
+ struct proxy *px = s->proxy;
+ struct listener *listener = NULL, *l;
+ int i;
+ const char *err_fmt = "Starting [%s:%s] check: out of memory.\n";
+
+ list_for_each_entry(l, &px->conf.listeners, by_fe)
+ /* Use the first INET, INET6 or UNIX listener */
+ if (l->addr.ss_family == AF_INET ||
+ l->addr.ss_family == AF_INET6 ||
+ l->addr.ss_family == AF_UNIX) {
+ listener = l;
+ break;
+ }
+
+ if (!listener) {
+ err_fmt = "Starting [%s:%s] check: no listener.\n";
+ goto err;
+ }
+
+ check->curpid = NULL;
+
+ check->argv = calloc(6, sizeof(check->argv));
+ if (!check->argv)
+ goto err;
+
+ check->argv[0] = px->check_req;
+
+ if (listener->addr.ss_family == AF_INET ||
+ listener->addr.ss_family == AF_INET6) {
+ if (getnameinfo((struct sockaddr *)&listener->addr,
+ sizeof(listener->addr), host, sizeof(host),
+ serv, sizeof(serv),
+ NI_NUMERICHOST | NI_NUMERICSERV)) {
+ err_fmt = "Starting [%s:%s] check: getnameinfo failed
on listener address.\n";
+ goto err_free;
+ }
+ check->argv[1] = strdup(host);
+ check->argv[2] = strdup(serv);
+ } else if (listener->addr.ss_family == AF_UNIX) {
+ const struct sockaddr_un *un;
+
+ un = (struct sockaddr_un *)&listener->addr;
+ check->argv[1] = strdup(un->sun_path);
+ check->argv[2] = strdup("NOT_USED");
+ } else {
+ goto err_free;
+ }
+
+ if (getnameinfo((struct sockaddr *)&s->addr, sizeof(s->addr),
+ host, sizeof(host), serv, sizeof(serv),
+ NI_NUMERICHOST | NI_NUMERICSERV)) {
+ err_fmt = "Starting [%s:%s] check: getnameinfo failed on check
address.\n";
+ goto err_free;
+ }
+ check->argv[3] = strdup(host);
+ check->argv[4] = strdup(serv);
+
+ for (i = 0; i < 5; i++)
+ if (!check->argv[i])
+ goto err_free;
+
+ return 0;
+err_free:
+ for (i = 1; i < 5; i++)
+ free(check->argv[i]);
+ free(check->argv);
+ check->argv = NULL;
+err:
+ Alert(err_fmt, px->id, s->id);
+ return -1;
+}
+
+/*
+ * establish a server health-check that makes use of a process.
+ *
+ * It can return one of :
+ * - SN_ERR_NONE if everything's OK
+ * - SN_ERR_SRVTO if there are no more servers
+ * - SN_ERR_SRVCL if the connection was refused by the server
+ * - SN_ERR_PRXCOND if the connection has been limited by the proxy (maxconn)
+ * - SN_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports,
...)
+ * - SN_ERR_INTERNAL for any other purely internal errors
+ * Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be
emitted.
+ *
+ * Blocks and then unblocks SIGCHLD
+ */
+static int establish_proc_chk(struct task *t)
+{
+ struct check *check = t->context;
+ struct server *s = check->server;
+ struct proxy *px = s->proxy;
+ int status;
+ pid_t pid;
+
+ if (!check->argv) {
+ status = prepare_external_check(check);
+ if (status < 0)
+ return SN_ERR_RESOURCE;
+ }
+
+ status = SN_ERR_RESOURCE;
+
+ block_sigchld();
+
+ pid = fork();
+ if (pid < 0) {
+ Alert("Failed to fork process for external health check: %s.
Aborting.\n",
+ strerror(errno));
+ set_server_check_status(check, HCHK_STATUS_SOCKERR,
strerror(errno));
+ goto out;
+ }
+ if (pid == 0) {
+ /* Child */
+ execvp(px->check_req, check->argv);
+ Alert("Failed to exec process for external health check: %s.
Aborting.\n",
+ strerror(errno));
+ exit(-1);
+ }
+
+ /* Parent */
+ if (check->result == SRV_CHK_UNKNOWN) {
+ if (pid_list_add(pid, t) != NULL) {
+ t->expire = tick_add(now_ms, MS_TO_TICKS(check->inter));
+
+ if (px->timeout.check && px->timeout.connect) {
+ int t_con = tick_add(now_ms,
px->timeout.connect);
+ t->expire = tick_first(t->expire, t_con);
+ }
+ status = SN_ERR_NONE;
+ goto out;
+ }
+ else {
+ set_server_check_status(check, HCHK_STATUS_SOCKERR,
strerror(errno));
+ }
+ kill(pid, SIGTERM); /* process creation error */
+ }
+ else
+ set_server_check_status(check, HCHK_STATUS_SOCKERR,
strerror(errno));
+
+out:
+ unblock_sigchld();
+ return status;
+}
+
+/*
+ * establish a server health-check.
+ *
+ * It can return one of :
+ * - SN_ERR_NONE if everything's OK
+ * - SN_ERR_SRVTO if there are no more servers
+ * - SN_ERR_SRVCL if the connection was refused by the server
+ * - SN_ERR_PRXCOND if the connection has been limited by the proxy (maxconn)
+ * - SN_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports,
...)
+ * - SN_ERR_INTERNAL for any other purely internal errors
+ * Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be
emitted.
+ */
+static int establish_chk(struct task *t)
+{
+ struct check *check = t->context;
+
+ if (check->type == PR_O2_EXT_CHK)
+ return establish_proc_chk(t);
+ return establish_conn_chk(t);
+}
+
/*
* manages a server health-check. Returns
* the time the task accepts to wait, or TIME_ETERNITY for infinity.
@@ -1465,6 +1746,7 @@ static struct task *process_chk(struct task *t)
int rv;
int ret;
int expired = tick_is_expired(t->expire, now_ms);
+ int has_conn = check->type != PR_O2_EXT_CHK;
if (!(check->state & CHK_RUNNING)) {
/* no check currently running */
@@ -1485,10 +1767,13 @@ static struct task *process_chk(struct task *t)
set_server_check_status(check, HCHK_STATUS_START, NULL);
check->state |= CHK_RUNNING;
- check->bi->p = check->bi->data;
- check->bi->i = 0;
- check->bo->p = check->bo->data;
- check->bo->o = 0;
+
+ if (has_conn) {
+ check->bi->p = check->bi->data;
+ check->bi->i = 0;
+ check->bo->p = check->bo->data;
+ check->bo->o = 0;
+ }
ret = establish_chk(t);
switch (ret) {
@@ -1503,7 +1788,9 @@ static struct task *process_chk(struct task *t)
int t_con = tick_add(now_ms,
s->proxy->timeout.connect);
t->expire = tick_first(t->expire, t_con);
}
- conn_data_poll_recv(conn); /* prepare for reading a
possible reply */
+ if (has_conn) {
+ conn_data_poll_recv(conn); /* prepare for
reading a possible reply */
+ }
goto reschedule;
case SN_ERR_SRVTO: /* ETIMEDOUT */
@@ -1541,15 +1828,15 @@ static struct task *process_chk(struct task *t)
* First, let's check whether there was an uncaught error,
* which can happen on connect timeout or error.
*/
- if (s->check.result == SRV_CHK_UNKNOWN) {
- if ((conn->flags &
(CO_FL_CONNECTED|CO_FL_WAIT_L4_CONN)) == CO_FL_WAIT_L4_CONN) {
+ if (check->result == SRV_CHK_UNKNOWN) {
+ if (has_conn && (conn->flags &
(CO_FL_CONNECTED|CO_FL_WAIT_L4_CONN)) == CO_FL_WAIT_L4_CONN) {
/* L4 not established (yet) */
if (conn->flags & CO_FL_ERROR)
set_server_check_status(check,
HCHK_STATUS_L4CON, NULL);
else if (expired)
set_server_check_status(check,
HCHK_STATUS_L4TOUT, NULL);
}
- else if ((conn->flags &
(CO_FL_CONNECTED|CO_FL_WAIT_L6_CONN)) == CO_FL_WAIT_L6_CONN) {
+ else if (has_conn && (conn->flags &
(CO_FL_CONNECTED|CO_FL_WAIT_L6_CONN)) == CO_FL_WAIT_L6_CONN) {
/* L6 not established (yet) */
if (conn->flags & CO_FL_ERROR)
set_server_check_status(check,
HCHK_STATUS_L6RSP, NULL);
@@ -1563,6 +1850,23 @@ static struct task *process_chk(struct task *t)
else
set_server_check_status(check,
HCHK_STATUS_L4OK, NULL);
}
+ else if (!has_conn) {
+ struct pid_list *elem = check->curpid;
+ int status;
+
+ if (elem->exited) {
+ status = elem->status; /* Save in case
the process exits between use below */
+ if (!WIFEXITED(status))
+ check->code = -1;
+ else
+ check->code =
WEXITSTATUS(status);
+ if (!WIFEXITED(status) ||
WEXITSTATUS(status))
+ status = HCHK_STATUS_PROCERR;
+ else
+ status = HCHK_STATUS_PROCOK;
+ set_server_check_status(check, status,
NULL);
+ }
+ }
else if (expired) {
/* connection established but expired check */
if (check->type == PR_O2_SSL3_CHK)
@@ -1576,7 +1880,7 @@ static struct task *process_chk(struct task *t)
}
/* check complete or aborted */
- if (conn->xprt) {
+ if (has_conn && conn->xprt) {
/* The check was aborted and the connection was not yet
closed.
* This can happen upon timeout, or when an external
event such
* as a failed response coupled with "observe layer7"
caused the
@@ -1600,6 +1904,9 @@ static struct task *process_chk(struct task *t)
process_result(&check->server->agent);
}
+ if (!has_conn)
+ pid_list_del(check->curpid);
+
rv = 0;
if (global.spread_checks > 0) {
rv = srv_getinter(check) * global.spread_checks / 100;
@@ -1614,7 +1921,6 @@ static struct task *process_chk(struct task *t)
out_wait:
return t;
}
-
static int start_check_task(struct check *check, int mininter,
int nbcheck, int srvpos)
{
@@ -1683,6 +1989,13 @@ int start_checks() {
* the number of servers, weighted by the server's position in the list.
*/
for (px = proxy; px; px = px->next) {
+ if ((px->options2 & PR_O2_CHK_ANY) == PR_O2_EXT_CHK) {
+ if (init_pid_list()) {
+ Alert("Starting [%s] check: out of memory.\n",
px->id);
+ return -1;
+ }
+ }
+
for (s = px->srv; s; s = s->next) {
if (s->slowstart) {
if ((t = task_new()) == NULL) {
--
1.8.3.2
