When a socket is closed, typically because a proxy is deleted, don't close the file descriptor. Rather keep a cache which can be looked-up and used when a subsequent proxy is started.
The aim is to avoid unbinding and binding to ports and thus interrupting services unnecessarily. This will be used in conjunction with a facility to allow haproxy to reinitialise itself. --- include/proto/fd.h | 5 ++ include/types/fd.h | 28 ++++++++++ include/types/global.h | 1 + src/fd.c | 133 ++++++++++++++++++++++++++++++++++++++++++++++++ src/haproxy.c | 4 ++ src/proto_tcp.c | 10 +++- src/proto_uxst.c | 12 ++++- 7 files changed, 191 insertions(+), 2 deletions(-) diff --git a/include/proto/fd.h b/include/proto/fd.h index 1cba33b..3ca4a97 100644 --- a/include/proto/fd.h +++ b/include/proto/fd.h @@ -87,6 +87,11 @@ static inline void fd_insert(int fd) maxfd = fd + 1; } +void socket_cache_make_all_available(void); +int socket_cache_get(const struct listener *listener); +void socket_cache_gc(void); +int socket_cache_add(int fd, struct listener *listener); + #endif /* _PROTO_FD_H */ diff --git a/include/types/fd.h b/include/types/fd.h index f698863..eb253a5 100644 --- a/include/types/fd.h +++ b/include/types/fd.h @@ -136,6 +136,34 @@ extern int maxfd; /* # of the highest fd + 1 */ extern int totalconn; /* total # of terminated sessions */ extern int actconn; /* # of active sessions */ +/* The socket cache allows bound sockets to be looked up + * by the code that would bind them + */ +struct socket_cache { + struct socket_cache *next; /* next entry in the cache, or NULL */ + int fd; /* the listen socket */ + int state; /* state: NEW, ASSIGNED */ + char *interface; /* interface name or NULL */ + + /* Body */ + int sock_type; /* As passed to socket() */ + int sock_prot; /* As passed to socket() */ + socklen_t sock_addrlen; /* socket address length, used by bind() */ + struct sockaddr_storage addr; /* the address we listen to */ + int options; /* socket options : LI_O_* */ + int maxconn; /* maximum connections allowed on this listener */ + unsigned int backlog; /* if set, listen backlog */ + union { /* protocol-dependant access restrictions */ + struct { /* UNIX socket permissions */ + uid_t uid; /* -1 to leave unchanged */ + gid_t gid; /* -1 to leave unchanged */ + mode_t mode; /* 0 to leave unchanged */ + int level; /* access level (ACCESS_LVL_*) */ + } ux; + } perm; + int maxseg; /* for TCP, advertised MSS */ +}; + #endif /* _TYPES_FD_H */ /* diff --git a/include/types/global.h b/include/types/global.h index b6c60dd..535b833 100644 --- a/include/types/global.h +++ b/include/types/global.h @@ -121,6 +121,7 @@ extern const int zero; extern const int one; extern const struct linger nolinger; extern int stopping; /* non zero means stopping in progress */ +extern int is_master; extern char hostname[MAX_HOSTNAME_LEN]; extern char localpeer[MAX_HOSTNAME_LEN]; diff --git a/src/fd.c b/src/fd.c index 80bddd6..c2f16bb 100644 --- a/src/fd.c +++ b/src/fd.c @@ -18,6 +18,8 @@ #include <common/compat.h> #include <common/config.h> +#include <types/protocols.h> + #include <proto/fd.h> #include <proto/port_range.h> @@ -31,6 +33,7 @@ struct poller pollers[MAX_POLLERS]; struct poller cur_poller; int nbpollers = 0; +static struct socket_cache *socket_cache = NULL; /* Deletes an FD from the fdsets, and recomputes the maxfd limit. * The file descriptor is also closed. @@ -174,6 +177,136 @@ int fork_poller() return 1; } +enum { + SC_AVAILABLE, + SC_INVALID, + SC_INUSE +}; + +void socket_cache_make_all_available(void) +{ + struct socket_cache *e; + + for (e = socket_cache; e; e = e->next) + e->state = SC_AVAILABLE; +} + +void socket_cache_gc(void) +{ + struct socket_cache *e, *next, *prev = NULL; + + for (e = socket_cache; e; e = next) { + next = e->next; + if (e->state == SC_INUSE) { + prev = e; + continue; + } + if (prev) + prev->next = e->next; + else + socket_cache = e->next; + if (e->state != SC_INVALID) + fd_delete(e->fd); + free(e); + } +} + +static void socket_cache_body_assign(struct socket_cache *e, + const struct listener *listener) +{ + e->sock_type = listener->proto->sock_type; + e->sock_prot = listener->proto->sock_prot; + e->sock_addrlen = listener->proto->sock_addrlen; + e->options = listener->options & + (LI_O_NOLINGER|LI_O_FOREIGN|LI_O_NOQUICKACK|LI_O_DEF_ACCEPT); + e->addr = listener->addr; + e->maxconn = listener->maxconn; + e->backlog = listener->backlog; + memcpy(&e->perm, &listener->perm, sizeof(e->perm)); + e->maxseg = listener->maxseg; +} + +#define offsetof(type, member) __builtin_offsetof (type, member) + +static int socket_cache_cmp(const struct socket_cache *a, + const struct socket_cache *b) +{ + size_t start = offsetof(typeof(*a), sock_type); + size_t end = offsetof(typeof(*a), addr) + sizeof(a->addr); + + return memcmp((const char *)a + start, (const char *)b + start, + end - start); +} + +static int socket_cache_cmp_detail(const struct socket_cache *a, + const struct socket_cache *b) +{ + size_t start = offsetof(typeof(*a), options); + size_t end = offsetof(typeof(*a), maxseg) + sizeof(a->maxseg); + + if ((a->interface || b->interface) && + (!a->interface || !b->interface || + strcmp(a->interface, b->interface))) + return -1; + + return memcmp((const char *)a + start, (const char *)b + start, + end - start); +} + +int socket_cache_get(const struct listener *listener) +{ + struct socket_cache a = {}, *b; + + socket_cache_body_assign(&a, listener); + a.interface = listener->interface; + + /* First find a cache entry whose type, protocol and address match + * which is available. There should only ever be at most one match. + * If this match matches the listener's details then use it. + * Else close its fd and invalidate it as it is of no use + * and will prevent binding of a fresh socket. + */ + for (b = socket_cache; b; b = b->next) { + if (b->state == SC_AVAILABLE && !socket_cache_cmp(&a, b)) { + if (!socket_cache_cmp_detail(&a, b)) { + b->state = SC_INUSE; + return b->fd; + } + fd_delete(b->fd); + b->state = SC_INVALID; + break; + } + } + + return -1; +} + +int socket_cache_add(int fd, struct listener *listener) +{ + struct socket_cache *e; + + e = calloc(1, sizeof(struct socket_cache)); + if (!e) + return -1; + + e->fd = fd; + e->state = SC_INUSE; + if (listener->interface) { + e->interface = strdup(listener->interface); + if (!e->interface) { + free(e); + return -1; + } + } + socket_cache_body_assign(e, listener); + + if (socket_cache) + e->next = socket_cache; + socket_cache = e; + + return 0; +} + /* * Local variables: * c-indent-level: 8 diff --git a/src/haproxy.c b/src/haproxy.c index 577de61..363c5d1 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -1039,6 +1039,8 @@ int main(int argc, char **argv) FILE *pidfile = NULL; char errmsg[100]; + socket_cache_make_all_available(); + init(argc, argv); signal_register_fct(SIGQUIT, dump, SIGQUIT); signal_register_fct(SIGUSR1, sig_soft_stop, SIGUSR1); @@ -1227,6 +1229,8 @@ int main(int argc, char **argv) argv[0], (int)limit.rlim_cur, global.maxconn, global.maxsock, global.maxsock); } + socket_cache_gc(); + if (global.mode & MODE_DAEMON) { struct proxy *px; int ret = 0; diff --git a/src/proto_tcp.c b/src/proto_tcp.c index 6328d0a..eb46796 100644 --- a/src/proto_tcp.c +++ b/src/proto_tcp.c @@ -447,6 +447,7 @@ int tcp_bind_listener(struct listener *listener, char *errmsg, int errlen) __label__ tcp_return, tcp_close_return; int fd, err; const char *msg = NULL; + int is_cached = 0; /* ensure we never return garbage */ if (errmsg && errlen) @@ -457,7 +458,9 @@ int tcp_bind_listener(struct listener *listener, char *errmsg, int errlen) err = ERR_NONE; - if ((fd = socket(listener->addr.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((fd = socket_cache_get(listener)) >= 0) + is_cached = 1; + else if ((fd = socket(listener->addr.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { err |= ERR_RETRYABLE | ERR_ALERT; msg = "cannot create listening socket"; goto tcp_return; @@ -469,6 +472,9 @@ int tcp_bind_listener(struct listener *listener, char *errmsg, int errlen) goto tcp_close_return; } + if (is_cached) + goto cached; + if (fcntl(fd, F_SETFL, O_NONBLOCK) == -1) { err |= ERR_FATAL | ERR_ALERT; msg = "cannot make socket non-blocking"; @@ -544,6 +550,8 @@ int tcp_bind_listener(struct listener *listener, char *errmsg, int errlen) setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, (char *) &zero, sizeof(zero)); #endif + socket_cache_add(fd, listener); + cached: /* the socket is ready */ listener->fd = fd; listener->state = LI_LISTEN; diff --git a/src/proto_uxst.c b/src/proto_uxst.c index 37abb4c..9dbba9d 100644 --- a/src/proto_uxst.c +++ b/src/proto_uxst.c @@ -132,9 +132,17 @@ static int uxst_bind_listener(struct listener *listener, char *errmsg, int errle if (listener->state != LI_ASSIGNED) return ERR_NONE; /* already bound */ - + path = ((struct sockaddr_un *)&listener->addr)->sun_path; + if ((fd = socket_cache_get(listener)) >= 0) { + if (fd >= global.maxsock) { + msg = "socket(): not enough free sockets, raise -n argument"; + goto err_unlink_temp; + } + goto cached; + } + /* 1. create socket names */ if (!path[0]) { msg = "Invalid empty name for a UNIX socket"; @@ -227,6 +235,8 @@ static int uxst_bind_listener(struct listener *listener, char *errmsg, int errle /* 6. cleanup */ unlink(backname); /* no need to keep this one either */ + socket_cache_add(fd, listener); + cached: /* the socket is now listening */ listener->fd = fd; listener->state = LI_LISTEN; -- 1.7.2.3