Author: pjd Date: Fri Aug 27 14:26:37 2010 New Revision: 211882 URL: http://svn.freebsd.org/changeset/base/211882
Log: Implement keepalive mechanism inside HAST protocol so we can detect secondary node failures quickly for HAST resources that are rarely modified. Remove XXX from a comment now that the guard thread never sleeps infinitely. MFC after: 2 weeks Obtained from: Wheel Systems Sp. z o.o. http://www.wheelsystems.com Modified: head/sbin/hastd/hast.h head/sbin/hastd/primary.c head/sbin/hastd/secondary.c Modified: head/sbin/hastd/hast.h ============================================================================== --- head/sbin/hastd/hast.h Fri Aug 27 14:12:53 2010 (r211881) +++ head/sbin/hastd/hast.h Fri Aug 27 14:26:37 2010 (r211882) @@ -48,7 +48,12 @@ #include "proto.h" -#define HAST_PROTO_VERSION 0 +/* + * Version history: + * 0 - initial version + * 1 - HIO_KEEPALIVE added + */ +#define HAST_PROTO_VERSION 1 #define EHAST_OK 0 #define EHAST_NOENTRY 1 @@ -74,6 +79,7 @@ #define HIO_WRITE 2 #define HIO_DELETE 3 #define HIO_FLUSH 4 +#define HIO_KEEPALIVE 5 #define HAST_TIMEOUT 5 #define HAST_CONFIG "/etc/hast.conf" Modified: head/sbin/hastd/primary.c ============================================================================== --- head/sbin/hastd/primary.c Fri Aug 27 14:12:53 2010 (r211881) +++ head/sbin/hastd/primary.c Fri Aug 27 14:26:37 2010 (r211882) @@ -151,7 +151,11 @@ static pthread_mutex_t metadata_lock; */ #define HAST_NCOMPONENTS 2 /* - * Number of seconds to sleep before next reconnect try. + * Number of seconds to sleep between keepalive packets. + */ +#define KEEPALIVE_SLEEP 10 +/* + * Number of seconds to sleep between reconnect retries. */ #define RECONNECT_SLEEP 5 @@ -886,11 +890,14 @@ remote_close(struct hast_resource *res, sync_stop(); /* - * Wake up guard thread, so it can immediately start reconnect. + * Wake up guard thread (if we are not called from within guard thread), + * so it can immediately start reconnect. */ - mtx_lock(&hio_guard_lock); - cv_signal(&hio_guard_cond); - mtx_unlock(&hio_guard_lock); + if (!mtx_owned(&hio_guard_lock)) { + mtx_lock(&hio_guard_lock); + cv_signal(&hio_guard_cond); + mtx_unlock(&hio_guard_lock); + } } /* @@ -1734,7 +1741,7 @@ sighandler(int sig) assert(!"invalid condition"); } /* - * XXX: Racy, but if we cannot obtain hio_guard_lock here, we don't + * Racy, but if we cannot obtain hio_guard_lock here, we don't * want to risk deadlock. */ unlock = mtx_trylock(&hio_guard_lock); @@ -1851,6 +1858,32 @@ failed: pjdlog_warning("Configuration not reloaded."); } +static void +keepalive_send(struct hast_resource *res, unsigned int ncomp) +{ + struct nv *nv; + + nv = nv_alloc(); + nv_add_uint8(nv, HIO_KEEPALIVE, "cmd"); + if (nv_error(nv) != 0) { + nv_free(nv); + pjdlog_debug(1, + "keepalive_send: Unable to prepare header to send."); + return; + } + if (hast_proto_send(res, res->hr_remoteout, nv, NULL, 0) < 0) { + pjdlog_common(LOG_DEBUG, 1, errno, + "keepalive_send: Unable to send request"); + nv_free(nv); + rw_unlock(&hio_remote_lock[ncomp]); + remote_close(res, ncomp); + rw_rlock(&hio_remote_lock[ncomp]); + return; + } + nv_free(nv); + pjdlog_debug(2, "keepalive_send: Request sent."); +} + /* * Thread guards remote connections and reconnects when needed, handles * signals, etc. @@ -1874,14 +1907,8 @@ guard_thread(void *arg) sighup_received = false; config_reload(); } - /* - * If all the connection will be fine, we will sleep until - * someone wakes us up. - * If any of the connections will be broken and we won't be - * able to connect, we will sleep only for RECONNECT_SLEEP - * seconds so we can retry soon. - */ - timeout = 0; + + timeout = KEEPALIVE_SLEEP; pjdlog_debug(2, "remote_guard: Checking connections."); mtx_lock(&hio_guard_lock); for (ii = 0; ii < ncomps; ii++) { @@ -1891,6 +1918,11 @@ guard_thread(void *arg) if (ISCONNECTED(res, ii)) { assert(res->hr_remotein != NULL); assert(res->hr_remoteout != NULL); + keepalive_send(res, ii); + } + if (ISCONNECTED(res, ii)) { + assert(res->hr_remotein != NULL); + assert(res->hr_remoteout != NULL); rw_unlock(&hio_remote_lock[ii]); pjdlog_debug(2, "remote_guard: Connection to %s is ok.", Modified: head/sbin/hastd/secondary.c ============================================================================== --- head/sbin/hastd/secondary.c Fri Aug 27 14:12:53 2010 (r211881) +++ head/sbin/hastd/secondary.c Fri Aug 27 14:26:37 2010 (r211882) @@ -413,6 +413,9 @@ reqlog(int loglevel, int debuglevel, int "WRITE(%ju, %ju).", (uintmax_t)hio->hio_offset, (uintmax_t)hio->hio_length); break; + case HIO_KEEPALIVE: + (void)snprintf(msg + len, sizeof(msg) - len, "KEEPALIVE."); + break; default: (void)snprintf(msg + len, sizeof(msg) - len, "UNKNOWN(%u).", (unsigned int)hio->hio_cmd); @@ -433,6 +436,8 @@ requnpack(struct hast_resource *res, str goto end; } switch (hio->hio_cmd) { + case HIO_KEEPALIVE: + break; case HIO_READ: case HIO_WRITE: case HIO_DELETE: @@ -517,7 +522,14 @@ recv_thread(void *arg) } reqlog(LOG_DEBUG, 2, -1, hio, "recv: (%p) Got request header: ", hio); - if (hio->hio_cmd == HIO_WRITE) { + if (hio->hio_cmd == HIO_KEEPALIVE) { + pjdlog_debug(2, + "recv: (%p) Moving request to the free queue.", + hio); + nv_free(hio->hio_nv); + QUEUE_INSERT(free, hio); + continue; + } else if (hio->hio_cmd == HIO_WRITE) { if (hast_proto_recv_data(res, res->hr_remotein, hio->hio_nv, hio->hio_data, MAXPHYS) < 0) { pjdlog_exit(EX_TEMPFAIL, _______________________________________________ svn-src-all@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
