Hi, As requested, I have prepared a test suite for dynamic backends. This patch set also contains API changes and bug fixes (see #1755).
Best Regards, Dridi
From b242ae0716855e346ca5b3ce373f58e97193448f Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Mon, 6 Jul 2015 08:10:26 +0200 Subject: [PATCH 1/5] Change the VRT API for adding/deleting backends A VRT_CTX may not be available in all situations for backends deletion. Adding a backend to a warm VCL needs a proper initialization of the backend, through an event. Events can now be sent outside of the CLI thread for dynamically-created backends. --- bin/varnishd/cache/cache.h | 1 + bin/varnishd/cache/cache_backend_cfg.c | 22 ++++++++-------------- bin/varnishd/cache/cache_vcl.c | 7 +++++++ include/vrt.h | 4 ++-- lib/libvcc/vcc_backend.c | 2 +- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b96d765..9393f30 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1046,6 +1046,7 @@ const struct vrt_backend_probe *VCL_DefaultProbe(const struct vcl *); void VCL_Init(void); const char *VCL_Method_Name(unsigned); const char *VCL_Name(const struct vcl *); +unsigned VCL_IsWarm(const struct vcl *); void VCL_Panic(struct vsb *, const struct vcl *); void VCL_Poll(void); void VCL_Ref(struct vcl *); diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index 057dea4..a4f0c83 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -55,21 +55,18 @@ static struct lock backends_mtx; */ struct director * -VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt) +VRT_new_backend(struct vcl *vcl, const struct vrt_backend *vrt) { struct backend *b; char buf[128]; - struct vcl *vcl; struct tcp_pool *tp = NULL; const struct vrt_backend_probe *vbp; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(vcl); CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC); assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL); - vcl = ctx->vcl; - AN(vcl); AN(vrt->vcl_name); assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL); @@ -112,24 +109,22 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt) if (vbp != NULL) VBP_Insert(b, vbp, tp); - VCL_AddBackend(ctx->vcl, b); + VCL_AddBackend(vcl, b); + if (VCL_IsWarm(vcl)) + VBE_Event(b, VCL_EVENT_WARM); return (b->director); } void -VRT_delete_backend(VRT_CTX, struct director **dp) +VRT_delete_backend(struct vcl *vcl, struct director *d) { - struct director *d; struct backend *be; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(dp); - d = *dp; - *dp = NULL; + AN(vcl); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC); - VCL_DelBackend(ctx->vcl, be); + VCL_DelBackend(vcl, be); } /*--------------------------------------------------------------------- @@ -140,7 +135,6 @@ void VBE_Event(struct backend *be, enum vcl_event_e ev) { - ASSERT_CLI(); CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); if (ev == VCL_EVENT_WARM) { diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 6df2a2f..6352aa1 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -335,6 +335,13 @@ VCL_DefaultProbe(const struct vcl *vcl) return (vcl->conf->default_probe); } +unsigned +VCL_IsWarm(const struct vcl *vcl) +{ + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + return (vcl->temp == vcl_temp_warm); +} + /*--------------------------------------------------------------------*/ void diff --git a/include/vrt.h b/include/vrt.h index 98168a3..931fa5c 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -273,8 +273,8 @@ void VRT_Rollback(VRT_CTX, const struct http *); void VRT_synth_page(VRT_CTX, const char *, ...); /* Backend related */ -struct director *VRT_new_backend(VRT_CTX, const struct vrt_backend *); -void VRT_delete_backend(VRT_CTX, struct director **); +struct director *VRT_new_backend(struct vcl *, const struct vrt_backend *); +void VRT_delete_backend(struct vcl *, struct director *); /* Suckaddr related */ int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst); diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index c0501a3..e1ffd79 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -427,7 +427,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) ifp = New_IniFin(tl); VSB_printf(ifp->ini, - "\t%s =\n\t VRT_new_backend(ctx, &vgc_dir_priv_%s);", + "\t%s =\n\t VRT_new_backend(ctx->vcl, &vgc_dir_priv_%s);", vgcname, vgcname); } -- 2.1.0
From 9eb8df6ab1bb9a012035180724bcd9f569f12012 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Mon, 6 Jul 2015 08:19:43 +0200 Subject: [PATCH 2/5] Keep track of backends references and connections This is important for: - max connections checks - dynamic backends deletion It introduces a new backend_state enum: - bs_active: for static and active dynamic backends - bs_inactive: for dynamic backends marked as deleted - bs_gone: set when an inactive backend has no references A conditional deletion of the backend is performed outside the critical section when the refcount is decremented. VMOD writers don't need to keep track of deleted backends and actually delete them when their refcount reaches zero. Fixes #1755 --- bin/varnishd/cache/cache_backend.c | 49 +++++++++++++++++++++++++++++++--- bin/varnishd/cache/cache_backend.h | 10 ++++++- bin/varnishd/cache/cache_backend_cfg.c | 29 +++++++++++++++++++- 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index bd8a8c5..bcc4c6e 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -76,6 +76,21 @@ VBE_Healthy(const struct backend *backend, double *changed) } /*-------------------------------------------------------------------- + * Decrement refcount, must be used in a critical section + */ + +static void +vbe_unref(struct backend *bp) +{ + + bp->refcount--; + if (bp->backend_state == bs_inactive && bp->refcount == 0) { + AZ(bp->n_conn); + bp->backend_state = bs_gone; + } +} + +/*-------------------------------------------------------------------- * Get a connection to the backend */ @@ -92,12 +107,18 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo) CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC); AN(bp->vsc); + Lck_Lock(&bp->mtx); + assert(bp->backend_state == bs_active); + bp->refcount++; + Lck_Unlock(&bp->mtx); + if (!VBE_Healthy(bp, NULL)) { // XXX: per backend stats ? VSC_C_main->backend_unhealthy++; return (NULL); } + assert(bp->n_conn >= 0); if (bp->max_connections > 0 && bp->n_conn >= bp->max_connections) { // XXX: per backend stats ? VSC_C_main->backend_busy++; @@ -124,7 +145,6 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo) AN(vc->addr); Lck_Lock(&bp->mtx); - bp->refcount++; bp->n_conn++; bp->vsc->conn++; Lck_Unlock(&bp->mtx); @@ -189,7 +209,12 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, #define ACCT(foo) bp->vsc->foo += bo->acct.foo; #include "tbl/acct_fields_bereq.h" #undef ACCT + + bp->n_conn--; + vbe_unref(bp); Lck_Unlock(&bp->mtx); + + VBE_CondDelete(bp, bo->vcl); bo->htc = NULL; } @@ -218,7 +243,13 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, vbc = vbe_dir_getfd(wrk, bp, bo); if (vbc == NULL) { VSLb(bo->vsl, SLT_FetchError, "no backend connection"); - return (-1); + + Lck_Lock(&bp->mtx); + vbe_unref(bp); + Lck_Unlock(&bp->mtx); + + VBE_CondDelete(bp, bo->vcl); + break; } AN(bo->htc); if (vbc->state != VBC_STATE_STOLEN) @@ -238,6 +269,11 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, return (0); } + Lck_Lock(&bp->mtx); + if (bp->backend_state != bs_active) + extrachance = 0; + Lck_Unlock(&bp->mtx); + /* * If we recycled a backend connection, there is a finite chance * that the backend closed it before we got the bereq to it. @@ -314,6 +350,13 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) if (vbc == NULL) { VSLb(bo->vsl, SLT_FetchError, "no backend connection"); SES_Close(req->sp, SC_RX_TIMEOUT); + V1P_Charge(req, &v1a, bp->vsc); + + Lck_Lock(&bp->mtx); + vbe_unref(bp); + Lck_Unlock(&bp->mtx); + + VBE_CondDelete(bp, bo->vcl); } else { i = V1F_SendReq(req->wrk, bo, &v1a.bereq, 1); VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk)); @@ -324,9 +367,9 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) VSLb_ts_req(req, "PipeSess", W_TIM_real(req->wrk)); SES_Close(req->sp, SC_TX_PIPE); bo->htc->doclose = SC_TX_PIPE; + V1P_Charge(req, &v1a, bp->vsc); vbe_dir_finish(d, req->wrk, bo); } - V1P_Charge(req, &v1a, bp->vsc); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index cc27de6..50d789a 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -53,6 +53,12 @@ enum admin_health { ah_probe }; +enum backend_state { + bs_active = 0, + bs_inactive, + bs_gone +}; + struct backend { unsigned magic; #define BACKEND_MAGIC 0x64c4c7c6 @@ -60,6 +66,7 @@ struct backend { VTAILQ_ENTRY(backend) list; VTAILQ_ENTRY(backend) vcl_list; int refcount; + enum backend_state backend_state; struct lock mtx; VRT_BACKEND_FIELDS() @@ -114,7 +121,8 @@ unsigned VBE_Healthy(const struct backend *b, double *changed); #ifdef VCL_MET_MAX void VBE_Event(struct backend *, enum vcl_event_e); #endif -void VBE_Delete(struct backend *be); +void VBE_Delete(struct backend *); +void VBE_CondDelete(struct backend *, struct vcl *); /* cache_backend_poll.c */ void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index a4f0c83..5b3abc3 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -124,7 +124,13 @@ VRT_delete_backend(struct vcl *vcl, struct director *d) AN(vcl); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC); - VCL_DelBackend(vcl, be); + + Lck_Lock(&be->mtx); + be->backend_state = be->refcount == 0 ? bs_gone : bs_inactive; + Lck_Unlock(&be->mtx); + + if (be->backend_state == bs_gone) + VCL_DelBackend(vcl, be); } /*--------------------------------------------------------------------- @@ -160,9 +166,21 @@ VBE_Delete(struct backend *be) { CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); + if (be->vsc) { + /* deleted dynamic backend is not referenced anymore */ + assert(be->backend_state == bs_gone); + VBE_Event(be, VCL_EVENT_COLD); + } + + /* deleted dynamic backend still referenced? */ + assert(be->backend_state != bs_inactive); + if (be->probe != NULL) VBP_Remove(be); + AZ(be->refcount); + AZ(be->n_conn); + Lck_Lock(&backends_mtx); VTAILQ_REMOVE(&backends, be, list); VSC_C_main->n_backend--; @@ -181,6 +199,15 @@ VBE_Delete(struct backend *be) FREE_OBJ(be); } +void +VBE_CondDelete(struct backend *be, struct vcl *vcl) +{ + + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); + if (be->backend_state == bs_gone) + VCL_DelBackend(vcl, be); +} + /*--------------------------------------------------------------------- * String to admin_health */ -- 2.1.0
From 908e6698e38b496c770249867330ae3349d5e7f0 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Mon, 6 Jul 2015 08:24:54 +0200 Subject: [PATCH 3/5] Add a debug.sleep(DURATION) synchronization helper --- lib/libvmod_debug/vmod.vcc | 4 ++++ lib/libvmod_debug/vmod_debug.c | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 67edd62..305deaa 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -105,3 +105,7 @@ Register the vmod to receive expiry callbacks $Function VOID init_fail() Function to fail vcl_init{} + +$Function VOID sleep(DURATION) + +Block the current worker thread. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index f9f3361..1e46960 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -269,3 +269,12 @@ event_function(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e) priv->free = priv_vcl_free; return (0); } + +VCL_VOID __match_proto__(td_debug_sleep) +vmod_sleep(VRT_CTX, VCL_DURATION t) +{ + + CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC); + + sleep((unsigned int)t); +} -- 2.1.0
From c9ef180c819099de036435a9d8161d66965074b2 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Mon, 6 Jul 2015 08:31:06 +0200 Subject: [PATCH 4/5] Add a test suite for dynamic backends --- bin/varnishtest/tests/d00007.vtc | 28 +++++ bin/varnishtest/tests/d00008.vtc | 51 +++++++++ bin/varnishtest/tests/d00009.vtc | 57 ++++++++++ bin/varnishtest/tests/d00010.vtc | 58 ++++++++++ bin/varnishtest/tests/d00011.vtc | 55 ++++++++++ bin/varnishtest/tests/d00012.vtc | 76 ++++++++++++++ bin/varnishtest/tests/d00013.vtc | 52 +++++++++ lib/libvmod_debug/Makefile.am | 5 +- lib/libvmod_debug/vmod.vcc | 12 +++ lib/libvmod_debug/vmod_debug_dyn.c | 209 +++++++++++++++++++++++++++++++++++++ 10 files changed, 601 insertions(+), 2 deletions(-) create mode 100644 bin/varnishtest/tests/d00007.vtc create mode 100644 bin/varnishtest/tests/d00008.vtc create mode 100644 bin/varnishtest/tests/d00009.vtc create mode 100644 bin/varnishtest/tests/d00010.vtc create mode 100644 bin/varnishtest/tests/d00011.vtc create mode 100644 bin/varnishtest/tests/d00012.vtc create mode 100644 bin/varnishtest/tests/d00013.vtc create mode 100644 lib/libvmod_debug/vmod_debug_dyn.c diff --git a/bin/varnishtest/tests/d00007.vtc b/bin/varnishtest/tests/d00007.vtc new file mode 100644 index 0000000..298fbf7 --- /dev/null +++ b/bin/varnishtest/tests/d00007.vtc @@ -0,0 +1,28 @@ +varnishtest "Test dynamic backends" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl { + import ${vmod_debug}; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn("${s1_addr}", "${s1_port}"); + } + + sub vcl_recv { + set req.backend_hint = s1.backend(); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run diff --git a/bin/varnishtest/tests/d00008.vtc b/bin/varnishtest/tests/d00008.vtc new file mode 100644 index 0000000..ca5432f --- /dev/null +++ b/bin/varnishtest/tests/d00008.vtc @@ -0,0 +1,51 @@ +varnishtest "Test dynamic backends hot swap" + +server s1 { + rxreq + expect req.url == "/foo" + txresp +} -start + +server s2 { + rxreq + expect req.url == "/bar" + txresp +} -start + +varnish v1 -vcl { + import ${vmod_debug}; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn("${s1_addr}", "${s1_port}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.hot_swap(req.http.X-Addr ,req.http.X-Port); + return (synth(200)); + } + set req.backend_hint = s1.backend(); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + + txreq -req "SWAP" -hdr "X-Addr: ${s2_addr}" -hdr "X-Port: ${s2_port}" + rxresp + expect resp.status == 200 + + txreq -url "/bar" + rxresp + expect resp.status == 200 +} -run + +delay 1 + +varnish v1 -expect MAIN.n_backend == 2 diff --git a/bin/varnishtest/tests/d00009.vtc b/bin/varnishtest/tests/d00009.vtc new file mode 100644 index 0000000..394341d --- /dev/null +++ b/bin/varnishtest/tests/d00009.vtc @@ -0,0 +1,57 @@ +varnishtest "Test dynamic backends hot swap while being used" + +server s1 { + rxreq + expect req.url == "/foo" + sema r1 sync 2 + sema r2 sync 2 + txresp +} -start + +server s2 { + rxreq + expect req.url == "/bar" + sema r2 sync 2 + txresp +} -start + +varnish v1 -vcl { + import ${vmod_debug}; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn("${s1_addr}", "${s1_port}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.hot_swap(req.http.X-Addr ,req.http.X-Port); + return (synth(200)); + } + set req.backend_hint = s1.backend(); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 +} -start + +client c2 { + sema r1 sync 2 + txreq -req "SWAP" -hdr "X-Addr: ${s2_addr}" -hdr "X-Port: ${s2_port}" + rxresp + expect resp.status == 200 + + txreq -url "/bar" + rxresp + expect resp.status == 200 +} -run + +client c1 -wait + +varnish v1 -expect MAIN.n_backend == 2 diff --git a/bin/varnishtest/tests/d00010.vtc b/bin/varnishtest/tests/d00010.vtc new file mode 100644 index 0000000..5e7f954 --- /dev/null +++ b/bin/varnishtest/tests/d00010.vtc @@ -0,0 +1,58 @@ +varnishtest "Test dynamic backends hot swap during a pipe" + +server s1 { + rxreq + expect req.url == "/foo" + sema r1 sync 2 + sema r2 sync 2 + txresp +} -start + +server s2 { + rxreq + expect req.url == "/bar" + sema r2 sync 2 + txresp +} -start + +varnish v1 -vcl { + import ${vmod_debug}; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn("${s1_addr}", "${s1_port}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.hot_swap(req.http.X-Addr ,req.http.X-Port); + return (synth(200)); + } + set req.backend_hint = s1.backend(); + return (pipe); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 +} -start + +client c2 { + sema r1 sync 2 + txreq -req "SWAP" -hdr "X-Addr: ${s2_addr}" -hdr "X-Port: ${s2_port}" + rxresp + expect resp.status == 200 + + txreq -url "/bar" + rxresp + expect resp.status == 200 +} -run + +client c1 -wait + +varnish v1 -expect MAIN.n_backend == 2 diff --git a/bin/varnishtest/tests/d00011.vtc b/bin/varnishtest/tests/d00011.vtc new file mode 100644 index 0000000..dbf1512 --- /dev/null +++ b/bin/varnishtest/tests/d00011.vtc @@ -0,0 +1,55 @@ +varnishtest "Test a dynamic backend hot swap after it was picked by a bereq" + +server s1 { +} -start + +server s2 { + rxreq + txresp +} -start + +varnish v1 -vcl { + import ${vmod_debug}; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn("${s1_addr}", "${s1_port}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.hot_swap(req.http.X-Addr ,req.http.X-Port); + return (synth(200)); + } + } + + sub vcl_backend_fetch { + set bereq.backend = s1.backend(); + # hot swap should happen while we sleep + debug.sleep(2s); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq + sema r2 sync 2 + rxresp + expect resp.status == 200 +} + +client c2 { + sema r2 sync 2 + delay 0.1 + txreq -req "SWAP" -hdr "X-Addr: ${s2_addr}" -hdr "X-Port: ${s2_port}" + rxresp + expect resp.status == 200 +} + +client c1 -start +client c2 -run +client c1 -wait + +varnish v1 -expect MAIN.n_backend == 2 diff --git a/bin/varnishtest/tests/d00012.vtc b/bin/varnishtest/tests/d00012.vtc new file mode 100644 index 0000000..fbdb15f --- /dev/null +++ b/bin/varnishtest/tests/d00012.vtc @@ -0,0 +1,76 @@ +varnishtest "Test a dynamic backend discard during a request" + +# vcl.discard testing inspired by v00006.vtc from commit e1f7207 + +server s1 { + rxreq + expect req.url == "/foo" + sema r1 sync 2 + txresp +} -start + +varnish v1 -arg "-p thread_pools=1" -vcl { + import ${vmod_debug}; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn("${s1_addr}", "${s1_port}"); + } + + sub vcl_recv { + set req.backend_hint = s1.backend(); + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 +} -start + +varnish v1 -expect MAIN.n_backend == 2 +# expected: vcl1.dummy, vcl1.s1 + +server s2 { + rxreq + expect req.url == "/bar" + txresp +} -start + +varnish v1 -vcl { + import ${vmod_debug}; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s2 = debug.dyn("${s2_addr}", "${s2_port}"); + } + + sub vcl_recv { + set req.backend_hint = s2.backend(); + } +} + +varnish v1 -cli "vcl.discard vcl1" +sema r1 sync 2 + +client c1 -wait +delay 2 + +varnish v1 -expect MAIN.n_backend == 4 +# expected: vcl1.dummy, vcl1.s1, vcl2.dummy, vcl2.s2 +varnish v1 -expect n_vcl_avail == 1 +varnish v1 -expect n_vcl_discard == 1 + +client c1 { + txreq -url "/bar" + rxresp + expect resp.status == 200 +} -run + +varnish v1 -cli "vcl.list" +varnish v1 -expect MAIN.n_backend == 2 +# expected: vcl2.dummy, vcl2.s2 +varnish v1 -expect n_vcl_avail == 1 +varnish v1 -expect n_vcl_discard == 0 diff --git a/bin/varnishtest/tests/d00013.vtc b/bin/varnishtest/tests/d00013.vtc new file mode 100644 index 0000000..8459f8e --- /dev/null +++ b/bin/varnishtest/tests/d00013.vtc @@ -0,0 +1,52 @@ +varnishtest "Test a dynamic backend hot swap after it was hinted to a req" + +server s1 { +} -start + +server s2 { + rxreq + txresp +} -start + +varnish v1 -vcl { + import ${vmod_debug}; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn("${s1_addr}", "${s1_port}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.hot_swap(req.http.X-Addr ,req.http.X-Port); + return (synth(200)); + } + set req.backend_hint = s1.backend(); + # hot swap should happen while we sleep + debug.sleep(2s); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq + sema r2 sync 2 + rxresp + expect resp.status == 200 +} + +client c2 { + sema r2 sync 2 + delay 0.1 + txreq -req "SWAP" -hdr "X-Addr: ${s2_addr}" -hdr "X-Port: ${s2_port}" + rxresp + expect resp.status == 200 +} + +client c1 -start +client c2 -run +client c1 -wait + +varnish v1 -expect MAIN.n_backend == 2 diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index d189102..9115713 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -17,14 +17,15 @@ libvmod_debug_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version libvmod_debug_la_SOURCES = \ vmod_debug.c \ - vmod_debug_obj.c + vmod_debug_obj.c \ + vmod_debug_dyn.c nodist_libvmod_debug_la_SOURCES = \ vcc_if.c \ vcc_if.h # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build -vmod_debug.lo vmod_debug_obj.lo: vcc_if.h +vmod_debug.lo vmod_debug_obj.lo vmod_debug_dyn.lo: vcc_if.h vcc_if.c vcc_if.h vmod_debug.rst vmod_debug.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 305deaa..464b223 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -109,3 +109,15 @@ Function to fail vcl_init{} $Function VOID sleep(DURATION) Block the current worker thread. + +$Object dyn(STRING addr, STRING port) + +Dynamically create a single-backend director + +$Method BACKEND .backend() + +Return the dynamic backend. + +$Method VOID .hot_swap(STRING addr, STRING port) + +Dynamically replace the backend by a new one. diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c new file mode 100644 index 0000000..a7d620c --- /dev/null +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -0,0 +1,209 @@ +/*- + * Copyright (c) 2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp <[email protected]> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include <netdb.h> +#include <pthread.h> +#include <stdlib.h> + +#include <sys/socket.h> +#include <sys/types.h> + +#include "vcl.h" +#include "vrt.h" + +#include "cache/cache.h" +#include "cache/cache_director.h" +#include "cache/cache_backend.h" + +#include "vsa.h" +#include "vcc_if.h" + +struct vmod_debug_dyn { + unsigned magic; +#define VMOD_DEBUG_DYN_MAGIC 0x9b77ccbd + pthread_mutex_t mtx; + char *vcl_name; + struct director *dir; + struct director dyn_dir; +}; + +static unsigned __match_proto__(vdi_healthy_f) +dyn_dir_healthy(const struct director *d, const struct busyobj *bo, + double *changed) +{ + struct vmod_debug_dyn *dyn; + unsigned healthy; + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CAST_OBJ_NOTNULL(dyn, d->priv, VMOD_DEBUG_DYN_MAGIC); + + AZ(pthread_mutex_lock(&dyn->mtx)); + + AN(dyn->dir); + healthy = dyn->dir->healthy(dyn->dir, bo, changed); + + AZ(pthread_mutex_unlock(&dyn->mtx)); + + return (healthy); +} + +static const struct director * __match_proto__(vdi_resolve_f) +dyn_dir_resolve(const struct director *d, struct worker *wrk, + struct busyobj *bo) +{ + struct vmod_debug_dyn *dyn; + struct director *simple; + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CAST_OBJ_NOTNULL(dyn, d->priv, VMOD_DEBUG_DYN_MAGIC); + + AZ(pthread_mutex_lock(&dyn->mtx)); + + AN(dyn->dir); + simple = dyn->dir; + + AZ(pthread_mutex_unlock(&dyn->mtx)); + + return (simple); +} + +static void +dyn_dir_init(VRT_CTX, struct vmod_debug_dyn *dyn, + VCL_STRING addr, VCL_STRING port) +{ + struct addrinfo hints, *res = NULL; + struct suckaddr *sa; + struct director *dir; + struct vrt_backend vrt; + + CHECK_OBJ_NOTNULL(dyn, VMOD_DEBUG_DYN_MAGIC); + AN(addr); + AN(port); + + INIT_OBJ(&vrt, VRT_BACKEND_MAGIC); + vrt.ipv4_addr = addr; + vrt.port = port; + vrt.vcl_name = dyn->vcl_name; + vrt.hosthdr = vrt.ipv4_addr; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + AZ(getaddrinfo(vrt.ipv4_addr, vrt.port, &hints, &res)); + AZ(res->ai_next); + + sa = malloc(vsa_suckaddr_len); + AN(sa); + AN(VSA_Build(sa, res->ai_addr, res->ai_addrlen)); + vrt.ipv4_suckaddr = sa; + + freeaddrinfo(res); + + dir = VRT_new_backend(ctx->vcl, &vrt); + AN(dir); + + AZ(pthread_mutex_lock(&dyn->mtx)); + if (dir != NULL && dyn->dir != NULL) { + VRT_delete_backend(ctx->vcl, dyn->dir); + dyn->dir = NULL; + } + + if (dir != NULL) { + AZ(dyn->dir); + dyn->dir = dir; + } + AZ(pthread_mutex_unlock(&dyn->mtx)); + + free(sa); +} + +VCL_VOID +vmod_dyn__init(VRT_CTX, struct vmod_debug_dyn **dynp, + const char *vcl_name, VCL_STRING addr, VCL_STRING port) +{ + struct vmod_debug_dyn *dyn; + + ASSERT_CLI(); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(dynp); + AZ(*dynp); + AN(vcl_name); + + ALLOC_OBJ(dyn, VMOD_DEBUG_DYN_MAGIC); + AN(dyn); + REPLACE(dyn->vcl_name, vcl_name); + + INIT_OBJ(&dyn->dyn_dir, DIRECTOR_MAGIC); + dyn->dyn_dir.name = "debug.dyn"; + dyn->dyn_dir.vcl_name = dyn->vcl_name; + dyn->dyn_dir.healthy = &dyn_dir_healthy; + dyn->dyn_dir.resolve = &dyn_dir_resolve; + dyn->dyn_dir.priv = dyn; + + AZ(pthread_mutex_init(&dyn->mtx, NULL)); + + dyn_dir_init(ctx, dyn, addr, port); + if (dyn->dir) + *dynp = dyn; +} + +VCL_VOID +vmod_dyn__fini(struct vmod_debug_dyn **dynp) +{ + struct vmod_debug_dyn *dyn; + + AN(dynp); + CAST_OBJ_NOTNULL(dyn, *dynp, VMOD_DEBUG_DYN_MAGIC); + // XXX all backends will be deleted by the vcl + free(dyn->vcl_name); + AZ(pthread_mutex_destroy(&dyn->mtx)); + FREE_OBJ(dyn); + *dynp = NULL; +} + +VCL_BACKEND __match_proto__() +vmod_dyn_backend(VRT_CTX, struct vmod_debug_dyn *dyn) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(dyn, VMOD_DEBUG_DYN_MAGIC); + assert(dyn->dyn_dir.priv == dyn); + AN(dyn->dir); + // XXX what if we hot swap right after returning? + return (&dyn->dyn_dir); +} + +VCL_VOID +vmod_dyn_hot_swap(VRT_CTX, struct vmod_debug_dyn *dyn, + VCL_STRING addr, VCL_STRING port) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(dyn, VMOD_DEBUG_DYN_MAGIC); + dyn_dir_init(ctx, dyn, addr, port); +} -- 2.1.0
From 53f9abacca7ac9f2042c1d5d6c5894de2da461b6 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Mon, 6 Jul 2015 08:42:21 +0200 Subject: [PATCH 5/5] Add more information to a backend panic message --- bin/varnishd/cache/cache_backend.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index bcc4c6e..d9c78f8 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -400,6 +400,17 @@ vbe_panic(const struct director *d, struct vsb *vsb) else VSB_printf(vsb, "*invalid*"); VSB_printf(vsb, ", changed=%.1f\n", bp->health_changed); + VSB_printf(vsb, " refcount = %d\n", bp->refcount); + VSB_printf(vsb, " backend_state = "); + if (bp->backend_state == bs_active) + VSB_printf(vsb, "active"); + else if (bp->backend_state == bs_inactive) + VSB_printf(vsb, "inactive"); + else if (bp->backend_state == bs_gone) + VSB_printf(vsb, "gone"); + else + VSB_printf(vsb, "*invalid*"); + VSB_printf(vsb, "\n"); } /*--------------------------------------------------------------------*/ -- 2.1.0
_______________________________________________ varnish-dev mailing list [email protected] https://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
