Hi, Here is an update of 2 previous patches that got old and don't apply anymore. A third patch managed to sneak between them.
0001: should be trivial to review 0002: should be self-explanatory 0003: is hackish and most likely needs more details Best Regards, Dridi
From 49ead190a70194d6cd95ebb83a1507823619a947 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Tue, 16 Jun 2015 14:35:15 +0200 Subject: [PATCH 1/3] VMOD read-only access to VCL probe definitions Make sure probes can be managed outside of the cli thread, now that backends can be created any time. --- bin/varnishd/cache/cache_backend_probe.c | 3 --- doc/sphinx/reference/directors.rst | 3 ++- doc/sphinx/reference/vmod.rst | 5 ++++- include/vrt.h | 29 +++++++++++++++-------------- lib/libvcc/vcc_backend.c | 13 ++++++++++--- lib/libvcc/vcc_compile.h | 1 + lib/libvcc/vcc_expr.c | 20 ++++++++++++++++++++ lib/libvcc/vmodtool.py | 1 + 8 files changed, 53 insertions(+), 22 deletions(-) diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 8060bb7..e0590d8 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -514,7 +514,6 @@ VBP_Control(const struct backend *be, int enable) { struct vbp_target *vt; - ASSERT_CLI(); CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); vt = be->probe; CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); @@ -545,7 +544,6 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp, { struct vbp_target *vt; - ASSERT_CLI(); CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC); CHECK_OBJ_NOTNULL(vp, VRT_BACKEND_PROBE_MAGIC); @@ -570,7 +568,6 @@ VBP_Remove(struct backend *be) { struct vbp_target *vt; - ASSERT_CLI(); CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); vt = be->probe; CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); diff --git a/doc/sphinx/reference/directors.rst b/doc/sphinx/reference/directors.rst index cbd8a42..8a16c87 100644 --- a/doc/sphinx/reference/directors.rst +++ b/doc/sphinx/reference/directors.rst @@ -150,4 +150,5 @@ too is no longer needed. It is then Varnish that will take care of health probing and disabling the feature on cold VCL (see :ref:`ref-vmod-event-functions`). -.. TODO document VCL_PROBE if patchwork #310 is merged +Instead of initializing your own probe definition, you can get a ``VCL_PROBE`` +directly built from VCL (see :ref:`ref-vmod-vcl-c-types`). diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 57db2a8..0446e83 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -193,7 +193,10 @@ PRIV_TOP PRIV_VCL See :ref:`ref-vmod-private-pointers` below. -.. TODO document PROBE if patchwork #310 is merged +PROBE + C-type: ``const struct vrt_backend_probe *`` + + A named standalone backend probe definition. REAL C-type: ``double`` diff --git a/include/vrt.h b/include/vrt.h index 98168a3..4b429dd 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -67,20 +67,21 @@ struct ws; * (alphabetic order) */ -typedef const struct director * VCL_BACKEND; -typedef const struct vmod_priv * VCL_BLOB; -typedef unsigned VCL_BOOL; -typedef double VCL_BYTES; -typedef double VCL_DURATION; -typedef const char * VCL_ENUM; -typedef const struct gethdr_s * VCL_HEADER; -typedef struct http * VCL_HTTP; -typedef long VCL_INT; -typedef const struct suckaddr * VCL_IP; -typedef double VCL_REAL; -typedef const char * VCL_STRING; -typedef double VCL_TIME; -typedef void VCL_VOID; +typedef const struct director * VCL_BACKEND; +typedef const struct vmod_priv * VCL_BLOB; +typedef unsigned VCL_BOOL; +typedef double VCL_BYTES; +typedef double VCL_DURATION; +typedef const char * VCL_ENUM; +typedef const struct gethdr_s * VCL_HEADER; +typedef struct http * VCL_HTTP; +typedef long VCL_INT; +typedef const struct suckaddr * VCL_IP; +typedef const struct vrt_backend_probe * VCL_PROBE; +typedef double VCL_REAL; +typedef const char * VCL_STRING; +typedef double VCL_TIME; +typedef void VCL_VOID; /*********************************************************************** * This is the composite argument we pass to compiled VCL and VRT diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index c0501a3..14aedf3 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -249,7 +249,7 @@ void vcc_ParseProbe(struct vcc *tl) { struct token *t_probe; - int i; + struct symbol *sym; char *p; vcc_NextToken(tl); /* ID: probe */ @@ -258,11 +258,18 @@ vcc_ParseProbe(struct vcc *tl) ERRCHK(tl); t_probe = tl->t; vcc_NextToken(tl); - i = vcc_AddDef(tl, t_probe, SYM_PROBE); - if (i > 1) { + + sym = VCC_GetSymbolTok(tl, t_probe, SYM_PROBE); + AN(sym); + if (sym->ndef > 0) { VSB_printf(tl->sb, "Probe %.*s redefined\n", PF(t_probe)); vcc_ErrWhere(tl, t_probe); + return; } + sym->fmt = PROBE; + sym->eval = vcc_Eval_Probe; + sym->ndef++; + ERRCHK(tl); vcc_ParseProbeSpec(tl, t_probe, &p); if (vcc_IdIs(t_probe, "default")) { diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 86f731d..893369e 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -280,6 +280,7 @@ sym_expr_t vcc_Eval_SymFunc; void vcc_Eval_Func(struct vcc *tl, const char *cfunc, const char *extra, const char *name, const char *args); sym_expr_t vcc_Eval_Backend; +sym_expr_t vcc_Eval_Probe; /* vcc_obj.c */ extern const struct var vcc_vars[]; diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index b9c6531..294d12a 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -516,6 +516,23 @@ vcc_Eval_Backend(struct vcc *tl, struct expr **e, const struct symbol *sym) /*-------------------------------------------------------------------- */ + +void +vcc_Eval_Probe(struct vcc *tl, struct expr **e, const struct symbol *sym) +{ + + assert(sym->kind == SYM_PROBE); + + vcc_ExpectCid(tl); + vcc_AddRef(tl, tl->t, SYM_PROBE); + *e = vcc_mk_expr(PROBE, "&vgc_probe_%.*s", PF(tl->t)); + (*e)->constant = EXPR_VAR; /* XXX ? */ + vcc_NextToken(tl); +} + +/*-------------------------------------------------------------------- + */ + void vcc_Eval_Var(struct vcc *tl, struct expr **e, const struct symbol *sym) { @@ -812,6 +829,8 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt) sym = NULL; if (fmt == BACKEND) sym = VCC_FindSymbol(tl, tl->t, SYM_BACKEND); + if (fmt == PROBE) + sym = VCC_FindSymbol(tl, tl->t, SYM_PROBE); if (sym == NULL) sym = VCC_FindSymbol(tl, tl->t, SYM_VAR); if (sym == NULL) @@ -831,6 +850,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt) case SYM_VAR: case SYM_FUNC: case SYM_BACKEND: + case SYM_PROBE: AN(sym->eval); AZ(*e); sym->eval(tl, e, sym); diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 01de3a7..5a6c0fc 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -60,6 +60,7 @@ ctypes = { 'PRIV_VCL': "struct vmod_priv *", 'PRIV_TASK': "struct vmod_priv *", 'PRIV_TOP': "struct vmod_priv *", + 'PROBE': "VCL_PROBE", 'REAL': "VCL_REAL", 'STRING': "VCL_STRING", 'STRING_LIST': "const char *, ...", -- 2.1.0
From 31221dfb77c5a45ddfad98524d7d61850e31da8e Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Mon, 17 Aug 2015 15:50:03 +0200 Subject: [PATCH 2/3] Allow dynamic backend creation in cold VCLs Dynamic backends can be updated on a cold VCL without creating unwanted overhead, such as health probing or statistics. It allows VMODs to keep an up-to-date list of backends, ready to become usable as soon as the VCL becomes warm again. It makes also things easier for dynamic backends tied to a VCL object, they can benefit from the object's lifecycle (it can't exceed the VCL's) and don't need to subscribe to the VMOD's event notifications. --- bin/varnishd/cache/cache_vcl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 2b3c5d2..188fab6 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -205,8 +205,7 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) if (vcl->temp == vcl_temp_warm) { /* Only when adding backend to already warm VCL */ VBE_Event(be, VCL_EVENT_WARM); - } else if (vcl->temp != vcl_temp_init) - WRONG("Dynamic Backends can only be added to warm VCLs"); + } } void -- 2.1.0
From 2d6bdbd93452d3d2dbdd0435bd0601afc11e3b3e Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Thu, 18 Jun 2015 19:00:09 +0200 Subject: [PATCH 3/3] Allow ACL references in VMODs Only log ACL results when the check happens during a transaction. VMOD writers can still build a context if they want to log ACL checks outside of a transaction. The VCL-to-C type correspondance currently relies on fragile code parsing and generation. ACLs are exposed as functions, and have a 2-step declaration in `vrt.h`. It could also be implemented with a miniobj and a new VRT function instead, but it requires heavier changes in libvcc. --- bin/varnishd/cache/cache_vrt.c | 6 ++++-- bin/varnishtest/tests/m00023.vtc | 27 +++++++++++++++++++++++++++ doc/sphinx/reference/vmod.rst | 6 +++++- include/vrt.h | 9 +++++++-- lib/libvcc/vcc_acl.c | 17 ++++++++++++----- lib/libvcc/vcc_compile.h | 1 + lib/libvcc/vcc_expr.c | 21 +++++++++++++++++++++ lib/libvcc/vmodtool.py | 1 + lib/libvmod_debug/vmod.vcc | 4 ++++ lib/libvmod_debug/vmod_debug.c | 10 ++++++++++ 10 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 bin/varnishtest/tests/m00023.vtc diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 814d4af..200b4f8 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -66,8 +66,10 @@ void VRT_acl_log(VRT_CTX, const char *msg) { - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - VSLb(ctx->vsl, SLT_VCL_acl, "%s", msg); + CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC); + AN(msg); + if (ctx != NULL && ctx->vsl != NULL) + VSLb(ctx->vsl, SLT_VCL_acl, "%s", msg); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/tests/m00023.vtc b/bin/varnishtest/tests/m00023.vtc new file mode 100644 index 0000000..70ea375 --- /dev/null +++ b/bin/varnishtest/tests/m00023.vtc @@ -0,0 +1,27 @@ +varnishtest "Test VMOD ACLs" + +varnish v1 -vcl { + import ${vmod_debug}; + + backend dummy { + .host = "${bad_ip}"; + } + + acl loopback { + "127"/24; + } + + sub vcl_init { + debug.match_acl(loopback, "127.0.0.127"); + } + + sub vcl_recv { + debug.match_acl(loopback, client.ip); + return (synth(200)); + } +} -start + +client c1 { + txreq + rxresp +} diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 0446e83..aed1e7a 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -115,7 +115,11 @@ language representation. Here is a description of them. All but the PRIV and STRING_LIST types have typedefs: VCL_INT, VCL_REAL, etc. -.. TODO document ACL if patchwork #314 is merged +ACL + C-type: ``int(acl_f)(VRT_CTX, VCL_IP)*`` + + A function that checks an IP address against the named ACL declared in + VCL. BACKEND C-type: ``const struct director *`` diff --git a/include/vrt.h b/include/vrt.h index 4b429dd..991ac92 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -54,13 +54,17 @@ struct busyobj; struct director; struct http; struct req; +struct sess; struct suckaddr; struct vcl; +struct vrt_ctx; struct vmod; struct vsb; struct vsl_log; struct ws; +#define VRT_CTX const struct vrt_ctx *ctx + /*********************************************************************** * This is the central definition of the mapping from VCL types to * C-types. The python scripts read these from here. @@ -83,6 +87,9 @@ typedef const char * VCL_STRING; typedef double VCL_TIME; typedef void VCL_VOID; +typedef int (acl_f) (VRT_CTX, VCL_IP /* hack */ ); +typedef acl_f * VCL_ACL; + /*********************************************************************** * This is the composite argument we pass to compiled VCL and VRT * functions. @@ -119,8 +126,6 @@ struct vrt_ctx { void *specific; }; -#define VRT_CTX const struct vrt_ctx *ctx - /***********************************************************************/ struct vmod_data { diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 72dd304..15264be 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -352,7 +352,7 @@ vcc_acl_emit(struct vcc *tl, const char *acln, int anon) struct token *t; struct inifin *ifp; - Fh(tl, 0, "\nstatic int\n"); + Fh(tl, 0, "\nstatic int __match_proto__(acl_f)\n"); Fh(tl, 0, "match_acl_%s_%s(VRT_CTX, const VCL_IP p)\n", anon ? "anon" : "named", acln); @@ -470,7 +470,7 @@ void vcc_ParseAcl(struct vcc *tl) { struct token *an; - int i; + struct symbol *sym; char acln[1024]; vcc_NextToken(tl); @@ -486,13 +486,20 @@ vcc_ParseAcl(struct vcc *tl) an = tl->t; vcc_NextToken(tl); - i = vcc_AddDef(tl, an, SYM_ACL); - if (i > 1) { + bprintf(acln, "%.*s", PF(an)); + + sym = VCC_GetSymbolTok(tl, an, SYM_ACL); + AN(sym); + if (sym->ndef > 0) { VSB_printf(tl->sb, "ACL %.*s redefined\n", PF(an)); vcc_ErrWhere(tl, an); return; } - bprintf(acln, "%.*s", PF(an)); + sym->fmt = ACL; + sym->eval = vcc_Eval_Acl; + sym->eval_priv = TlDup(tl, acln); + sym->ndef++; + ERRCHK(tl); SkipToken(tl, '{'); diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 893369e..464ad13 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -279,6 +279,7 @@ sym_expr_t vcc_Eval_Var; sym_expr_t vcc_Eval_SymFunc; void vcc_Eval_Func(struct vcc *tl, const char *cfunc, const char *extra, const char *name, const char *args); +sym_expr_t vcc_Eval_Acl; sym_expr_t vcc_Eval_Backend; sym_expr_t vcc_Eval_Probe; diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 294d12a..2e19cc6 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -502,10 +502,28 @@ vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, const struct symbol *sym) */ void +vcc_Eval_Acl(struct vcc *tl, struct expr **e, const struct symbol *sym) +{ + + assert(sym->kind == SYM_ACL); + AN(sym->eval_priv); + + vcc_ExpectCid(tl); + vcc_AddRef(tl, tl->t, SYM_ACL); + *e = vcc_mk_expr(ACL, "&match_acl_named_%s", + (const char *)sym->eval_priv); + (*e)->constant = EXPR_VAR; /* XXX ? */ + vcc_NextToken(tl); +} +/*-------------------------------------------------------------------- + */ + +void vcc_Eval_Backend(struct vcc *tl, struct expr **e, const struct symbol *sym) { assert(sym->kind == SYM_BACKEND); + AN(sym->eval_priv); vcc_ExpectCid(tl); vcc_AddRef(tl, tl->t, SYM_BACKEND); @@ -827,6 +845,8 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt) * XXX: look for SYM_VAR first for consistency ? */ sym = NULL; + if (fmt == ACL) + sym = VCC_FindSymbol(tl, tl->t, SYM_ACL); if (fmt == BACKEND) sym = VCC_FindSymbol(tl, tl->t, SYM_BACKEND); if (fmt == PROBE) @@ -849,6 +869,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt) switch(sym->kind) { case SYM_VAR: case SYM_FUNC: + case SYM_ACL: case SYM_BACKEND: case SYM_PROBE: AN(sym->eval); diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 5a6c0fc..b4f6da1 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -46,6 +46,7 @@ from os.path import dirname, realpath, exists from pprint import pprint, pformat ctypes = { + 'ACL': "VCL_ACL", 'BACKEND': "VCL_BACKEND", 'BLOB': "VCL_BLOB", 'BOOL': "VCL_BOOL", diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 8b32e92..6ad6260 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -122,3 +122,7 @@ Return the dynamic backend. $Method VOID .refresh(STRING addr, STRING port) Dynamically refresh & (always!) replace the backend by a new one. + +$Function VOID match_acl(ACL acl, IP ip) + +Panic if ip doesn't match acl. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 3dddbac..335c52e 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -279,3 +279,13 @@ vmod_sleep(VRT_CTX, VCL_DURATION t) CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC); VTIM_sleep(t); } + +VCL_VOID +vmod_match_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip) { + + CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC); + AN(acl); + AN(ip); + + AN(acl(ctx, ip)); +} -- 2.1.0
_______________________________________________ varnish-dev mailing list [email protected] https://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
