BBlack has uploaded a new change for review.
https://gerrit.wikimedia.org/r/57761
Change subject: Initial code upload, WIP
......................................................................
Initial code upload, WIP
Change-Id: I67cbaf13366700762506e272839e2287a8024d94
---
A .gitignore
A COPYING
A LICENSE
A Makefile.am
A README.rst
A acaux/.gitignore
A autogen.sh
A configure.ac
A m4/.gitignore
A src/.gitignore
A src/Makefile.am
A src/tests/test01.vtc
A src/vmod_netmapper.c
A src/vmod_netmapper.vcc
14 files changed, 568 insertions(+), 0 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/operations/software/varnish/libvmod-netmapper
refs/changes/61/57761/1
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..748ee22
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,33 @@
+# Autotools junk at top level
+/autom4te.cache/
+/aclocal.m4
+/config.h
+/config.h.in
+/config.log
+/config.status
+/configure
+/libtool
+/stamp-h1
+
+# Various tool outputs in all dirs
+Makefile
+Makefile.in
+.dirstamp
+.deps
+.libs
+*.o
+*.so
+*.lo
+*.la
+*.gcov
+*.trucov
+*.info
+*.gcda
+*.gcno
+*.tmp
+*.log
+*.trs
+*~
+
+# Generated manpages
+*.[0-9]
diff --git a/COPYING b/COPYING
new file mode 100644
index 0000000..f792696
--- /dev/null
+++ b/COPYING
@@ -0,0 +1,7 @@
+Copyright (c) 2011 Varnish Software AS
+...
+See LICENSE for details.
+
+You're free to use and distribute this under terms in the
+LICENSE. Please add your relevant copyright statements.
+
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..264d421
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,20 @@
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
diff --git a/Makefile.am b/Makefile.am
new file mode 100644
index 0000000..d7dcb2b
--- /dev/null
+++ b/Makefile.am
@@ -0,0 +1,21 @@
+ACLOCAL_AMFLAGS = -I m4
+
+SUBDIRS = src
+
+EXTRA_DIST = README.rst
+
+dist_man_MANS = vmod_netmapper.3
+MAINTAINERCLEANFILES = $(dist_man_MANS)
+
+vmod_netmapper.3: README.rst
+
+%.1 %.2 %.3 %.4 %.5 %.6 %.7 %.8 %.9:
+if HAVE_RST2MAN
+ ${RST2MAN} $< $@
+else
+ @echo "========================================"
+ @echo "You need rst2man installed to make dist"
+ @echo "========================================"
+ @false
+endif
+
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..ed81b0b
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,100 @@
+=================
+libvmod_netmapper
+=================
+
+XXX this project was recently copied from vmod_example. Things like docs
+are not at all up to date with the experimental code!
+
+-------------------------------------------------
+Varnish module to map an IP address to a string
+-------------------------------------------------
+
+:Author: Martin Blix Grydeland
+:Date: 2011-05-26
+:Version: 1.0
+:Manual section: 3
+
+SYNOPSIS
+========
+
+import netmapper;
+
+DESCRIPTION
+===========
+
+Example Varnish vmod demonstrating how to write an out-of-tree Varnish vmod
+for Varnish 3.0 and later.
+
+Implements the traditional Hello World as a vmod.
+
+FUNCTIONS
+=========
+
+hello
+-----
+
+Prototype
+ ::
+
+ hello(STRING S)
+Return value
+ STRING
+Description
+ Returns "Hello, " prepended to S
+Example
+ ::
+
+ set resp.http.hello = netmapper.hello("World");
+
+INSTALLATION
+============
+
+This is an netmapper skeleton for developing out-of-tree Varnish
+vmods available from the 3.0 release. It implements the "Hello, World!"
+as a vmod callback. Not particularly useful in good hello world
+tradition,but demonstrates how to get the glue around a vmod working.
+
+The source tree is based on autotools to configure the building, and
+does also have the necessary bits in place to do functional unit tests
+using the varnishtest tool.
+
+Usage::
+
+ ./configure VARNISHSRC=DIR [VMODDIR=DIR]
+
+`VARNISHSRC` is the directory of the Varnish source tree for which to
+compile your vmod. Both the `VARNISHSRC` and `VARNISHSRC/include`
+will be added to the include search paths for your module.
+
+Optionally you can also set the vmod install directory by adding
+`VMODDIR=DIR` (defaults to the pkg-config discovered directory from your
+Varnish installation).
+
+Make targets:
+
+* make - builds the vmod
+* make install - installs your vmod in `VMODDIR`
+* make check - runs the unit tests in ``src/tests/*.vtc``
+
+In your VCL you could then use this vmod along the following lines::
+
+ import netmapper;
+
+ sub vcl_deliver {
+ # This sets resp.http.hello to "Hello, World"
+ set resp.http.hello = netmapper.hello("World");
+ }
+
+HISTORY
+=======
+
+This manual page was released as part of the libvmod-netmapper package,
+demonstrating how to create an out-of-tree Varnish vmod.
+
+COPYRIGHT
+=========
+
+This document is licensed under the same license as the
+libvmod-netmapper project. See LICENSE for details.
+
+* Copyright (c) 2011 Varnish Software
diff --git a/acaux/.gitignore b/acaux/.gitignore
new file mode 100644
index 0000000..c470f5d
--- /dev/null
+++ b/acaux/.gitignore
@@ -0,0 +1,8 @@
+ar-lib
+config.guess
+config.sub
+depcomp
+install-sh
+ltmain.sh
+missing
+test-driver
diff --git a/autogen.sh b/autogen.sh
new file mode 100755
index 0000000..9a12ef5
--- /dev/null
+++ b/autogen.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+warn() {
+ echo "WARNING: $@" 1>&2
+}
+
+case `uname -s` in
+Darwin)
+ LIBTOOLIZE=glibtoolize
+ ;;
+FreeBSD)
+ LIBTOOLIZE=libtoolize
+ ;;
+Linux)
+ LIBTOOLIZE=libtoolize
+ ;;
+SunOS)
+ LIBTOOLIZE=libtoolize
+ ;;
+*)
+ warn "unrecognized platform:" `uname -s`
+ LIBTOOLIZE=libtoolize
+esac
+
+automake_version=`automake --version | tr ' ' '\n' | egrep
'^[0-9]\.[0-9a-z.-]+'`
+if [ -z "$automake_version" ] ; then
+ warn "unable to determine automake version"
+else
+ case $automake_version in
+ 0.*|1.[0-8]|1.[0-8][.-]*)
+ warn "automake ($automake_version) detected; 1.9 or
newer recommended"
+ ;;
+ *)
+ ;;
+ esac
+fi
+
+set -ex
+
+aclocal -I m4
+$LIBTOOLIZE --copy --force
+autoheader
+automake --add-missing --copy --foreign
+autoconf
diff --git a/configure.ac b/configure.ac
new file mode 100644
index 0000000..192fcea
--- /dev/null
+++ b/configure.ac
@@ -0,0 +1,83 @@
+AC_PREREQ(2.59)
+AC_COPYRIGHT([Copyright (c) 2011 Varnish Software AS])
+AC_INIT([libvmod-netmapper], [trunk])
+AC_CONFIG_MACRO_DIR([m4])
+AC_CONFIG_AUX_DIR([acaux])
+AC_CONFIG_SRCDIR(src/vmod_netmapper.vcc)
+AM_CONFIG_HEADER(config.h)
+
+AC_CANONICAL_SYSTEM
+AC_LANG(C)
+
+AM_INIT_AUTOMAKE([foreign])
+
+AC_GNU_SOURCE
+AC_PROG_CC
+AC_PROG_CC_STDC
+if test "x$ac_cv_prog_cc_c99" = xno; then
+ AC_MSG_ERROR([Could not find a C99 compatible compiler])
+fi
+AC_PROG_CPP
+
+AC_PROG_INSTALL
+AC_PROG_LIBTOOL
+AC_PROG_MAKE_SET
+
+# Check for rst utilities
+AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], "no")
+if test "x$RST2MAN" = "xno"; then
+ AC_MSG_WARN([rst2man not found - not building man pages])
+fi
+AM_CONDITIONAL(HAVE_RST2MAN, [test "x$RST2MAN" != "xno"])
+
+# Check for pkg-config
+PKG_PROG_PKG_CONFIG
+
+# Checks for header files.
+AC_HEADER_STDC
+AC_CHECK_HEADERS([sys/stdlib.h])
+
+# Check for python
+AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6
python2.5 python2 python], [AC_MSG_ERROR([Python is needed to build this vmod,
please install python.])])
+
+# Varnish source tree
+AC_ARG_VAR([VARNISHSRC], [path to Varnish source tree (mandatory)])
+if test "x$VARNISHSRC" = x; then
+ AC_MSG_ERROR([No Varnish source tree specified])
+fi
+VARNISHSRC=`cd $VARNISHSRC && pwd`
+AC_CHECK_FILE([$VARNISHSRC/include/varnishapi.h],
+ [],
+ [AC_MSG_FAILURE(["$VARNISHSRC" is not a Varnish source directory])]
+)
+
+# Check that varnishtest is built in the varnish source directory
+AC_CHECK_FILE([$VARNISHSRC/bin/varnishtest/varnishtest],
+ [],
+ [AC_MSG_FAILURE([Can't find "$VARNISHSRC/bin/varnishtest/varnishtest".
Please build your varnish source directory])]
+)
+
+# vmod installation dir
+AC_ARG_VAR([VMODDIR], [vmod installation directory
@<:@LIBDIR/varnish/vmods@:>@])
+if test "x$VMODDIR" = x; then
+ VMODDIR=`pkg-config --variable=vmoddir varnishapi`
+ if test "x$VMODDIR" = x; then
+ AC_MSG_FAILURE([Can't determine vmod installation directory])
+ fi
+fi
+
+# userspace-rcu for lockless netmap reload
+AC_CHECK_HEADER(urcu-qsbr.h,[
+ AC_CHECK_LIB([urcu-qsbr],[perror],[],AC_MSG_ERROR("liburcu-qsbr
missing!"))
+], AC_MSG_ERROR("urcu-qsbr.h missing!"))
+
+# JSON parser for the input data
+AC_CHECK_HEADER(jansson.h,[
+ AC_CHECK_LIB([jansson],[json_object_update],[],AC_MSG_ERROR("libjansson
missing!"))
+], AC_MSG_ERROR("jansson.h missing!"))
+
+AC_CONFIG_FILES([
+ Makefile
+ src/Makefile
+])
+AC_OUTPUT
diff --git a/m4/.gitignore b/m4/.gitignore
new file mode 100644
index 0000000..38066dd
--- /dev/null
+++ b/m4/.gitignore
@@ -0,0 +1,5 @@
+libtool.m4
+ltoptions.m4
+ltsugar.m4
+ltversion.m4
+lt~obsolete.m4
diff --git a/src/.gitignore b/src/.gitignore
new file mode 100644
index 0000000..64eef33
--- /dev/null
+++ b/src/.gitignore
@@ -0,0 +1 @@
+vcc_if.[ch]
diff --git a/src/Makefile.am b/src/Makefile.am
new file mode 100644
index 0000000..3d59033
--- /dev/null
+++ b/src/Makefile.am
@@ -0,0 +1,28 @@
+INCLUDES = -I$(VARNISHSRC)/include -I$(VARNISHSRC)
+
+vmoddir = $(VMODDIR)
+vmod_LTLIBRARIES = libvmod_netmapper.la
+
+libvmod_netmapper_la_LDFLAGS = -module -export-dynamic -avoid-version -shared
+libvmod_netmapper_la_LIBS = -lurcu-qsbr -ljansson
+libvmod_netmapper_la_SOURCES = \
+ vcc_if.c \
+ vcc_if.h \
+ vmod_netmapper.c
+
+vcc_if.c vcc_if.h: $(VARNISHSRC)/lib/libvmod_std/vmod.py
$(top_srcdir)/src/vmod_netmapper.vcc
+ @PYTHON@ $(VARNISHSRC)/lib/libvmod_std/vmod.py
$(top_srcdir)/src/vmod_netmapper.vcc
+
+VMOD_TESTS = tests/*.vtc
+.PHONY: $(VMOD_TESTS)
+
+tests/*.vtc:
+ $(VARNISHSRC)/bin/varnishtest/varnishtest
-Dvarnishd=$(VARNISHSRC)/bin/varnishd/varnishd
-Dvmod_topbuild=$(abs_top_builddir) $@
+
+check: $(VMOD_TESTS)
+
+EXTRA_DIST = \
+ vmod_netmapper.vcc \
+ $(VMOD_TESTS)
+
+CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h
diff --git a/src/tests/test01.vtc b/src/tests/test01.vtc
new file mode 100644
index 0000000..721e867
--- /dev/null
+++ b/src/tests/test01.vtc
@@ -0,0 +1,22 @@
+varnishtest "Test netmapper vmod"
+
+server s1 {
+ rxreq
+ txresp
+} -start
+
+varnish v1 -vcl+backend {
+ import netmapper from "${vmod_topbuild}/src/.libs/libvmod_netmapper.so";
+
+ sub vcl_deliver {
+ set resp.http.hello = netmapper.hello("World");
+ }
+} -start
+
+client c1 {
+ txreq -url "/"
+ rxresp
+ expect resp.http.hello == "Hello, World"
+}
+
+client c1 -run
diff --git a/src/vmod_netmapper.c b/src/vmod_netmapper.c
new file mode 100644
index 0000000..c778267
--- /dev/null
+++ b/src/vmod_netmapper.c
@@ -0,0 +1,193 @@
+#include "vrt.h"
+#include "bin/varnishd/cache.h"
+#include "vcc_if.h"
+
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+
+#include <jansson.h>
+
+#define _LGPL_SOURCE 1
+#include <urcu-qsbr.h>
+
+// XXX this whole thing needs error retval checking on lib calls, etc...
+
+typedef struct {
+ unsigned len; // includes NUL
+ char* data; // NUL-terminated
+} str_t;
+
+str_t* str_new(const char* input) {
+ assert(input);
+ str_t* rv = malloc(sizeof(str_t));
+ rv->len = strlen(input) + 1;
+ rv->data = malloc(rv->len);
+ memcpy(rv->data, input, rv->len);
+ return rv;
+}
+
+void str_destroy(str_t* str) {
+ free(str->data);
+ free(str);
+}
+
+// Copy a str_t*'s data to a const char* in the session workspace,
+// so that after return we're not holding references to data in
+// the ndb, so that it can be swapped for update between...
+static const char* str_to_vcl(struct sess* sp, const str_t* str) {
+ const char* rv = NULL;
+ unsigned used = 0;
+ const unsigned space = WS_Reserve(sp->ws, 0);
+ if(space < str->len) {
+ // XXX not SLT_LostHeader, what?
+ WSP(sp, SLT_LostHeader, "vmod_netmapper: no space for string retval!");
+ }
+ else {
+ used = str->len;
+ rv = sp->ws->f;
+ memcpy(sp->ws->f, str->data, used);
+ }
+ WS_Release(sp->ws, used);
+ return rv;
+}
+
+/* ndb_t */
+
+static pthread_t ndb_updater;
+
+typedef struct {
+ // XXX TODO: flesh this out
+ str_t* the_only_one;
+} ndb_t;
+
+// global singleton, swapped out on update
+static ndb_t* ndb;
+
+static void ndb_destruct(ndb_t* n) {
+ str_destroy(n->the_only_one);
+ free(n);
+}
+
+static ndb_t* ndb_parse(void) {
+
+ /* XXX TODO: actually parse JSON into a real datastructure...
+ json_error_t errobj;
+ json_t* toplevel = json_load_file("/home/bblack/test.json", 0, &errobj);
+ */
+
+ ndb_t* n = malloc(sizeof(ndb_t));
+ n->the_only_one = str_new("001-01");
+ return n; // XXX or NULL if parse fails
+}
+
+static void* ndb_updater_start(void* x) {
+ while(1) {
+ sleep(1); // XXX more like ~15-60s for the real thing, this
+ // is to help show bugs :)
+ if(1) { // XXX stat-check indicates reload necc...
+ ndb_t* new_db = ndb_parse();
+ if(new_db) {
+ ndb_t* old_db = ndb;
+ rcu_assign_pointer(ndb, new_db);
+ synchronize_rcu();
+ ndb_destruct(old_db);
+ }
+ }
+ }
+
+ return NULL;
+}
+
+static void ndb_fini(void) {
+ // clean up the updater thread
+ pthread_cancel(ndb_updater);
+ pthread_join(ndb_updater, NULL);
+
+ // free the most-recent data
+ ndb_destruct(ndb);
+}
+
+static void ndb_init(void) {
+ // initial database load
+ ndb = ndb_parse();
+ // start the updater thread
+ pthread_create(&ndb_updater, NULL, ndb_updater_start, NULL);
+}
+
+static const str_t* ndb_lookup(const char* addr_str) {
+ // important!
+ ndb_t* cur_db = rcu_dereference(ndb);
+
+ // XXX actually look up a specific string
+ // based on the text address in addr_str
+ return cur_db->the_only_one;
+}
+
+// Crazy hack to get per-thread rcu register/unregister, even though
+// Varnish doesn't give us per-thread hooks for the workers
+// (at least, not that I noticed...)
+static pthread_key_t unreg_hack;
+static pthread_once_t unreg_hack_once = PTHREAD_ONCE_INIT;
+static void destruct_rcu(void* x) { pthread_setspecific(unreg_hack, NULL);
rcu_unregister_thread(); }
+static void make_unreg_hack(void) { pthread_key_create(&unreg_hack,
destruct_rcu); }
+
+static const char* ndb_map(struct sess* sp, const char* ip_string) {
+ assert(ndb); assert(sp); assert(ip_string);
+
+ // The rest of the rcu register/unregister hack
+ static __thread bool rcu_registered = false;
+ if(!rcu_registered) {
+ pthread_once(&unreg_hack_once, make_unreg_hack);
+ pthread_setspecific(unreg_hack, (void*)1);
+ rcu_register_thread();
+ rcu_registered = true;
+ }
+
+ const char* rv = NULL;
+
+ // normal rcu reader stuff
+ rcu_thread_online();
+ rcu_read_lock();
+
+ // search net database. if match, convert
+ // string to a vcl string and return it...
+ const str_t* str = ndb_lookup(ip_string);
+ if(str)
+ rv = str_to_vcl(sp, str);
+
+ // normal rcu reader stuff
+ rcu_read_unlock();
+ rcu_thread_offline();
+
+ return rv;
+}
+
+/*****************************
+ * Actual VMOD/VCL/VRT Hooks *
+ *****************************/
+
+// init-tracking
+static unsigned vcl_count = 0;
+
+static void fini_func(void* x) {
+ if(!--vcl_count)
+ ndb_fini();
+}
+
+int init_function(struct vmod_priv *priv, const struct VCL_conf *conf) {
+ assert(priv);
+
+ priv->free = fini_func;
+ if(!vcl_count++)
+ ndb_init();
+ return 0;
+}
+
+const char* vmod_map(struct sess *sp, const char* ip_string) {
+ assert(sp);
+
+ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); // XXX ???
+ return ndb_map(sp, ip_string);
+}
diff --git a/src/vmod_netmapper.vcc b/src/vmod_netmapper.vcc
new file mode 100644
index 0000000..6c6642c
--- /dev/null
+++ b/src/vmod_netmapper.vcc
@@ -0,0 +1,3 @@
+Module netmapper
+Init init_function
+Function STRING map(STRING)
--
To view, visit https://gerrit.wikimedia.org/r/57761
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I67cbaf13366700762506e272839e2287a8024d94
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/varnish/libvmod-netmapper
Gerrit-Branch: master
Gerrit-Owner: BBlack <bbl...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
