Ottomata has submitted this change and it was merged. Change subject: Initial Debian version ......................................................................
Initial Debian version Change-Id: Ib53267d85e7a638d34b8660557b3c62638abdd54 --- M .gitignore A debian/70-varnishkafka.conf A debian/changelog A debian/compat A debian/control A debian/copyright A debian/gbp.conf A debian/rules A debian/source/format A debian/varnishkafka.conf A debian/varnishkafka.default A debian/varnishkafka.dirs A debian/varnishkafka.init A debian/varnishkafka.install A debian/varnishkafka.logrotate A debian/varnishkafka.postinst M varnishkafka.conf.example 17 files changed, 562 insertions(+), 1 deletion(-) Approvals: Ottomata: Verified; Looks good to me, approved diff --git a/.gitignore b/.gitignore index 7edc56e..94bc924 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ varnishkafka *~ .emacs* +.pc diff --git a/debian/70-varnishkafka.conf b/debian/70-varnishkafka.conf new file mode 100644 index 0000000..acadf42 --- /dev/null +++ b/debian/70-varnishkafka.conf @@ -0,0 +1,2 @@ +# rsyslogd varnishkafka config. +if $programname == 'varnishkafka' then /var/log/varnishkafka.log diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..8969bb0 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +varnishkafka (1.0.0-1) unstable; urgency=low + + * Initial release. + + -- Andrew Otto (WMF) <o...@wikimedia.org> Wed, 04 Dec 2013 19:56:07 +0000 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..b871e76 --- /dev/null +++ b/debian/control @@ -0,0 +1,35 @@ +Source: varnishkafka +Priority: optional +Maintainer: Faidon Liambotis <parav...@debian.org> +Build-Depends: debhelper (>= 9), libvarnishapi-dev, librdkafka-dev, libyajl-dev +Standards-Version: 3.9.4 +Section: web + +Package: varnishkafka +Architecture: any +Depends: varnish, ${shlibs:Depends}, ${misc:Depends} +Enhances: varnish +Description: Varnish to Kafka log streamer + varnishkafka reads Varnish shared memory logs, formats them and prints them to + standard output or streams them to Kafka brokers, using the Apache Kafka + protocol. It can format the logs into Apache/NCSA "combined" log + format, custom string format, or JSON format. + . + varnishkafka is similar to varnishncsa, as shipped by the Varnish source, + although it has more features and was written with high performance in mind. + +Package: varnishkafka-dbg +Architecture: any +Section: debug +Priority: extra +Depends: varnishkafka (= ${binary:Version}), ${misc:Depends} +Description: Varnish to Kafka log streamer + varnishkafka reads Varnish shared memory logs, formats them and prints them to + standard output or streams them to Kafka brokers, using the Apache Kafka + protocol. It can format the logs into Apache/NCSA "combined" log + format, custom string format, or JSON format. + . + varnishkafka is similar to varnishncsa, as shipped by the Varnish source, + although it has more features and was written with high performance in mind. + . + This package provides the debugging symbols for varnishkafka. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..7abe17e --- /dev/null +++ b/debian/copyright @@ -0,0 +1,46 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: varnishkafka +Source: https://github.com/wikimedia/varnishkafka + +License: BSD-2-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + . + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + . + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +Files: * +Copyright: 2013 Wikimedia Foundation, Inc. + 2013 Magnus Edenhill +License: BSD-2-clause + +Files: base64.c +Copyright: not applicable +License: public-domain + The file header indicates that it was placed into the public domain by its + author, Poul-Henning Kamp <p...@phk.freebsd.dk> + +Files: base64.h +Copyright: 2006 Verdens Gang AS + 2006-2011 Varnish Software AS +License: BSD-2-clause + +Files: debian/* +Copyright: 2013 Faidon Liambotis <parav...@debian.org> + 2013 Wikimedia Foundation, Inc. +License: BSD-2-clause diff --git a/debian/gbp.conf b/debian/gbp.conf new file mode 100644 index 0000000..7d895d1 --- /dev/null +++ b/debian/gbp.conf @@ -0,0 +1,8 @@ +[git-buildpackage] +upstream-tree=tag +upstream-branch=master +debian-branch=debian +upstream-tag=%(version)s +debian-tag=debian/%(version)s +tarball-dir = ../tarballs +export-dir = ../build-area diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..bad830f --- /dev/null +++ b/debian/rules @@ -0,0 +1,18 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + +# FIXME +export VER=1 + +%: + dh $@ + +override_dh_auto_install: + # don't run "make install", use debian/*.install instead + +override_dh_strip: + dh_strip --dbg-package=varnishkafka-dbg + +.PHONY: override_dh_auto_install override_dh_strip diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/varnishkafka.conf b/debian/varnishkafka.conf new file mode 100644 index 0000000..ebd2294 --- /dev/null +++ b/debian/varnishkafka.conf @@ -0,0 +1,275 @@ +####################################################################### +# # +# varnishkafka configuration file # +# # +# # +####################################################################### +# # +# Syntax: # +# <property-name> = <value> # +# # +# Boolean property values: # +# >0, "true", "yes", "on" - interpreted as true # +# everything else - interpreted as false # +# # +####################################################################### + # + # + # +####################################################################### +# # +# Varnish log formatting # +# # +# format.type - format output type, one of: # +# string - ASCII string output # +# json - JSON output # +# # +# # +# format - format string # +# %X # +# where 'X' is one of the standard varnishncsa(1) formatters. # +# Example: %u # +# # +# # +# %{VAR}X # +# Name-Value tokens where X is 'x', 'i' or 'o' and 'VAR' is the # +# Name to extract the value for. # +# Example: %{User-Agent}i # +# # +# # +# %{?DEFAULT@FIELD!OPTION!OPTION..}X # +# where 'X' is any formatter, # +# # +# 'DEFAULT' is the default string to use if no tag was matched, # +# the default default string is "-". # +# # +# 'FIELD' is the field name to use with the JSON formatter. # +# i.e., "%{@host}l" will be JSON encoded as: {"host":"1.2.3.4"} # +# # +# 'OPTION' is one or more of the formatting options: # +# escape - escape non-printable characters to \<octalcode> # +# and \t\n\r\v\f " to their canonical # +# backslashed notations (\t\n\r\v\f\"\ ). # +# num - for typed formatters, such as JSON, try to encode # +# the value as a number. # +# # +# # +# This syntax can be combined with %{VAR}X. # +# Example: %{User-Agent?Mozilla!escape}i # +# %{?nouser}u # +# %{!escape}q # +# %{@host}l # +# # +# # +# # +# Additional formatter specials: # +# %{<strftime-format>}t - format timestamp according to supplied # +# strftime(3) compatible format string. # +# # +# # +# # +# Non %-prefixed strings are copied verbatim to the # +# output log string. # +# Example: "User: %u;" would render "User: snaps;" # +# # +# # +####################################################################### + +# Standard string formatter +format.type = string +format = %l %n %t %{Varnish:time_firstbyte}x %h %{Varnish:handling}x/%s %b %m http://%{Host}i%U%q - %{Content-Type}o %{Referer}i %{X-Forwarded-For}i %{User-agent!escape}i %{Accept-Language}i + +# JSON formatter +# JSON field names are specified with %{@FIELDNAME}x +#format.type = json +#format = %{@host}l %{@seq_num!num}n %{@timestamp}t %{Varnish:time_firstbyte@response!num}x %{@ip}h %{Varnish:handling@handling}x %{@http_status}s %{@bytes_sent!num}b %{@request_method}m %{Host@host}i %{@uri}U %{@query}q %{Content-Type@mime_type}o %{Referer@referer}i %{X-Forwarded-For@x_forwarded_for}i %{User-Agent@user_agent}i %{Accept-Language@accept_language}i + + + +# Optional secondary formatting. +# 'output = kafka': The rendered 'format.key' will be provided as the +# Kafka message Key +# 'output = string': Print string to stdout. +# Supports the same formating and type as 'format' and 'format.type'. +# +format.key.type = string +format.key = %{%s}t + + + +# Where to output varnish log lines: +# kafka - (default) send to kafka broker +# stdout - just print to stdout (behave like varnishncsa) +# null - (test) collect all tags specified by format but dont output anything +output = kafka + + +# The maximum accepted log tag size. +# Larger tags will be truncated to this size. +# Defaults to 2048 +#tag.size.max = 2048 + + +# EXPERIMENTAL +# Indicates if the log tag data read from VSL files should be copied instantly +# when read (true). If this is set to false the data is assumed to be +# persistent (for the duration of collecting and formatting a single request) +# and no copies will be made, thus improving performance. +# +# NOTE: +# Must be set to true for offline files (-r file..) due to the way +# libvarnishapi reads its data. +logline.data.copy = true + + +# TUNING +# Logline cache hash tuning +# 'logline.hash.size * logline.hash.max' dictates the maximum number of +# cached logline entries in memory. + +# Number of hash buckets (keyed by log id). +# Higher number yields more performance at the expense of memory. +# Set this to avg_requests_per_second / 5. +# Defaults to 5000 +#logline.hash.size = 5000 + +# Maximum number of loglines per hash bucket +# Higher number yields less memory consumption at the expense of performance. +# Set this to avg_requests_per_second / logline.hash.size. +# Defaults to 5 +#logline.hash.max = 5 + +# Size of per logline scratch buffer. +# The scratch buffer is used as a temporary storage space while +# collecting tags for the log line. +# If the scratch size is too small the logline tag match will be incomplete. +# Defaults to 4096 bytes. +#logline.scratch.size = 4096 + +# Start for sequence number (%n) +# Either a number, or the string "time" which will set it to the current +# unix time in seconds multiplied by 1,000,000. +# Defaults to 0. +sequence.number = time + +# +# varnishkafka log messages configuration +# Debugging, error reporting, etc, not to be confused with varnish logs. +# + +# varnishkafka log level (1 = emergencies .. 7 = debug) +log.level = 6 + +# specify log output (multiples allowed) + +log.stderr = false +log.syslog = true + +# Maximum number of error logs produced per log.rate.period seconds +# This setting is applied per error type. +# log.rate.max defaults to 100 +# log.rate.period defaults to 60 +#log.rate.max = 100 +#log.rate.period = 60 + +# Kafka: log message delivery failures (requires required.acks > 0) +log.kafka.msg.error = true + +# +# JSON Statistics +# +# Statistics is collected from varnishkafka itself as well as librdkafka +# Each JSON object has a top level key of either 'varnishkafka' or +# 'kafka' to indicate which type of statistics the object contains. +# Each line is a valid JSON object. +# + +# Statistics output interval +# Defaults to 60 seconds, use 0 to disable. +#log.statistics.interval = 60 + +# Statistics output file +# Defaults to /tmp/varnishkafka.stats.json +log.statistics.file = /var/cache/varnishkafka/varnishkafka.stats.json + + +# daemonize varnishkafka (boolean) +daemonize = false + + + +####################################################################### +# # +# Standard varnish VSL command line arguments # +# # +# Syntax: # +# varnish.arg.<c> = <value>, where <c> is a command line option. # +# # +# See varnishncsa(1) and varnishlog(1) for valid options. # +# # +####################################################################### + +# -m tag:regex +varnish.arg.m = RxRequest:^(?!PURGE$) + +# Examples: +# -C: ignore case when matching regex +# Non-value arguments need a dummy value to pass parsing, such as 'true'. +#varnish.arg.C = true + +# -n: varnishd instance to get logs from. +#varnish.arg.n = frontend + + +####################################################################### +# # +# Kafka configuration # +# # +# Kafka configuration properties are prefixed with "kafka." # +# and topic properties are prefixed with "kafka.topic.". # +# # +# For the full range of Kafka handle and topic configuration # +# properties, see: # +# http://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md # +# # +# And the Apache Kafka configuration reference: # +# http://kafka.apache.org/08/configuration.html # +# # +####################################################################### + +# Initial list of kafka brokers +kafka.metadata.broker.list = localhost:9092 + +# Maximum number of messages allowed on the local producer queue +# Defaults to 100000 +#kafka.queue.buffering.max.messages = 100000 + +# Maximum number of retries per messageset. +#kafka.message.send.max.retries = 2 + +# Use compression when sending to Kafka.. Default is off. +#kafka.compression.codec = snappy + +# +# Topic configuration +# + +# Topic to produce messages to +kafka.topic = varnish + +# Partition (-1: random, else one of the available partitions) +kafka.partition = -1 + +# Required number of acks +#kafka.topic.request.required.acks = 1 + + +# Local message timeout (milliseconds) +#kafka.topic.message.timeout.ms = 300000 + +# Request ack timeout +#kafka.topic.request.timeout.ms = 5000 + +# SO_SNDBUFF Socket send buffer size. System default is used if 0. +# Default is 0. +#kafka.socket.send.buffer.bytes = 0 diff --git a/debian/varnishkafka.default b/debian/varnishkafka.default new file mode 100644 index 0000000..05c90df --- /dev/null +++ b/debian/varnishkafka.default @@ -0,0 +1,5 @@ +# Configuration file for varnishkafka +# +# This shell script fragment is sourced by /etc/init.d/varnishkafka +# +# VARNISHKAFKA_ENABLED=1 diff --git a/debian/varnishkafka.dirs b/debian/varnishkafka.dirs new file mode 100644 index 0000000..a396e32 --- /dev/null +++ b/debian/varnishkafka.dirs @@ -0,0 +1 @@ +var/cache/varnishkafka \ No newline at end of file diff --git a/debian/varnishkafka.init b/debian/varnishkafka.init new file mode 100644 index 0000000..abe3385 --- /dev/null +++ b/debian/varnishkafka.init @@ -0,0 +1,106 @@ +#! /bin/sh + +### BEGIN INIT INFO +# Provides: varnishkafka +# Required-Start: $local_fs $remote_fs $network +# Required-Stop: $local_fs $remote_fs $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Varnish to Kafka logging daemon +# Description: Varnish to Kafka logging daemon +### END INIT INFO + +# Source function library +. /lib/lsb/init-functions + +NAME=varnishkafka +DESC="HTTP accelerator log deamon" +PATH=/sbin:/bin:/usr/sbin:/usr/bin +DAEMON=/usr/bin/$NAME +PIDFILE=/var/run/$NAME/$NAME.pid +USER=varnishlog + +# Include defaults if available +if [ -f /etc/default/$NAME ] ; then + . /etc/default/$NAME +fi + +# If unset, or set to "0" or "no", exit +if [ -z "${VARNISHKAFKA_ENABLED}" ] || \ + [ "${VARNISHKAFKA_ENABLED}" = "0" ] || \ + [ "${VARNISHKAFKA_ENABLED}" = "no" ]; then + exit 0; +fi + +test -x $DAEMON || exit 0 + +start_varnishkafka() { + output=$(/bin/tempfile -s.varnish) + log_daemon_msg "Starting $DESC" "$NAME" + create_pid_directory + if start-stop-daemon --start --quiet --background \ + --make-pidfile --pidfile ${PIDFILE} \ + --chuid $USER --exec ${DAEMON} -- ${DAEMON_OPTS} \ + > ${output} 2>&1; then + log_end_msg 0 + else + log_end_msg 1 + cat $output + exit 1 + fi + rm $output +} + +stop_varnishkafka(){ + log_daemon_msg "Stopping $DESC" "$NAME" + if start-stop-daemon --stop --quiet --pidfile $PIDFILE \ + --retry 10 --exec $DAEMON; then + log_end_msg 0 + else + log_end_msg 1 + fi +} + +reload_varnishkafka(){ + log_daemon_msg "Reloading $DESC" "$NAME" + if kill -HUP $(cat $PIDFILE) >/dev/null 2>&1; then + log_end_msg 0 + else + log_end_msg 1 + exit 1 + fi +} + +status_varnishkafka(){ + status_of_proc -p "${PIDFILE}" "${DAEMON}" "${NAME}" + exit $? +} + +create_pid_directory() { + install -o $USER -g $USER -d $(dirname $PIDFILE) +} + +case "$1" in + start) + start_varnishkafka + ;; + stop) + stop_varnishkafka + ;; + reload) + reload_varnishkafka + ;; + status) + status_varnishkafka + ;; + restart|force-reload) + $0 stop + $0 start + ;; + *) + log_success_msg "Usage: $0 {start|stop|restart|force-reload|reload}" + exit 1 + ;; +esac + +exit 0 diff --git a/debian/varnishkafka.install b/debian/varnishkafka.install new file mode 100644 index 0000000..099c2a2 --- /dev/null +++ b/debian/varnishkafka.install @@ -0,0 +1,3 @@ +varnishkafka usr/bin +debian/varnishkafka.conf etc +debian/70-varnishkafka.conf etc/rsyslog.d diff --git a/debian/varnishkafka.logrotate b/debian/varnishkafka.logrotate new file mode 100644 index 0000000..6cb1aac --- /dev/null +++ b/debian/varnishkafka.logrotate @@ -0,0 +1,23 @@ +/var/log/varnishkafka.log { + daily + rotate 7 + create 0664 syslog adm + missingok + compress + delaycompress + postrotate + reload rsyslog + endscript +} + +/var/cache/varnishkafka/varnishkafka.stats.json { + weekly + rotate 4 + create 0644 varnishlog root + missingok + compress + delaycompress + postrotate + reload varnishkafka + endscript +} diff --git a/debian/varnishkafka.postinst b/debian/varnishkafka.postinst new file mode 100644 index 0000000..8326162 --- /dev/null +++ b/debian/varnishkafka.postinst @@ -0,0 +1,31 @@ +#!/bin/sh + +# postinst script for varnishkafka +# +# see: dh_installdeb(1) + +set -e + +case "$1" in + configure|reconfigure) + # force rsyslog to create /var/log/varnishkafka.log on install. + restart rsyslog + # chown /var/cache/varnishkafka for writing + chown varnishlog:varnishlog /var/cache/varnishkafka + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/varnishkafka.conf.example b/varnishkafka.conf.example index ce22bca..8080ecd 100644 --- a/varnishkafka.conf.example +++ b/varnishkafka.conf.example @@ -230,7 +230,7 @@ # # # For the full range of Kafka handle and topic configuration # # properties, see: # -# http://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md # +# https://github.com/edenhill/librdkafka/blob/master/rdkafka.h # # # # And the Apache Kafka configuration reference: # # http://kafka.apache.org/08/configuration.html # -- To view, visit https://gerrit.wikimedia.org/r/78782 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ib53267d85e7a638d34b8660557b3c62638abdd54 Gerrit-PatchSet: 29 Gerrit-Project: operations/software/varnish/varnishkafka Gerrit-Branch: debian Gerrit-Owner: Faidon Liambotis <fai...@wikimedia.org> Gerrit-Reviewer: Akosiaris <akosia...@wikimedia.org> Gerrit-Reviewer: Edenhill <mag...@edenhill.se> Gerrit-Reviewer: Faidon Liambotis <fai...@wikimedia.org> Gerrit-Reviewer: Mark Bergsma <m...@wikimedia.org> Gerrit-Reviewer: Ottomata <o...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits