Ottomata has uploaded a new change for review. https://gerrit.wikimedia.org/r/110958
Change subject: Rearranging order of configs in kafkatee.conf.example ...................................................................... Rearranging order of configs in kafkatee.conf.example I've moved the global configs closer to the top of the file. Outputs and inputs are now all at the bottom, as is the include documentation. By putting the include at the bottom of the file, I believe any subsequently included configs will override any previously set ones. Correct me if I am wrong. Change-Id: Ie4cd0f571bb31cdf89323cb84da2fc8e48de3081 --- M kafkatee.conf.example 1 file changed, 144 insertions(+), 145 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/kafkatee refs/changes/58/110958/1 diff --git a/kafkatee.conf.example b/kafkatee.conf.example index 029d147..c4752b0 100644 --- a/kafkatee.conf.example +++ b/kafkatee.conf.example @@ -24,144 +24,6 @@ ####################################################################### -####################################################################### -# # -# Configuration properties # -# # -####################################################################### - - -####################################################################### -# # -# Kafka configuration # -# # -# Kafka configuration properties are prefixed with "kafka." # -# and topic properties are prefixed with "kafka.topic.". # -# # -# For the full range of Kafka handle and topic configuration # -# properties, see: # -# http://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md # -# # -# And the Apache Kafka configuration reference: # -# http://kafka.apache.org/08/configuration.html # -# # -####################################################################### - -# Initial list of kafka brokers -# Default: none -#kafka.metadata.broker.list = localhost - -# Offset file directory. -# Each topic + partition combination has its own offset file. -# Default: current directory -#kafka.offset.store.path = /var/run/offsets/ - -# If the request offset was not found on broker, or there is no -# initial offset known (no stored offset), then reset the offset according -# to this configuration. -# Values: smallest (oldest/beginning), largest (newest/end), error (fail) -# Default: largest -#kafka.topic.auto.offset.reset = smallest - -# Maximum message size. -# Should be synchronized on all producers, brokers and consumers. -# Default: 4000000 -#kafka.message.max.bytes = 10000000 - -# Kafka debugging -# Default: none -#kafka.debug = msg,topic,broker - - - - - - - -####################################################################### -# # -# Message transformation # -# # -# A message read from one of the inputs may be transformed before # -# being enqueued on the output queues. # -# # -# Transformation requires that the input and output encoding differs, # -# i.e., 'input [encoding=json] ..' and 'output.encoding=string' # -# # -# While the input encoding is configured per input, the output # -# encoding is configured globally, all outputs will receive the same # -# message. # -# # -# The currently supported transformation(s) are: # -# JSON input -> string output: # -# JSON data is formatted according to the output.format # -# configuration. The %{field} syntax references the field in the # -# original JSON object by the same name and outputs its value. # -# # -# If the input and output encoding matches then the message remains # -# untouched. # -# # -# The output message delimiter (defaults to newline (\n)) is # -# configurable (output.delimiter) and always appended to all output # -# messages regardless of transformation. # -# The input is always stripped of its delimiter (which is newline # -# for pipe inputs). # -# # -####################################################################### - - -# Output encoding: string or json -# Default: string -#output.encoding = string - - -####################################################################### -# Output formatting # -# # -# The format string is made up of %{..}-tokens and literals. # -# # -# Tokens: # -# # -# %{FIELD} # -# Retrieves the value from the JSON object's field with the # -# same name. # -# # -# %{FIELD?DEFAULT} # -# 'DEFAULT' is the default string to use if no field was matched, # -# the default default string is "-". # -# # -# Literals are copied verbatim to the output string. # -# # -# Example JSON: {"task":19, "name":"Mike"} # -# Example format: Got task %{task} for user %{name?unknown} # -# Example output: Got task 19 for user Mike # -# # -# Note: Multi-level JSON objects are flattened: # -# JSON: {"a": {"b": 9}, "c": "hi"} # -# Gives: { "b": 9, "c": "hi" } # -# # -####################################################################### - -# Output format for JSON -> string transformation. -# Default: none -#output.format = %{hostname} %{sequence} %{dt} %{time_firstbyte} %{ip} %{handling}/%{http_status} %{bytes_sent} %{request_method} http://%{host}%{uri}%{query} - %{mime_type} %{referer} %{x_forwarded_for} %{user_agent} %{accept_language} %{x_analytics} - - -# Output delimiter -# The output message ends with this delimiter. -# Supports \n, \r, \t, \0. -# Default: newline (\n) -#output.delimiter = ;END;\n - - -# Maximum queue size for each output, in number of messages -# Default: 100000 -#output.queue.size = 1000000 - - - - - ####################################################################### @@ -212,15 +74,138 @@ # Default: none #command.term = ./my-cleanup-script.sh - -# Include other config file -#include local.conf - # Set environment variable which will be available for all sub-sequent # command executions (command.*, input pipe, output pipe, ..) #setenv.NMSGS=12 # clear: #setenv.LD_LIBRARY_PATH= + + + + +####################################################################### +# # +# Kafka configuration # +# # +# Kafka configuration properties are prefixed with "kafka." # +# and topic properties are prefixed with "kafka.topic.". # +# # +# For the full range of Kafka handle and topic configuration # +# properties, see: # +# http://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md # +# # +# And the Apache Kafka configuration reference: # +# http://kafka.apache.org/08/configuration.html # +# # +####################################################################### + +# Initial list of kafka brokers +# Default: none +#kafka.metadata.broker.list = localhost + +# Offset file directory. +# Each topic + partition combination has its own offset file. +# Default: current directory +#kafka.offset.store.path = /var/run/offsets/ + +# If the request offset was not found on broker, or there is no +# initial offset known (no stored offset), then reset the offset according +# to this configuration. +# Values: smallest (oldest/beginning), largest (newest/end), error (fail) +# Default: largest +#kafka.topic.auto.offset.reset = smallest + +# Maximum message size. +# Should be synchronized on all producers, brokers and consumers. +# Default: 4000000 +#kafka.message.max.bytes = 10000000 + +# Kafka debugging +# Default: none +#kafka.debug = msg,topic,broker + + + + +####################################################################### +# # +# Message transformation # +# # +# A message read from one of the inputs may be transformed before # +# being enqueued on the output queues. # +# # +# Transformation requires that the input and output encoding differs, # +# i.e., 'input [encoding=json] ..' and 'output.encoding=string' # +# # +# While the input encoding is configured per input, the output # +# encoding is configured globally, all outputs will receive the same # +# message. # +# # +# The currently supported transformation(s) are: # +# JSON input -> string output: # +# JSON data is formatted according to the output.format # +# configuration. The %{field} syntax references the field in the # +# original JSON object by the same name and outputs its value. # +# # +# If the input and output encoding matches then the message remains # +# untouched. # +# # +# The output message delimiter (defaults to newline (\n)) is # +# configurable (output.delimiter) and always appended to all output # +# messages regardless of transformation. # +# The input is always stripped of its delimiter (which is newline # +# for pipe inputs). # +# # +####################################################################### + +# Output encoding: string or json +# Default: string +#output.encoding = string + + + + +####################################################################### +# Output formatting # +# # +# The format string is made up of %{..}-tokens and literals. # +# # +# Tokens: # +# # +# %{FIELD} # +# Retrieves the value from the JSON object's field with the # +# same name. # +# # +# %{FIELD?DEFAULT} # +# 'DEFAULT' is the default string to use if no field was matched, # +# the default default string is "-". # +# # +# Literals are copied verbatim to the output string. # +# # +# Example JSON: {"task":19, "name":"Mike"} # +# Example format: Got task %{task} for user %{name?unknown} # +# Example output: Got task 19 for user Mike # +# # +# Note: Multi-level JSON objects are flattened: # +# JSON: {"a": {"b": 9}, "c": "hi"} # +# Gives: { "b": 9, "c": "hi" } # +# # +####################################################################### + +# Output format for JSON -> string transformation. +# Default: none +#output.format = %{hostname} %{sequence} %{dt} %{time_firstbyte} %{ip} %{handling}/%{http_status} %{bytes_sent} %{request_method} http://%{host}%{uri}%{query} - %{mime_type} %{referer} %{x_forwarded_for} %{user_agent} %{accept_language} %{x_analytics} + +# Output delimiter +# The output message ends with this delimiter. +# Supports \n, \r, \t, \0. +# Default: newline (\n) +#output.delimiter = ;END;\n + +# Maximum queue size for each output, in number of messages +# Default: 100000 +#output.queue.size = 1000000 + @@ -251,6 +236,7 @@ # enveloped by brackets: [encoding=string,foo=bar] # # # ####################################################################### + ####################################################################### # Kafka consumer input syntax: # @@ -288,8 +274,6 @@ #input [encoding=string] pipe tail -f a.log | grep -v ^DEBUG: #input pipe wget -q -O- http://example.com/api/events - - @@ -350,4 +334,19 @@ # SIGHUP signal, thus allowing log file rotation. # ####################################################################### -#output file 100 /tmp/sampled-100.txt \ No newline at end of file +#output file 100 /tmp/sampled-100.txt + + + + +####################################################################### +# Include other config files using include clause. # +# You may use file glob matching to include files, e.g. # +# include /etc/kafkatee.d/*.conf # +# # +# These files will be include in alphabetic order and override # +# any previously set configs. # +####################################################################### + +# Include other config file +#include local.conf \ No newline at end of file -- To view, visit https://gerrit.wikimedia.org/r/110958 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ie4cd0f571bb31cdf89323cb84da2fc8e48de3081 Gerrit-PatchSet: 1 Gerrit-Project: analytics/kafkatee Gerrit-Branch: master Gerrit-Owner: Ottomata <o...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
