EBernhardson has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/344637 )
Change subject: [WIP] Update logstash plugins for 5.x ...................................................................... [WIP] Update logstash plugins for 5.x Parts of this will need to be completely rewritten. I was originally thinking the source .gems could also be held in archiva, but after uploading them i don't even see them in archiva. Most likely will adjust the build script to use logstash-plugin to build a zip file, then add our own plugin to that zip, and prompt the user to upload to archiva or something. Perhaps embed some sha1 hashes into the script for what .gem's to expect Change-Id: I1ed9d1b30ffa3e39765b5f0ebc50c7f63e4d73b8 --- A .gitattributes A .gitfat M .gitignore A build.sh A logstash-filters-wikimedia/Gemfile R logstash-filters-wikimedia/lib/logstash/filters/de_dot.rb R logstash-filters-wikimedia/lib/logstash/outputs/sentry.rb R logstash-filters-wikimedia/logstash-filters-wikimedia.gemspec D logstash/filters/prune.rb A plugins-0.5.0.zip 10 files changed, 51 insertions(+), 152 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/software/logstash/plugins refs/changes/37/344637/1 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8e447cd --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.gem filter=fat -text +*.zip filter=fat -text diff --git a/.gitfat b/.gitfat new file mode 100644 index 0000000..0b69f2d --- /dev/null +++ b/.gitfat @@ -0,0 +1,4 @@ +[rsync] +remote = archiva.wikimedia.org::archiva/git-fat +options = --copy-links --verbose + diff --git a/.gitignore b/.gitignore index 3300a23..4929e1c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -*.gem Gemfile.lock .bundle vendor +/target +/logstash-filters-wikimedia/*.gem diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..ee9026a --- /dev/null +++ b/build.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +echo ------------------------ +echo 'Cleanup previous builds' +rm -rf target/ +rm -f wikimedia-logstash-plugins.zip +rm -f logstash-filters-wikimedia/logstash-filters-wikimedia-*.gem +echo + +echo ------------------------ +echo Create an empty place to build our directory structure +mkdir -p target/logstash +echo + +echo ------------------------ +echo Build the wikimedia gem +pushd logstash-filters-wikimedia +gem build logstash-filters-wikimedia.gemspec +popd +echo + +echo ------------------------ +echo Copy all the files we need into the plugin pack +cp logstash-filters-wikimedia/logstash-filters-wikimedia-*.gem target/logstash +cp -rf external/* target/logstash +echo + +echo ------------------------ +echo Zip it up into a logstash 'plugin pack' +pushd target +zip -r ../wikimedia-logstash-plugins.zip . +popd +echo + +echo ------------------------ +echo "Build wikimedia-logstash-plugins.zip complete" diff --git a/logstash-filters-wikimedia/Gemfile b/logstash-filters-wikimedia/Gemfile new file mode 100644 index 0000000..851fabc --- /dev/null +++ b/logstash-filters-wikimedia/Gemfile @@ -0,0 +1,2 @@ +source 'https://rubygems.org' +gemspec diff --git a/logstash/filters/de_dot.rb b/logstash-filters-wikimedia/lib/logstash/filters/de_dot.rb similarity index 100% rename from logstash/filters/de_dot.rb rename to logstash-filters-wikimedia/lib/logstash/filters/de_dot.rb diff --git a/logstash/outputs/sentry.rb b/logstash-filters-wikimedia/lib/logstash/outputs/sentry.rb similarity index 100% rename from logstash/outputs/sentry.rb rename to logstash-filters-wikimedia/lib/logstash/outputs/sentry.rb diff --git a/logstash-filters-wikimedia.gemspec b/logstash-filters-wikimedia/logstash-filters-wikimedia.gemspec similarity index 88% rename from logstash-filters-wikimedia.gemspec rename to logstash-filters-wikimedia/logstash-filters-wikimedia.gemspec index eaa183f..3fd2aa3 100644 --- a/logstash-filters-wikimedia.gemspec +++ b/logstash-filters-wikimedia/logstash-filters-wikimedia.gemspec @@ -1,10 +1,10 @@ Gem::Specification.new do |s| s.name = 'logstash-filters-wikimedia' - s.version = '0.1.5' + s.version = '0.5.0' s.licenses = ['Apache License (2.0)'] s.summary = "Backports of logstash plugins for wikimedia installation. Includes the prune and de_dot filters" - s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" + s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install /path/to/gemfile. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'i...@elastic.co' s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"; diff --git a/logstash/filters/prune.rb b/logstash/filters/prune.rb deleted file mode 100644 index d85be39..0000000 --- a/logstash/filters/prune.rb +++ /dev/null @@ -1,149 +0,0 @@ -# encoding: utf-8 -require "logstash/filters/base" -require "logstash/namespace" - - -# The prune filter is for pruning event data from `@fields` based on whitelist/blacklist -# of field names or their values (names and values can also be regular expressions). - -class LogStash::Filters::Prune < LogStash::Filters::Base - config_name "prune" - - # Trigger whether configation fields and values should be interpolated for - # dynamic values. - # Probably adds some performance overhead. Defaults to false. - config :interpolate, :validate => :boolean, :default => false - - # Include only fields only if their names match specified regexps, default to empty list which means include everything. - # [source,ruby] - # filter { - # %PLUGIN% { - # tags => [ "apache-accesslog" ] - # whitelist_names => [ "method", "(referrer|status)", "${some}_field" ] - # } - # } - config :whitelist_names, :validate => :array, :default => [] - - # Exclude fields which names match specified regexps, by default exclude unresolved `%{field}` strings. - # [source,ruby] - # filter { - # %PLUGIN% { - # tags => [ "apache-accesslog" ] - # blacklist_names => [ "method", "(referrer|status)", "${some}_field" ] - # } - # } - config :blacklist_names, :validate => :array, :default => [ "%\{[^}]+\}" ] - - # Include specified fields only if their values match regexps. - # In case field values are arrays, the fields are pruned on per array item - # thus only matching array items will be included. - # [source,ruby] - # filter { - # %PLUGIN% { - # tags => [ "apache-accesslog" ] - # whitelist_values => [ "uripath", "/index.php", - # "method", "(GET|POST)", - # "status", "^[^2]" ] - # } - # } - config :whitelist_values, :validate => :hash, :default => {} - - # Exclude specified fields if their values match regexps. - # In case field values are arrays, the fields are pruned on per array item - # in case all array items are matched whole field will be deleted. - # [source,ruby] - # filter { - # %PLUGIN% { - # tags => [ "apache-accesslog" ] - # blacklist_values => [ "uripath", "/index.php", - # "method", "(HEAD|OPTIONS)", - # "status", "^[^2]" ] - # } - # } - config :blacklist_values, :validate => :hash, :default => {} - - public - def register - unless @interpolate - @whitelist_names_regexp = Regexp.union(@whitelist_names.map {|x| Regexp.new(x)}) - @blacklist_names_regexp = Regexp.union(@blacklist_names.map {|x| Regexp.new(x)}) - @whitelist_values.each do |key, value| - @whitelist_values[key] = Regexp.new(value) - end - @blacklist_values.each do |key, value| - @blacklist_values[key] = Regexp.new(value) - end - end - end # def register - - public - def filter(event) - return unless filter?(event) - - hash = event.to_hash - - # We need to collect fields which needs to be remove ,and only in the end - # actually remove it since then interpolation mode you can get unexpected - # results as fields with dynamic values will not match since the fields to - # which they refer have already been removed. - fields_to_remove = [] - - unless @whitelist_names.empty? - @whitelist_names_regexp = Regexp.union(@whitelist_names.map {|x| Regexp.new(event.sprintf(x))}) if @interpolate - hash.each_key do |field| - fields_to_remove << field unless field.match(@whitelist_names_regexp) - end - end - - unless @blacklist_names.empty? - @blacklist_names_regexp = Regexp.union(@blacklist_names.map {|x| Regexp.new(event.sprintf(x))}) if @interpolate - hash.each_key do |field| - fields_to_remove << field if field.match(@blacklist_names_regexp) - end - end - - @whitelist_values.each do |key, value| - if @interpolate - key = event.sprintf(key) - value = Regexp.new(event.sprintf(value)) - end - if hash[key] - if hash[key].is_a?(Array) - subvalues_to_remove = hash[key].find_all{|x| not x.match(value)} - unless subvalues_to_remove.empty? - fields_to_remove << (subvalues_to_remove.length == hash[key].length ? key : { :key => key, :values => subvalues_to_remove }) - end - else - fields_to_remove << key if not hash[key].match(value) - end - end - end - - @blacklist_values.each do |key, value| - if @interpolate - key = event.sprintf(key) - value = Regexp.new(event.sprintf(value)) - end - if hash[key] - if hash[key].is_a?(Array) - subvalues_to_remove = hash[key].find_all{|x| x.match(value)} - unless subvalues_to_remove.empty? - fields_to_remove << (subvalues_to_remove.length == hash[key].length ? key : { :key => key, :values => subvalues_to_remove }) - end - else - fields_to_remove << key if hash[key].match(value) - end - end - end - - fields_to_remove.each do |field| - if field.is_a?(Hash) - hash[field[:key]] = hash[field[:key]] - field[:values] - else - hash.delete(field) - end - end - - filter_matched(event) - end # def filter -end # class LogStash::Filters::Prune diff --git a/plugins-0.5.0.zip b/plugins-0.5.0.zip new file mode 100644 index 0000000..36f3629 --- /dev/null +++ b/plugins-0.5.0.zip @@ -0,0 +1 @@ +#$# git-fat fca84d690146884e851329fa59ef4a2c7078f9dd 93466 -- To view, visit https://gerrit.wikimedia.org/r/344637 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1ed9d1b30ffa3e39765b5f0ebc50c7f63e4d73b8 Gerrit-PatchSet: 1 Gerrit-Project: operations/software/logstash/plugins Gerrit-Branch: master Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
