Add a small filter language that allows rich expressions to be used
when doing discovery.
A simple parser/scanner was included in MCollective::Matcher and
it's available on all SimpleRPC clients with the -S option and
on any MCollective::RPC::Client using the #compound_filter method.
Signed-off-by: Pieter Loubser <ploub...@gmail.com>
---
Local-branch: feature/master/8181
lib/mcollective.rb | 1 +
lib/mcollective/matcher.rb | 17 +++++
lib/mcollective/matcher/parser.rb | 93 +++++++++++++++++++++++++++
lib/mcollective/matcher/scanner.rb | 123 ++++++++++++++++++++++++++++++++++++
lib/mcollective/optionparser.rb | 4 +
lib/mcollective/rpc/client.rb | 6 ++
lib/mcollective/security/base.rb | 30 +++++++++
lib/mcollective/util.rb | 18 +++++
spec/unit/matcher/parser_spec.rb | 106 +++++++++++++++++++++++++++++++
spec/unit/matcher/scanner_spec.rb | 71 +++++++++++++++++++++
spec/unit/util_spec.rb | 23 +++++++
website/simplerpc/clients.md | 2 +-
12 files changed, 493 insertions(+), 1 deletions(-)
create mode 100644 lib/mcollective/matcher.rb
create mode 100644 lib/mcollective/matcher/parser.rb
create mode 100644 lib/mcollective/matcher/scanner.rb
create mode 100644 spec/unit/matcher/parser_spec.rb
create mode 100644 spec/unit/matcher/scanner_spec.rb
diff --git a/lib/mcollective.rb b/lib/mcollective.rb
index 994b06d..98be7de 100644
--- a/lib/mcollective.rb
+++ b/lib/mcollective.rb
@@ -46,6 +46,7 @@ module MCollective
autoload :Registration, "mcollective/registration"
autoload :PluginManager, "mcollective/pluginmanager"
autoload :RPC, "mcollective/rpc"
+ autoload :Matcher, "mcollective/matcher"
autoload :Message, "mcollective/message"
autoload :SSL, "mcollective/ssl"
autoload :Application, "mcollective/application"
diff --git a/lib/mcollective/matcher.rb b/lib/mcollective/matcher.rb
new file mode 100644
index 0000000..431a378
--- /dev/null
+++ b/lib/mcollective/matcher.rb
@@ -0,0 +1,17 @@
+module MCollective
+ # A parser and scanner that creates a stack machine for a simple
+ # fact and class matching language used on the CLI to facilitate
+ # a rich discovery language
+ #
+ # Language EBNF
+ #
+ # compound = ["("] expression [")"] {["("] expression [")"]}
+ # expression = [!|not]statement ["and"|"or"] [!|not] statement
+ # char = A-Z | a-z | < | > | => | =< | _ | - |* | / { A-Z | a-z | < | > |
=> | =< | _ | - | * | / | }
+ # int = 0|1|2|3|4|5|6|7|8|9{|0|1|2|3|4|5|6|7|8|9|0}
+ module Matcher
+ autoload :Parser, "mcollective/matcher/parser"
+ autoload :Scanner, "mcollective/matcher/scanner"
+ end
+end
+# vi:tabstop=4:expandtab:ai
diff --git a/lib/mcollective/matcher/parser.rb
b/lib/mcollective/matcher/parser.rb
new file mode 100644
index 0000000..67331b2
--- /dev/null
+++ b/lib/mcollective/matcher/parser.rb
@@ -0,0 +1,93 @@
+module MCollective
+ module Matcher
+ class Parser
+ attr_reader :scanner, :execution_stack
+
+ def initialize(args)
+ @scanner = Scanner.new(args)
+ @execution_stack = []
+ parse
+ end
+
+ # Parse the input string, one token at a time a contruct the call
stack
+ def parse
+ p_token,p_token_value = nil
+ c_token,c_token_value = @scanner.get_token
+ parenth = 0
+
+ while (c_token != nil)
+ @scanner.token_index += 1
+ n_token, n_token_value = @scanner.get_token
+
+ unless n_token == " "
+ case c_token
+ when "and"
+ unless (n_token =~ /not|statement|\(/) ||
(scanner.token_index == scanner.arguments.size)
+ raise "Error at column
#{scanner.token_index}. \nExpected 'not', 'statement' or '('. Found
'#{n_token_value}'"
+ end
+
+ if p_token == nil
+ raise "Error at column
#{scanner.token_index}. \n Expression cannot start with 'and'"
+ elsif (p_token == "and" || p_token == "or")
+ raise "Error at column
#{scanner.token_index}. \n #{p_token} cannot be followed by 'and'"
+ end
+
+ when "or"
+ unless (n_token =~ /not|statement|\(/) ||
(scanner.token_index == scanner.arguments.size)
+ raise "Error at column
#{scanner.token_index}. \nExpected 'not', 'statement', '('. Found
'#{n_token_value}'"
+ end
+
+ if p_token == nil
+ raise "Error at column
#{scanner.token_index}. \n Expression cannot start with 'or'"
+ elsif (p_token == "and" || p_token == "or")
+ raise "Error at column
#{scanner.token_index}. \n #{p_token} cannot be followed by 'or'"
+ end
+
+ when "not"
+ unless n_token =~ /statement|\(|not/
+ raise "Error at column
#{scanner.token_index}. \nExpected 'statement' or '('. Found '#{n_token_value}'"
+ end
+
+ when "statement"
+ unless n_token =~ /and|or|\)/
+ unless scanner.token_index ==
scanner.arguments.size
+ raise "Error at column
#{scanner.token_index}. \nExpected 'and', 'or', ')'. Found '#{n_token_value}'"
+ end
+ end
+
+ when ")"
+ unless (n_token =~ /|and|or|not|\(/)
+ unless(scanner.token_index ==
scanner.arguments.size)
+ raise "Error at column
#{scanner.token_index}. \nExpected 'and', 'or', 'not' or '('. Found
'#{n_token_value}'"
+ end
+ end
+ parenth += 1
+
+ when "("
+ unless n_token =~ /statement|not|\(/
+ raise "Error at column
#{scanner.token_index}. \nExpected 'statement', '(', not. Found
'#{n_token_value}'"
+ end
+ parenth -= 1
+
+ else
+ raise "Unexpected token found at column
#{scanner.token_index}. '#{c_token_value}'"
+ end
+
+ unless n_token == " "
+ @execution_stack << {c_token => c_token_value}
+ end
+
+ p_token, p_token_value = c_token, c_token_value
+ c_token, c_token_value = n_token, n_token_value
+ end
+ end
+
+ if parenth < 0
+ raise "Error. Missing parentheses ')'."
+ elsif parenth > 0
+ raise "Error. Missing parentheses '('."
+ end
+ end
+ end
+ end
+end
diff --git a/lib/mcollective/matcher/scanner.rb
b/lib/mcollective/matcher/scanner.rb
new file mode 100644
index 0000000..8dcb91b
--- /dev/null
+++ b/lib/mcollective/matcher/scanner.rb
@@ -0,0 +1,123 @@
+module MCollective
+ module Matcher
+ class Scanner
+ attr_accessor :arguments, :token_index
+
+ def initialize(arguments)
+ @token_index = 0
+ @arguments = arguments
+ end
+
+ # Scans the input string and identifies single language tokens
+ def get_token
+ if @token_index >= @arguments.size
+ return nil
+ end
+
+ begin
+ case @arguments.split("")[@token_index]
+ when "("
+ return "(", "("
+
+ when ")"
+ return ")", ")"
+
+ when "n"
+ if (@arguments.split("")[@token_index + 1] == "o")
&& (@arguments.split("")[@token_index + 2] == "t") &&
((@arguments.split("")[@token_index + 3] == " ") ||
(@arguments.split("")[@token_index + 3] == "("))
+ @token_index += 2
+ return "not", "not"
+ else
+ gen_statement
+ end
+
+ when "!"
+ return "not", "not"
+
+ when "a"
+ if (@arguments.split("")[@token_index + 1] == "n")
&& (@arguments.split("")[@token_index + 2] == "d") &&
((@arguments.split("")[@token_index + 3] == " ") ||
(@arguments.split("")[@token_index + 3] == "("))
+ @token_index += 2
+ return "and", "and"
+ else
+ gen_statement
+ end
+
+ when "o"
+ if (@arguments.split("")[@token_index + 1] == "r")
&& ((@arguments.split("")[@token_index + 2] == " ") ||
(@arguments.split("")[@token_index + 2] == "("))
+ @token_index += 1
+ return "or", "or"
+ else
+ gen_statement
+ end
+
+ when " "
+ return " ", " "
+
+ else
+ gen_statement
+ end
+ end
+ rescue NoMethodError => e
+ pp e
+ raise "Cannot end statement with 'and', 'or', 'not'"
+ end
+
+ private
+ # Helper generates a statement token
+ def gen_statement
+ current_token_value = ""
+ j = @token_index
+
+ begin
+ if (@arguments.split("")[j] == "/")
+ begin
+ current_token_value << @arguments.split("")[j]
+ j += 1
+ if @arguments.split("")[j] == "/"
+ current_token_value << "/"
+ break
+ end
+ end until (j >= @arguments.size) ||
(@arguments.split("")[j] =~ /\//)
+ elsif (@arguments.split("")[j] =~ /=|<|>/)
+ while !(@arguments.split("")[j] =~ /=|<|>/)
+ current_token_value << @arguments.split("")[j]
+ j += 1
+ end
+
+ current_token_value << @arguments.split("")[j]
+ j += 1
+
+ if @arguments.split("")[j] == "/"
+ begin
+ current_token_value << @arguments.split("")[j]
+ j += 1
+ if @arguments.split("")[j] == "/"
+ current_token_value << "/"
+ break
+ end
+ end until (j >= @arguments.size) ||
(@arguments.split("")[j] =~ /\//)
+ else
+ while (j < @arguments.size) &&
((@arguments.split("")[j] != " ") && (@arguments.split("")[j] != ")"))
+ current_token_value << @arguments.split("")[j]
+ j += 1
+ end
+ end
+ else
+ begin
+ current_token_value << @arguments.split("")[j]
+ j += 1
+ end until (j >= @arguments.size) ||
(@arguments.split("")[j] =~ /\s|\)/)
+ end
+ rescue Exception => e
+ raise "Invalid token found - '#{current_token_value}'"
+ end
+
+ if current_token_value =~ /^(and|or|not|!)$/
+ raise "Class name cannot be 'and', 'or', 'not'. Found
'#{current_token_value}'"
+ end
+
+ @token_index += current_token_value.size - 1
+ return "statement", current_token_value
+ end
+ end
+ end
+end
diff --git a/lib/mcollective/optionparser.rb b/lib/mcollective/optionparser.rb
index a6bfcb7..9460fa3 100644
--- a/lib/mcollective/optionparser.rb
+++ b/lib/mcollective/optionparser.rb
@@ -73,6 +73,10 @@ module MCollective
end
end
+ @parser.on('-S', '--select FILTER', 'Compound filter combining
facts and classes') do |f|
+ @options[:filter]["compound"] =
MCollective::Matcher::Parser.new(f).execution_stack
+ end
+
@parser.on('-F', '--wf', '--with-fact fact=val', 'Match hosts with
a certain fact') do |f|
fact_parsed = parse_fact(f)
diff --git a/lib/mcollective/rpc/client.rb b/lib/mcollective/rpc/client.rb
index ba62841..547c47a 100644
--- a/lib/mcollective/rpc/client.rb
+++ b/lib/mcollective/rpc/client.rb
@@ -303,6 +303,12 @@ module MCollective
reset
end
+ # Set a compound filter
+ def compound_filter(filter)
+ @filter["compound"] =
Matcher::Parser.new(filter).execution_stack
+ reset
+ end
+
# Resets various internal parts of the class, most importantly it
clears
# out the cached discovery
def reset
diff --git a/lib/mcollective/security/base.rb b/lib/mcollective/security/base.rb
index af33d01..b8ebdb2 100644
--- a/lib/mcollective/security/base.rb
+++ b/lib/mcollective/security/base.rb
@@ -72,6 +72,36 @@ module MCollective
end
end
+ when "compound"
+ result = []
+
+ filter[key].each do |expression|
+ case expression.keys.first
+ when "statement"
+ result <<
Util.eval_compound_statement(expression).to_s
+ when "and"
+ result << "&&"
+ when "or"
+ result << "||"
+ when "("
+ result << "("
+ when ")"
+ result << ")"
+ when "not"
+ result << "!"
+ end
+ end
+
+ result = eval(result.join(" "))
+
+ if result
+ Log.debug("Passing based on class and fact
composition")
+ passed +=1
+ else
+ Log.debug("Failing based on class and fact
composition")
+ failed +=1
+ end
+
when "agent"
filter[key].each do |f|
if Util.has_agent?(f) || f == "mcollective"
diff --git a/lib/mcollective/util.rb b/lib/mcollective/util.rb
index 1a909ae..22abe1a 100644
--- a/lib/mcollective/util.rb
+++ b/lib/mcollective/util.rb
@@ -233,6 +233,24 @@ module MCollective
return str
end
+
+ def self.eval_compound_statement(expression)
+ if expression.values.first =~ /^\//
+ return Util.has_cf_class?(expression.values.first)
+ elsif expression.values.first =~ />=|<=|=|<|>/
+ optype = expression.values.first.match(/>=|<=|=|<|>/)
+ name, value = expression.values.first.split(optype[0])
+ unless value.split("")[0] == "/"
+ optype[0] == "=" ? optype = "==" : optype = optype[0]
+ else
+ optype = "=~"
+ end
+
+ return Util.has_fact?(name,value, optype).to_s
+ else
+ return Util.has_cf_class?(expression.values.first)
+ end
+ end
end
end
diff --git a/spec/unit/matcher/parser_spec.rb b/spec/unit/matcher/parser_spec.rb
new file mode 100644
index 0000000..319bb6e
--- /dev/null
+++ b/spec/unit/matcher/parser_spec.rb
@@ -0,0 +1,106 @@
+#! /usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module MCollective
+ module Matcher
+ describe Parser do
+ describe '#parse' do
+ it "should parse statements seperated by '='" do
+ parser = Parser.new("foo=bar")
+ parser.execution_stack.should == [{"statement" =>
"foo=bar"}]
+ end
+
+ it "should parse statements seperated by '<'" do
+ parser = Parser.new("foo<bar")
+ parser.execution_stack.should == [{"statement" =>
"foo<bar"}]
+ end
+
+ it "should parse statements seperated by '>'" do
+ parser = Parser.new("foo>bar")
+ parser.execution_stack.should == [{"statement" =>
"foo>bar"}]
+ end
+
+ it "should parse statements seperated by '<='" do
+ parser = Parser.new("foo<=bar")
+ parser.execution_stack.should == [{"statement" =>
"foo<=bar"}]
+ end
+
+ it "should parse statements seperated by '>='" do
+ parser = Parser.new("foo>=bar")
+ parser.execution_stack.should == [{"statement" =>
"foo>=bar"}]
+ end
+
+ it "should parse class regex statements" do
+ parser = Parser.new("/foo/")
+ parser.execution_stack.should == [{"statement" => "/foo/"}]
+ end
+
+ it "should parse fact regex statements" do
+ parser = Parser.new("foo=/bar/")
+ parser.execution_stack.should == [{"statement" =>
"foo=/bar/"}]
+ end
+
+ it "should parse a correct 'and' token" do
+ parser = Parser.new("foo=bar and bar=foo")
+ parser.execution_stack.should == [{"statement" =>
"foo=bar"}, {"and" => "and"}, {"statement" => "bar=foo"}]
+ end
+
+ it "should not parse an incorrect and token" do
+ expect {
+ parser = Parser.new("and foo=bar")
+ }.to raise_error("Error at column 10. \n Expression cannot
start with 'and'")
+ end
+
+ it "should parse a correct 'or' token" do
+ parser = Parser.new("foo=bar or bar=foo")
+ parser.execution_stack.should == [{"statement" =>
"foo=bar"}, {"or" => "or"}, {"statement" => "bar=foo"}]
+ end
+
+ it "should not parse an incorrect and token" do
+ expect {
+ parser = Parser.new("or foo=bar")
+ }.to raise_error("Error at column 9. \n Expression cannot
start with 'or'")
+ end
+
+ it "should parse a correct 'not' token" do
+ parser = Parser.new("! bar=foo")
+ parser.execution_stack.should == [{"not" => "not"},
{"statement" => "bar=foo"}]
+ parser = Parser.new("not bar=foo")
+ parser.execution_stack.should == [{"not" => "not"},
{"statement" => "bar=foo"}]
+ end
+
+ it "should not parse an incorrect 'not' token" do
+ expect {
+ parser = Parser.new("foo=bar !")
+ }.to raise_error("Error at column 8. \nExpected 'and',
'or', ')'. Found 'not'")
+ end
+
+ it "should parse correct parentheses" do
+ parser = Parser.new("(foo=bar)")
+ parser.execution_stack.should == [{"(" => "("},
{"statement" => "foo=bar"}, {")" => ")"}]
+ end
+
+ it "should fail on incorrect parentheses" do
+ expect {
+ parser = Parser.new(")foo=bar(")
+ }.to raise_error("Error. Missing parentheses '('.")
+ end
+
+ it "should fail on missing parentheses" do
+ expect {
+ parser = Parser.new("(foo=bar")
+ }.to raise_error("Error. Missing parentheses ')'.")
+ end
+
+ it "should parse correctly formatted compound statements" do
+ parser = Parser.new("(foo=bar or foo=rab) and (bar=foo)")
+ parser.execution_stack.should == [{"(" => "("},
{"statement"=>"foo=bar"}, {"or"=>"or"}, {"statement"=>"foo=rab"},
+ {")"=>")"},
{"and"=>"and"}, {"("=>"("}, {"statement"=>"bar=foo"},
+ {")"=>")"}]
+ end
+
+ end
+ end
+ end
+end
diff --git a/spec/unit/matcher/scanner_spec.rb
b/spec/unit/matcher/scanner_spec.rb
new file mode 100644
index 0000000..aab2172
--- /dev/null
+++ b/spec/unit/matcher/scanner_spec.rb
@@ -0,0 +1,71 @@
+#! /usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module MCollective
+ module Matcher
+ describe 'scanner' do
+ it "should identify a '(' token" do
+ scanner = Scanner.new("(")
+ token = scanner.get_token
+ token.should == ["(", "("]
+ end
+
+ it "should identify a ')' token" do
+ scanner = Scanner.new(")")
+ token = scanner.get_token
+ token.should == [")", ")"]
+ end
+
+ it "should identify a 'and' token" do
+ scanner = Scanner.new("and ")
+ token = scanner.get_token
+ token.should == ["and", "and"]
+ end
+
+ it "should identify a 'or' token" do
+ scanner = Scanner.new("or ")
+ token = scanner.get_token
+ token.should == ["or", "or"]
+ end
+
+ it "should identify a 'not' token" do
+ scanner = Scanner.new("not ")
+ token = scanner.get_token
+ token.should == ["not", "not"]
+ end
+
+ it "should identify a '!' token" do
+ scanner = Scanner.new("!")
+ token = scanner.get_token
+ token.should == ["not", "not"]
+ end
+
+ it "should identify a fact statement token" do
+ scanner = Scanner.new("foo=bar")
+ token = scanner.get_token
+ token.should == ["statement", "foo=bar"]
+ end
+
+ it "should identify a fact statement token" do
+ scanner = Scanner.new("foo=bar")
+ token = scanner.get_token
+ token.should == ["statement", "foo=bar"]
+ end
+
+ it "should identify a class statement token" do
+ scanner = Scanner.new("/class/")
+ token = scanner.get_token
+ token.should == ["statement", "/class/"]
+ end
+
+ it "should fail if expression terminates with 'and'" do
+ scanner = Scanner.new("and")
+
+ expect {
+ token = scanner.get_token
+ }.to raise_error("Class name cannot be 'and', 'or', 'not'.
Found 'and'")
+ end
+ end
+ end
+end
diff --git a/spec/unit/util_spec.rb b/spec/unit/util_spec.rb
index 0f29b90..5b91f62 100644
--- a/spec/unit/util_spec.rb
+++ b/spec/unit/util_spec.rb
@@ -283,5 +283,28 @@ module MCollective
Util.parse_fact_string("foo == bar").should == {:fact =>
"foo", :value => "bar", :operator => "=="}
end
end
+
+ describe "#eval_compound_statement" do
+ it "should return correctly on a regex class statement" do
+ Util.expects(:has_cf_class?).with("/foo/").returns(true)
+ Util.eval_compound_statement({"statement" => "/foo/"}).should
== true
+ Util.expects(:has_cf_class?).with("/foo/").returns(false)
+ Util.eval_compound_statement({"statement" => "/foo/"}).should
== false
+ end
+
+ it "should return correcly for string and regex facts" do
+ Util.expects(:has_fact?).with("foo", "bar", "==").returns(true)
+ Util.eval_compound_statement({"statement" =>
"foo=bar"}).should == "true"
+ Util.expects(:has_fact?).with("foo", "/bar/",
"=~").returns(false)
+ Util.eval_compound_statement({"statement" =>
"foo=/bar/"}).should == "false"
+ end
+
+ it "should return correctly on a string class statement" do
+ Util.expects(:has_cf_class?).with("foo").returns(true)
+ Util.eval_compound_statement({"statement" => "foo"}).should ==
true
+ Util.expects(:has_cf_class?).with("foo").returns(false)
+ Util.eval_compound_statement({"statement" => "foo"}).should ==
false
+ end
+ end
end
end
diff --git a/website/simplerpc/clients.md b/website/simplerpc/clients.md
index 5bd89aa..a19f1f1 100644
--- a/website/simplerpc/clients.md
+++ b/website/simplerpc/clients.md
@@ -186,7 +186,7 @@ mc.fact_filter "country", "uk"
printrpc mc.echo(:msg => "Welcome to MCollective Simple RPC")
{% endhighlight %}
-You can set other filters like _agent`_`filter_ and _identity`_`filter_.
+You can set other filters like _agent`_`filter_, _identity`_`filter_ and
_compound`_`filter_.
As of version 1.1.0 the fact_filter method supports a few other forms in
adition to above:
--
1.7.1
--
You received this message because you are subscribed to the Google Groups
"Puppet Developers" group.
To post to this group, send email to puppet-dev@googlegroups.com.
To unsubscribe from this group, send email to
puppet-dev+unsubscr...@googlegroups.com.
For more options, visit this group at
http://groups.google.com/group/puppet-dev?hl=en.
