AMBARI-8605 Query predicate .matches doesn't work for stacks endpoint with passed logical OR (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f77be4c4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f77be4c4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f77be4c4 Branch: refs/heads/trunk Commit: f77be4c4df25aef336f00a7aa9bc0591b347dfb6 Parents: 9f296a2 Author: Dmytro Sen <d...@apache.org> Authored: Thu Jan 22 15:13:08 2015 +0200 Committer: Dmytro Sen <d...@apache.org> Committed: Thu Jan 22 19:23:35 2015 +0200 ---------------------------------------------------------------------- .../ambari/server/api/predicate/QueryLexer.java | 195 +++++++++++++++++-- .../server/api/predicate/QueryLexerTest.java | 51 ++++- 2 files changed, 231 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f77be4c4/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java index b645040..e7051a1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java @@ -62,6 +62,7 @@ public class QueryLexer { */ private static final Set<String> SET_IGNORE = new HashSet<String>(); + /** * Constructor. * Register token handlers. @@ -88,14 +89,18 @@ public class QueryLexer { listHandlers = new ArrayList<TokenHandler>(); listHandlers.add(new CloseBracketTokenHandler()); - listHandlers.add(new ValueOperandTokenHandler()); + listHandlers.add(new ComplexValueOperandTokenHandler()); TOKEN_HANDLERS.put(Token.TYPE.RELATIONAL_OPERATOR_FUNC, listHandlers); listHandlers = new ArrayList<TokenHandler>(); listHandlers.add(new CloseBracketTokenHandler()); listHandlers.add(new LogicalOperatorTokenHandler()); - TOKEN_HANDLERS.put(Token.TYPE.VALUE_OPERAND, listHandlers); TOKEN_HANDLERS.put(Token.TYPE.BRACKET_CLOSE, listHandlers); + + listHandlers = new ArrayList<TokenHandler>(listHandlers); + // complex value operands can span multiple tokens + listHandlers.add(0, new ComplexValueOperandTokenHandler()); + TOKEN_HANDLERS.put(Token.TYPE.VALUE_OPERAND, listHandlers); } @@ -139,6 +144,9 @@ public class QueryLexer { tok + "\', previous token type=" + ctx.getLastTokenType()); } } + + ctx.validateEndState(); + return ctx.getTokenList().toArray(new Token[ctx.getTokenList().size()]); } @@ -234,6 +242,23 @@ public class QueryLexer { private Set<String> m_propertiesToIgnore = new HashSet<String>(); /** + * Bracket score. This score is the difference between the number of + * opening brackets and the number of closing brackets processed by + * a handler. Only handlers which process values containing brackets + * will be interested in this information. + */ + private int bracketScore = 0; + + /** + * Intermediate tokens are tokens which are used by a handler which may + * process several adjacent tokens. A handler might push intermediate + * tokens and then in subsequent invocations combine/alter/remove/etc + * these tokens prior to adding them to the context tokens. + */ + private Deque<Token> m_intermediateTokens = new ArrayDeque<Token>(); + + + /** * Constructor. */ private ScanContext() { @@ -329,6 +354,86 @@ public class QueryLexer { m_propertiesToIgnore.addAll(ignoredProperties); } } + + /** + * Add an intermediate token. + * + * @param token the token to add + */ + public void pushIntermediateToken(Token token) { + if (m_ignoreSegmentEndToken == null) { + m_intermediateTokens.add(token); + } else if (token.getType() == m_ignoreSegmentEndToken) { + m_ignoreSegmentEndToken = null; + } + } + + /** + * Return the intermediate tokens if any. + * + * @return the intermediate tokens. Will never return null. + */ + public Deque<Token> getIntermediateTokens() { + return m_intermediateTokens; + } + + /** + * Move all intermediate tokens to the context tokens. + */ + public void addIntermediateTokens() { + m_listTokens.addAll(m_intermediateTokens); + m_intermediateTokens.clear(); + } + + /** + * Obtain the bracket score. This count is the number of outstanding opening brackets. + * A value of 0 indicates all opening and closing brackets are matched + * @return the current bracket score + */ + public int getBracketScore() { + return bracketScore; + } + + /** + * Increment the bracket score by n. This indicates that n unmatched opening brackets + * have been encountered. + * + * @param n amount to increment + * @return the new bracket score after incrementing + */ + public int incrementBracketScore(int n) { + return bracketScore += n; + } + + /** + * Decrement the bracket score. This is done when matching a closing bracket with a previously encountered + * opening bracket. If the requested decrement would result in a negative number an exception is thrown + * as this isn't a valid state. + * + * @param decValue amount to decrement + * @return the new bracket score after decrementing + * @throws InvalidQueryException if the decrement operation will result in a negative value + */ + public int decrementBracketScore(int decValue) throws InvalidQueryException { + bracketScore -= decValue; + if (bracketScore < 0) { + throw new InvalidQueryException("Unexpected closing bracket. Last token type: " + getLastTokenType() + + ", Current property operand: " + getPropertyOperand() + ", tokens: " + getTokenList()); + } + return bracketScore; + } + + //todo: most handlers should implement this + /** + * Validate the end state of the scan context. + * Iterates over each handler associated with the final token type and asks it to validate the context. + * @throws InvalidQueryException if the context is determined to in an invalid end state + */ + public void validateEndState() throws InvalidQueryException { + for (TokenHandler handler : TOKEN_HANDLERS.get(getLastTokenType())) { + handler.validateEndState(this); + } + } } /** @@ -346,7 +451,7 @@ public class QueryLexer { * @throws InvalidQueryException if an invalid token is encountered */ public boolean handleToken(String token, ScanContext ctx) throws InvalidQueryException { - if (handles(token, ctx.getLastTokenType())) { + if (handles(token, ctx)) { _handleToken(token, ctx); ctx.setLastTokenType(getType()); return true; @@ -355,6 +460,12 @@ public class QueryLexer { } } + public void validateEndState(ScanContext ctx) throws InvalidQueryException { + if (! ctx.getIntermediateTokens().isEmpty()) { + throw new InvalidQueryException("Unexpected end of expression."); + } + } + /** * Process a token. * @@ -374,12 +485,12 @@ public class QueryLexer { /** * Determine if a handler handles a specific token type. * - * @param token the token type - * @param previousTokenType the previous token type * + * @param token the token type + * @param ctx scan context * @return true if the handler handles the specified type; false otherwise */ - public abstract boolean handles(String token, Token.TYPE previousTokenType); + public abstract boolean handles(String token, ScanContext ctx); } /** @@ -411,7 +522,7 @@ public class QueryLexer { } @Override - public boolean handles(String token, Token.TYPE previousTokenType) { + public boolean handles(String token, ScanContext ctx) { return token.matches("[^!&\\|<=|>=|!=|=|<|>\\(\\)]+"); } } @@ -431,7 +542,7 @@ public class QueryLexer { } @Override - public boolean handles(String token, Token.TYPE previousTokenType) { + public boolean handles(String token, ScanContext ctx) { return token.matches("[^!&\\|<=|>=|!=|=|<|>]+"); } } @@ -451,7 +562,7 @@ public class QueryLexer { } @Override - public boolean handles(String token, Token.TYPE previousTokenType) { + public boolean handles(String token, ScanContext ctx) { return token.matches("\\("); } } @@ -471,7 +582,7 @@ public class QueryLexer { } @Override - public boolean handles(String token, Token.TYPE previousTokenType) { + public boolean handles(String token, ScanContext ctx) { return token.matches("\\)"); } } @@ -492,7 +603,7 @@ public class QueryLexer { } @Override - public boolean handles(String token, Token.TYPE previousTokenType) { + public boolean handles(String token, ScanContext ctx) { return token.matches("<=|>=|!=|=|<|>"); } } @@ -514,11 +625,67 @@ public class QueryLexer { //todo: add a unary relational operator func @Override - public boolean handles(String token, Token.TYPE previousTokenType) { + public boolean handles(String token, ScanContext ctx) { return token.matches("\\.[a-zA-Z]+\\("); } } + /** + * Complex Value Operand token handler. + * Supports values that span multiple tokens. + */ + private class ComplexValueOperandTokenHandler extends TokenHandler { + @Override + public void _handleToken(String token, ScanContext ctx) throws InvalidQueryException { + if (token.equals(")")) { + ctx.decrementBracketScore(1); + } else if (token.endsWith("(")) { + // .endsWith() is used because of tokens ".matches(",".in(" and".isEmpty(" + ctx.incrementBracketScore(1); + } + + String tokenValue = token; + if (ctx.getBracketScore() > 0) { + Deque<Token> intermediateTokens = ctx.getIntermediateTokens(); + if (! intermediateTokens.isEmpty()) { + Token lastToken = intermediateTokens.peek(); + if (lastToken.getType() == Token.TYPE.VALUE_OPERAND) { + intermediateTokens.pop(); + tokenValue = lastToken.getValue() + token; + } + } + ctx.pushIntermediateToken(new Token(Token.TYPE.VALUE_OPERAND, tokenValue)); + } + + if (ctx.getBracketScore() == 0) { + ctx.addIntermediateTokens(); + ctx.addToken(new Token(Token.TYPE.BRACKET_CLOSE, ")")); + } + } + + @Override + public Token.TYPE getType() { + return Token.TYPE.VALUE_OPERAND; + } + + @Override + public boolean handles(String token, ScanContext ctx) { + Token.TYPE lastTokenType = ctx.getLastTokenType(); + if (lastTokenType == Token.TYPE.RELATIONAL_OPERATOR_FUNC) { + ctx.incrementBracketScore(1); + return true; + } else { + return ctx.getBracketScore() > 0; + } + } + + @Override + public void validateEndState(ScanContext ctx) throws InvalidQueryException { + if (ctx.getBracketScore() > 0) { + throw new InvalidQueryException("Missing closing bracket for function: " + ctx.getTokenList()); + } + } + } /** * Logical Operator token handler. @@ -535,7 +702,7 @@ public class QueryLexer { } @Override - public boolean handles(String token, Token.TYPE previousTokenType) { + public boolean handles(String token, ScanContext ctx) { return token.matches("[!&\\|]"); } } @@ -555,7 +722,7 @@ public class QueryLexer { } @Override - public boolean handles(String token, Token.TYPE previousTokenType) { + public boolean handles(String token, ScanContext ctx) { return "!".equals(token); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f77be4c4/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java index 2c04d6c..8caa821 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java @@ -22,7 +22,6 @@ package org.apache.ambari.server.api.predicate; import org.junit.Test; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -360,4 +359,54 @@ public class QueryLexerTest { //expected } } + + @Test + public void testTokens_matchesRegexp_simple() throws InvalidQueryException { + List<Token> listTokens = new ArrayList<Token>(); + listTokens.add(new Token(Token.TYPE.RELATIONAL_OPERATOR_FUNC, ".matches(")); + listTokens.add(new Token(Token.TYPE.PROPERTY_OPERAND, "StackConfigurations/property_type")); + listTokens.add(new Token(Token.TYPE.VALUE_OPERAND, "(.*USER.*)|(.*GROUP.*)")); + listTokens.add(new Token(Token.TYPE.BRACKET_CLOSE, ")")); + + QueryLexer lexer = new QueryLexer(); + Token[] tokens = lexer.tokens("StackConfigurations/property_type.matches((.*USER.*)|(.*GROUP.*))"); + + assertArrayEquals(listTokens.toArray(new Token[listTokens.size()]), tokens); + } + + @Test + public void testTokens_matchesRegexp() throws InvalidQueryException { + List<Token> listTokens = new ArrayList<Token>(); + listTokens.add(new Token(Token.TYPE.BRACKET_OPEN, "(")); + listTokens.add(new Token(Token.TYPE.RELATIONAL_OPERATOR_FUNC, ".matches(")); + listTokens.add(new Token(Token.TYPE.PROPERTY_OPERAND, "StackConfigurations/property_type")); + listTokens.add(new Token(Token.TYPE.VALUE_OPERAND, "(([^=])|=|!=),.in(&).*USER.*.isEmpty(a).matches(b)")); + listTokens.add(new Token(Token.TYPE.BRACKET_CLOSE, ")")); + listTokens.add(new Token(Token.TYPE.LOGICAL_OPERATOR, "|")); + listTokens.add(new Token(Token.TYPE.RELATIONAL_OPERATOR_FUNC, ".matches(")); + listTokens.add(new Token(Token.TYPE.PROPERTY_OPERAND, "StackConfigurations/property_type")); + listTokens.add(new Token(Token.TYPE.VALUE_OPERAND, "fields format to from .*GROUP.*")); + listTokens.add(new Token(Token.TYPE.BRACKET_CLOSE, ")")); + listTokens.add(new Token(Token.TYPE.BRACKET_CLOSE, ")")); + + QueryLexer lexer = new QueryLexer(); + Token[] tokens = lexer.tokens("(StackConfigurations/property_type.matches((([^=])|=|!=),.in(&).*USER.*" + + ".isEmpty(a).matches(b))|StackConfigurations/property_type.matches(fields format to from .*GROUP.*))"); + + assertArrayEquals("All characters between \".matches(\" and corresponding closing \")\" bracket should " + + "come to VALUE_OPERAND.", listTokens.toArray(new Token[listTokens.size()]), tokens); + } + + @Test(expected = InvalidQueryException.class) + public void testTokens_matchesRegexpInvalidQuery() throws InvalidQueryException { + QueryLexer lexer = new QueryLexer(); + lexer.tokens("StackConfigurations/property_type.matches((.*USER.*)|(.*GROUP.*)"); + } + + @Test(expected = InvalidQueryException.class) + public void testTokens_matchesRegexpInvalidQuery2() throws InvalidQueryException { + QueryLexer lexer = new QueryLexer(); + lexer.tokens("StackConfigurations/property_type.matches((.*USER.*)|(.*GROUP.*)|StackConfigurations/property_type.matches(.*GROUP.*)"); + } + }