Repository: nifi Updated Branches: refs/heads/master e5eda6370 -> c4be80068
NIFI-2950 Adding support for whole number hex values and a fromRadix function NIFI-2950 Fixing typo This closes #1161 Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/c4be8006 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/c4be8006 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/c4be8006 Branch: refs/heads/master Commit: c4be800688bf23a3bdea8def75b84c0f4ded243d Parents: e5eda63 Author: jpercivall <joeperciv...@yahoo.com> Authored: Wed Oct 26 16:34:14 2016 -0400 Committer: Oleg Zhurakousky <o...@suitcase.io> Committed: Thu Nov 17 07:27:51 2016 -0500 ---------------------------------------------------------------------- .../language/antlr/AttributeExpressionLexer.g | 1 + .../language/antlr/AttributeExpressionParser.g | 2 +- .../attribute/expression/language/Query.java | 6 ++ .../evaluation/cast/DecimalCastEvaluator.java | 8 ++- .../evaluation/cast/NumberCastEvaluator.java | 10 +++- .../cast/WholeNumberCastEvaluator.java | 13 ++++- .../functions/FromRadixEvaluator.java | 58 ++++++++++++++++++++ .../language/evaluation/util/NumberParsing.java | 3 +- .../expression/language/TestQuery.java | 19 ++++++- .../asciidoc/expression-language-guide.adoc | 32 +++++++++++ 10 files changed, 141 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g index cf76808..9375579 100644 --- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g +++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g @@ -167,6 +167,7 @@ MINUS : 'minus'; MULTIPLY : 'multiply'; DIVIDE : 'divide'; MATH : 'math'; +FROM_RADIX : 'fromRadix'; TO_RADIX : 'toRadix'; OR : 'or'; AND : 'and'; http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g index 5542fb0..eb50a28 100644 --- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g +++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g @@ -75,7 +75,7 @@ tokens { // functions that return Strings zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 ) LPAREN! RPAREN!; oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY | - PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH) LPAREN! anyArg RPAREN!) | + PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) | (TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!); twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) | (SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!); http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java index 49c7518..fb48b0f 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java @@ -47,6 +47,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.Equals import org.apache.nifi.attribute.expression.language.evaluation.functions.CharSequenceTranslatorEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.FindEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.FormatEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.functions.FromRadixEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.GetDelimitedFieldEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanOrEqualEvaluator; @@ -123,6 +124,7 @@ import org.antlr.runtime.CharStream; import org.antlr.runtime.CommonTokenStream; import org.antlr.runtime.tree.Tree; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FROM_RADIX; import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATH; import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_ATTRIBUTES; import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_DELINEATED_VALUES; @@ -1241,6 +1243,10 @@ public class Query { toWholeNumberEvaluator(argEvaluators.get(0)), toWholeNumberEvaluator(argEvaluators.get(1))), "toRadix"); } } + case FROM_RADIX: { + return addToken(new FromRadixEvaluator(toStringEvaluator(subjectEvaluator), + toWholeNumberEvaluator(argEvaluators.get(0))), "fromRadix"); + } case MOD: { return addToken(new ModEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "mod"); } http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java index 9418028..bce54f7 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java @@ -58,7 +58,13 @@ public class DecimalCastEvaluator extends DecimalEvaluator { case DECIMAL: return new DecimalQueryResult(Double.valueOf(trimmed)); case WHOLE_NUMBER: - final Long resultValue = Long.valueOf(trimmed); + Long resultValue; + try { + resultValue = Long.valueOf(trimmed); + } catch (NumberFormatException e){ + // Will only occur if trimmed is a hex number + resultValue = Long.decode(trimmed); + } return new DecimalQueryResult(resultValue.doubleValue()); case NOT_NUMBER: default: http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java index 4073dc4..00c9ef7 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java @@ -64,8 +64,14 @@ public class NumberCastEvaluator extends NumberEvaluator { case DECIMAL: return new NumberQueryResult(Double.valueOf(trimmed)); case WHOLE_NUMBER: - final Long resultValue = Long.valueOf(trimmed); - return new NumberQueryResult(Long.valueOf(trimmed)); + Long resultValue; + try { + resultValue = Long.valueOf(trimmed); + } catch (NumberFormatException e){ + // Will only occur if trimmed is a hex number + resultValue = Long.decode(trimmed); + } + return new NumberQueryResult(resultValue); case NOT_NUMBER: default: return new NumberQueryResult(null); http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java index 024fb44..736bf2c 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java @@ -56,10 +56,17 @@ public class WholeNumberCastEvaluator extends WholeNumberEvaluator { NumberParsing.ParseResultType parseType = NumberParsing.parse(trimmed); switch (parseType){ case DECIMAL: - final Double resultValue = Double.valueOf(trimmed); - return new WholeNumberQueryResult(resultValue.longValue()); + final Double doubleResultValue = Double.valueOf(trimmed); + return new WholeNumberQueryResult(doubleResultValue.longValue()); case WHOLE_NUMBER: - return new WholeNumberQueryResult(Long.valueOf(trimmed)); + Long longResultValue; + try { + longResultValue = Long.valueOf(trimmed); + } catch (NumberFormatException e){ + // Will only occur if trimmed is a hex number + longResultValue = Long.decode(trimmed); + } + return new WholeNumberQueryResult(longResultValue); case NOT_NUMBER: default: return new WholeNumberQueryResult(null); http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java new file mode 100644 index 0000000..8febd2e --- /dev/null +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language.evaluation.functions; + +import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberQueryResult; + +import java.util.Map; + +public class FromRadixEvaluator extends WholeNumberEvaluator { + + private final Evaluator<String> numberEvaluator; + private final Evaluator<Long> radixEvaluator; + + public FromRadixEvaluator(final Evaluator<String> subject, final Evaluator<Long> radixEvaluator) { + this.numberEvaluator = subject; + this.radixEvaluator = radixEvaluator; + } + + @Override + public QueryResult<Long> evaluate(final Map<String, String> attributes) { + final String result = numberEvaluator.evaluate(attributes).getValue(); + if (result == null) { + return new WholeNumberQueryResult(null); + } + + final Long radix = radixEvaluator.evaluate(attributes).getValue(); + if (radix == null) { + return new WholeNumberQueryResult(null); + } + + long longValue = Long.parseLong(result, radix.intValue()); + + return new WholeNumberQueryResult(longValue); + } + + @Override + public Evaluator<?> getSubjectEvaluator() { + return numberEvaluator; + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java index c79c922..bbfd4e2 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java @@ -21,7 +21,6 @@ import java.util.regex.Pattern; public class NumberParsing { - private static final Pattern NUMBER_PATTERN = Pattern.compile("-?\\d+"); public static enum ParseResultType { NOT_NUMBER, WHOLE_NUMBER, DECIMAL; @@ -69,6 +68,8 @@ public class NumberParsing { private static final Pattern DOUBLE_PATTERN = Pattern.compile(fpRegex); + private static final Pattern NUMBER_PATTERN = Pattern.compile("-?((\\d+)|(0[xX]" + HexDigits + "))"); + private NumberParsing(){ } http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java index f124f76..3b6896c 100644 --- a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java +++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java @@ -1057,11 +1057,14 @@ public class TestQuery { verifyEquals("${literal(5):toNumber()}", attributes, 5L); verifyEquals("${literal(5):toDecimal()}", attributes, 5D); - // Unquoted doubles are not due to more complicated parsing verifyEquals("${literal(\"5.5\")}", attributes, "5.5"); - verifyEquals("${literal(\"5.5\"):toNumber()}", attributes, 5L); - verifyEquals("${literal(\"5.5\"):toDecimal()}", attributes, 5.5D); + verifyEquals("${literal(5.5):toNumber()}", attributes, 5L); + verifyEquals("${literal(5.5):toDecimal()}", attributes, 5.5D); + verifyEquals("${literal('0xF.Fp10'):toDecimal()}", attributes, 0xF.Fp10D); + + verifyEquals("${literal('0xABC'):toNumber()}", attributes, 0xABCL); + verifyEquals("${literal('-0xABC'):toNumber()}", attributes, -0xABCL); } @Test @@ -1176,6 +1179,16 @@ public class TestQuery { } @Test + public void testFromRadix() { + final Map<String, String> attributes = new HashMap<>(); + attributes.put("test1", "ABCDEF"); + attributes.put("test2", "123"); + + verifyEquals("${test1:fromRadix(16)}", attributes, 0xABCDEFL); + verifyEquals("${test2:fromRadix(4)}", attributes, 27L); + } + + @Test public void testBase64Encode(){ final Map<String, String> attributes = new HashMap<>(); attributes.put("userpass", "admin:admin"); http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-docs/src/main/asciidoc/expression-language-guide.adoc ---------------------------------------------------------------------- diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc index 90205e8..c5e2a77 100644 --- a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc +++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc @@ -213,7 +213,11 @@ The Expression Language is generally able to automatically coerce a value of one data type for a function. However, functions do exist to manually coerce a value into a specific data type. See the <<type_cast>> section for more information. +Hex values are supported for Number and Decimal types but they must be quoted and prepended with "0x" when being +interpreted as literals. For example these two expressions are valid (without the quotes or "0x" the expression would fail to run properly): + - ${literal("0xF"):toNumber()} + - ${literal("0xF.Fp10"):toDecimal()} @@ -1645,6 +1649,34 @@ Divide. This is to preserve backwards compatibility and to not force rounding er |======================================================================================= [.function] +=== fromRadix + +*Description*: [.description]#Converts the Subject from a specified Radix (or number base) to a base ten whole number. The subject will converted as is, without interpretation, and all characters +must be valid for the base being converted from. For example converting "0xFF" from hex will not work due to "x" being a invalid hex character. + + If a decimal is passed as the subject, it will first be converted to a whole number and then processed.# + +*Subject Type*: [.subject]#String# + +*Arguments*: + + - [.argName]#_Subject Base_# : [.argDesc]#A Number between 2 and 36 (inclusive)# + +*Return Type*: [.returnType]#Number# + +*Examples*: If the "fileSize" attributes has a value of 1234A, then the following Expressions will yield + the following results: + + +.toRadix Examples +|======================================================================================= +| Expression | Value +| `${fileSize:fromRadix(11)}` | `17720` +| `${fileSize:fromRadix(16)}` | `74570` +| `${fileSize:fromRadix(20)}` | `177290` +|======================================================================================= + +[.function] === random *Description*: [.description]#Returns a random whole number ( 0 to 2^63 - 1) using an insecure random number generator.#
