Repository: nifi Updated Branches: refs/heads/master 06d45c3a6 -> 716587d09
NIFI-5169 This closes #2705. Upgrade to JSONPath 2.4 NIFI-5169 Added explicit checking for empty strings in JsonPath expression property. Signed-off-by: joewitt <joew...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/716587d0 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/716587d0 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/716587d0 Branch: refs/heads/master Commit: 716587d09f97cb01d685b79c010534af9aea1985 Parents: 06d45c3 Author: Mike Thomsen <mikerthom...@gmail.com> Authored: Tue May 15 05:49:45 2018 -0400 Committer: joewitt <joew...@apache.org> Committed: Wed May 23 16:16:25 2018 -0500 ---------------------------------------------------------------------- nifi-commons/nifi-expression-language/pom.xml | 2 +- .../standard/AbstractJsonPathProcessor.java | 14 +- .../util/JsonPathExpressionValidator.java | 503 ------------------- .../standard/TestEvaluateJsonPath.java | 40 ++ nifi-nar-bundles/nifi-standard-bundle/pom.xml | 2 +- .../nifi-record-serialization-services/pom.xml | 2 +- 6 files changed, 52 insertions(+), 511 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/716587d0/nifi-commons/nifi-expression-language/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/pom.xml b/nifi-commons/nifi-expression-language/pom.xml index e2cebb9..ae42bd2 100644 --- a/nifi-commons/nifi-expression-language/pom.xml +++ b/nifi-commons/nifi-expression-language/pom.xml @@ -76,7 +76,7 @@ <dependency> <groupId>com.jayway.jsonpath</groupId> <artifactId>json-path</artifactId> - <version>2.0.0</version> + <version>2.4.0</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> http://git-wip-us.apache.org/repos/asf/nifi/blob/716587d0/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java index 21172e9..82f31ff 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java @@ -29,8 +29,8 @@ import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processors.standard.util.JsonPathExpressionValidator; import org.apache.nifi.stream.io.BufferedInputStream; +import org.apache.nifi.util.StringUtils; import java.io.IOException; import java.io.InputStream; @@ -112,11 +112,15 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor { public ValidationResult validate(final String subject, final String input, final ValidationContext context) { String error = null; if (isStale(subject, input)) { - if (JsonPathExpressionValidator.isValidExpression(input)) { - JsonPath compiledJsonPath = JsonPath.compile(input); - cacheComputedValue(subject, input, compiledJsonPath); + if (!StringUtils.isBlank(input)) { + try { + JsonPath compiledJsonPath = JsonPath.compile(input); + cacheComputedValue(subject, input, compiledJsonPath); + } catch (Exception ex) { + error = String.format("specified expression was not valid: %s", input); + } } else { - error = "specified expression was not valid: " + input; + error = "the expression cannot be empty."; } } return new ValidationResult.Builder().subject(subject).valid(error == null).explanation(error).build(); http://git-wip-us.apache.org/repos/asf/nifi/blob/716587d0/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathExpressionValidator.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathExpressionValidator.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathExpressionValidator.java deleted file mode 100644 index 2a0bd43..0000000 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathExpressionValidator.java +++ /dev/null @@ -1,503 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.standard.util; - -import com.jayway.jsonpath.Filter; -import com.jayway.jsonpath.Predicate; -import com.jayway.jsonpath.internal.Utils; -import com.jayway.jsonpath.internal.token.ArrayPathToken; -import com.jayway.jsonpath.internal.token.PathToken; -import com.jayway.jsonpath.internal.token.PredicatePathToken; -import com.jayway.jsonpath.internal.token.PropertyPathToken; -import com.jayway.jsonpath.internal.token.RootPathToken; -import com.jayway.jsonpath.internal.token.ScanPathToken; -import com.jayway.jsonpath.internal.token.WildcardPathToken; -import org.apache.nifi.util.StringUtils; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.regex.Pattern; - -import static java.util.Arrays.asList; - -/** - * JsonPathExpressionValidator performs the same execution as com.jayway.jsonpath.internal.PathCompiler, but does not throw exceptions when an invalid path segment is found. Limited access to create - * JsonPath objects requires a separate flow of execution in avoiding exceptions. - * - * @see - * <a href="https://github.com/jayway/JsonPath";>https://github.com/jayway/JsonPath</a> - */ -public class JsonPathExpressionValidator { - - private static final String PROPERTY_OPEN = "['"; - private static final String PROPERTY_CLOSE = "']"; - private static final char DOCUMENT = '$'; - private static final char ANY = '*'; - private static final char PERIOD = '.'; - private static final char BRACKET_OPEN = '['; - private static final char BRACKET_CLOSE = ']'; - private static final char SPACE = ' '; - - /** - * Performs a validation of a provided JsonPath expression. - * <p/> - * Typically this is used in the context of: - * <code> - * <pre> - * JsonPath compiledJsonPath = null; - * if (JsonPathExpressionValidator.isValidExpression(input)) { - * compiledJsonPath = JsonPath.compile(input); - * ... - * } else { - * // error handling - * } - * </pre> - * </code> - * - * @param path to evaluate for validity - * @param filters applied to path expression; this is typically unused in the context of Processors - * @return true if the specified path is valid; false otherwise - */ - public static boolean isValidExpression(String path, Predicate... filters) { - path = path.trim(); - if (StringUtils.isBlank(path)) { - // "Path may not be null empty" - return false; - } - if (path.endsWith("..")) { - // "A path can not end with a scan." - return false; - } - - LinkedList<Predicate> filterList = new LinkedList<>(asList(filters)); - - if (path.charAt(0) != '$' && path.charAt(0) != '@') { - path = "$." + path; - } - - if (path.charAt(0) == '@') { - path = "$" + path.substring(1); - } - - if (path.length() > 1 && path.charAt(1) != '.' && path.charAt(1) != '[') { - // "Invalid path " + path - return false; - } - - RootPathToken root = null; - - int i = 0; - int positions; - String fragment = ""; - - do { - char current = path.charAt(i); - - switch (current) { - case SPACE: - // "Space not allowed in path" - return false; - case DOCUMENT: - fragment = "$"; - i++; - break; - case BRACKET_OPEN: - positions = fastForwardUntilClosed(path, i); - fragment = path.substring(i, i + positions); - i += positions; - break; - case PERIOD: - i++; - if (path.charAt(i) == PERIOD) { - //This is a deep scan - fragment = ".."; - i++; - } else { - positions = fastForward(path, i); - if (positions == 0) { - continue; - - } else if (positions == 1 && path.charAt(i) == '*') { - fragment = "[*]"; - } else { - fragment = PROPERTY_OPEN + path.substring(i, i + positions) + PROPERTY_CLOSE; - } - i += positions; - } - break; - case ANY: - fragment = "[*]"; - i++; - break; - default: - positions = fastForward(path, i); - - fragment = PROPERTY_OPEN + path.substring(i, i + positions) + PROPERTY_CLOSE; - i += positions; - break; - } - - /* - * Analyze each component represented by a fragment. If there is a failure to properly evaluate, - * a null result is returned - */ - PathToken analyzedComponent = PathComponentAnalyzer.analyze(fragment, filterList); - if (analyzedComponent == null) { - return false; - } - - if (root == null) { - root = (RootPathToken) analyzedComponent; - } else { - root.append(analyzedComponent); - } - - } while (i < path.length()); - - return true; - } - - private static int fastForward(String s, int index) { - int skipCount = 0; - while (index < s.length()) { - char current = s.charAt(index); - if (current == PERIOD || current == BRACKET_OPEN || current == SPACE) { - break; - } - index++; - skipCount++; - } - return skipCount; - } - - private static int fastForwardUntilClosed(String s, int index) { - int skipCount = 0; - int nestedBrackets = 0; - - //First char is always '[' no need to check it - index++; - skipCount++; - - while (index < s.length()) { - char current = s.charAt(index); - - index++; - skipCount++; - - if (current == BRACKET_CLOSE && nestedBrackets == 0) { - break; - } - if (current == BRACKET_OPEN) { - nestedBrackets++; - } - if (current == BRACKET_CLOSE) { - nestedBrackets--; - } - } - return skipCount; - } - - static class PathComponentAnalyzer { - - private static final Pattern FILTER_PATTERN = Pattern.compile("^\\[\\s*\\?\\s*[,\\s*\\?]*?\\s*]$"); //[?] or [?, ?, ...] - private int i; - private char current; - - private final LinkedList<Predicate> filterList; - private final String pathFragment; - - PathComponentAnalyzer(String pathFragment, LinkedList<Predicate> filterList) { - this.pathFragment = pathFragment; - this.filterList = filterList; - } - - static PathToken analyze(String pathFragment, LinkedList<Predicate> filterList) { - return new PathComponentAnalyzer(pathFragment, filterList).analyze(); - } - - public PathToken analyze() { - - if ("$".equals(pathFragment)) { - return new RootPathToken(); - } else if ("..".equals(pathFragment)) { - return new ScanPathToken(); - } else if ("[*]".equals(pathFragment)) { - return new WildcardPathToken(); - } else if (".*".equals(pathFragment)) { - return new WildcardPathToken(); - } else if ("[?]".equals(pathFragment)) { - return new PredicatePathToken(filterList.poll()); - } else if (FILTER_PATTERN.matcher(pathFragment).matches()) { - final int criteriaCount = Utils.countMatches(pathFragment, "?"); - List<Predicate> filters = new ArrayList<>(criteriaCount); - for (int i = 0; i < criteriaCount; i++) { - filters.add(filterList.poll()); - } - return new PredicatePathToken(filters); - } - - this.i = 0; - do { - current = pathFragment.charAt(i); - - switch (current) { - case '?': - return analyzeCriteriaSequence4(); - case '\'': - return analyzeProperty(); - default: - if (Character.isDigit(current) || current == ':' || current == '-' || current == '@') { - return analyzeArraySequence(); - } - i++; - break; - } - - } while (i < pathFragment.length()); - - //"Could not analyze path component: " + pathFragment - return null; - } - - public PathToken analyzeCriteriaSequence4() { - int[] bounds = findFilterBounds(); - if (bounds == null) { - return null; - } - i = bounds[1]; - - return new PredicatePathToken(Filter.parse(pathFragment.substring(bounds[0], bounds[1]))); - } - - int[] findFilterBounds() { - int end = 0; - int start = i; - - while (pathFragment.charAt(start) != '[') { - start--; - } - - int mem = ' '; - int curr = start; - boolean inProp = false; - int openSquareBracket = 0; - int openBrackets = 0; - while (end == 0) { - char c = pathFragment.charAt(curr); - switch (c) { - case '(': - if (!inProp) { - openBrackets++; - } - break; - case ')': - if (!inProp) { - openBrackets--; - } - break; - case '[': - if (!inProp) { - openSquareBracket++; - } - break; - case ']': - if (!inProp) { - openSquareBracket--; - if (openBrackets == 0) { - end = curr + 1; - } - } - break; - case '\'': - if (mem == '\\') { - break; - } - inProp = !inProp; - break; - default: - break; - } - mem = c; - curr++; - } - if (openBrackets != 0 || openSquareBracket != 0) { - // "Filter brackets are not balanced" - return null; - } - return new int[]{start, end}; - } - - //"['foo']" - private PathToken analyzeProperty() { - List<String> properties = new ArrayList<>(); - StringBuilder buffer = new StringBuilder(); - - boolean propertyIsOpen = false; - - while (current != ']') { - switch (current) { - case '\'': - if (propertyIsOpen) { - properties.add(buffer.toString()); - buffer.setLength(0); - propertyIsOpen = false; - } else { - propertyIsOpen = true; - } - break; - default: - if (propertyIsOpen) { - buffer.append(current); - } - break; - } - current = pathFragment.charAt(++i); - } - return new PropertyPathToken(properties); - } - - //"[-1:]" sliceFrom - //"[:1]" sliceTo - //"[0:5]" sliceBetween - //"[1]" - //"[1,2,3]" - //"[(@.length - 1)]" - private PathToken analyzeArraySequence() { - StringBuilder buffer = new StringBuilder(); - List<Integer> numbers = new ArrayList<>(); - - boolean contextSize = (current == '@'); - boolean sliceTo = false; - boolean sliceFrom = false; - boolean sliceBetween = false; - boolean indexSequence = false; - boolean singleIndex = false; - - if (contextSize) { - - current = pathFragment.charAt(++i); - current = pathFragment.charAt(++i); - while (current != '-') { - if (current == ' ' || current == '(' || current == ')') { - current = pathFragment.charAt(++i); - continue; - } - buffer.append(current); - current = pathFragment.charAt(++i); - } - String function = buffer.toString(); - buffer.setLength(0); - if (!function.equals("size") && !function.equals("length")) { - // "Invalid function: @." + function + ". Supported functions are: [(@.length - n)] and [(@.size() - n)]" - return null; - } - while (current != ')') { - if (current == ' ') { - current = pathFragment.charAt(++i); - continue; - } - buffer.append(current); - current = pathFragment.charAt(++i); - } - - } else { - - while (Character.isDigit(current) || current == ',' || current == ' ' || current == ':' || current == '-') { - - switch (current) { - case ' ': - break; - case ':': - if (buffer.length() == 0) { - //this is a tail slice [:12] - sliceTo = true; - current = pathFragment.charAt(++i); - while (Character.isDigit(current) || current == ' ' || current == '-') { - if (current != ' ') { - buffer.append(current); - } - current = pathFragment.charAt(++i); - } - numbers.add(Integer.parseInt(buffer.toString())); - buffer.setLength(0); - } else { - //we now this starts with [12:??? - numbers.add(Integer.parseInt(buffer.toString())); - buffer.setLength(0); - current = pathFragment.charAt(++i); - - //this is a tail slice [:12] - while (Character.isDigit(current) || current == ' ' || current == '-') { - if (current != ' ') { - buffer.append(current); - } - current = pathFragment.charAt(++i); - } - - if (buffer.length() == 0) { - sliceFrom = true; - } else { - sliceBetween = true; - numbers.add(Integer.parseInt(buffer.toString())); - buffer.setLength(0); - } - } - break; - case ',': - numbers.add(Integer.parseInt(buffer.toString())); - buffer.setLength(0); - indexSequence = true; - break; - default: - buffer.append(current); - break; - } - if (current == ']') { - break; - } - current = pathFragment.charAt(++i); - } - } - if (buffer.length() > 0) { - numbers.add(Integer.parseInt(buffer.toString())); - } - singleIndex = (numbers.size() == 1) && !sliceTo && !sliceFrom && !contextSize; - - ArrayPathToken.Operation operation = null; - - if (singleIndex) { - operation = ArrayPathToken.Operation.SINGLE_INDEX; - } else if (indexSequence) { - operation = ArrayPathToken.Operation.INDEX_SEQUENCE; - } else if (sliceFrom) { - operation = ArrayPathToken.Operation.SLICE_FROM; - } else if (sliceTo) { - operation = ArrayPathToken.Operation.SLICE_TO; - } else if (sliceBetween) { - operation = ArrayPathToken.Operation.SLICE_BETWEEN; - } else if (contextSize) { - operation = ArrayPathToken.Operation.CONTEXT_SIZE; - } - - assert operation != null; - - return new ArrayPathToken(numbers, operation); - - } - } -} http://git-wip-us.apache.org/repos/asf/nifi/blob/716587d0/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java index eff8bfe..29329d3 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java @@ -33,6 +33,8 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; @@ -51,6 +53,44 @@ public class TestEvaluateJsonPath { } @Test + public void testUpgradeToJsonPath24() throws Exception { + final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath()); + testRunner.setProperty(EvaluateJsonPath.DESTINATION, EvaluateJsonPath.DESTINATION_ATTRIBUTE); + List<String> badInputs = Arrays.asList("LoremIpsum []", "LoremIpsum[]", "$..", "$.xyz."); + for (String bad : badInputs) { + testRunner.setProperty("DefinitelyNotJsonPath", bad); + + testRunner.enqueue(JSON_SNIPPET); + testRunner.assertNotValid(); + } + + /* + * Not sure why these passed with v2, but they do. These are weird strings that oddly worked before. + */ + List<String> testWhatUsedToPass = Arrays.asList("LoremIpsum@$Q#$^Q$%Q#", "TestTest['sdfadsf']#$%#$^#$^.xyz"); + for (String old : testWhatUsedToPass) { + testRunner.setProperty("DefinitelyNotJsonPath", old); + + testRunner.enqueue(JSON_SNIPPET); + testRunner.assertValid(); + } + + /* + * Test some obviously good JsonPath strings + */ + List<String> goodStrings = Arrays.asList("$", "$.xyz", "$['xyz']", "$.*['xyz']"); + for (String good : goodStrings) { + testRunner.setProperty("DefinitelyNotJsonPath", good); + + testRunner.enqueue(JSON_SNIPPET); + testRunner.assertValid(); + } + + testRunner.setProperty("DefinitelyNotJsonPath", " "); + testRunner.assertNotValid(); + } + + @Test public void testInvalidJsonDocument() throws Exception { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath()); testRunner.setProperty(EvaluateJsonPath.DESTINATION, EvaluateJsonPath.DESTINATION_ATTRIBUTE); http://git-wip-us.apache.org/repos/asf/nifi/blob/716587d0/nifi-nar-bundles/nifi-standard-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/pom.xml index 7c8e3ea..0916f75 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml @@ -265,7 +265,7 @@ <dependency> <groupId>com.jayway.jsonpath</groupId> <artifactId>json-path</artifactId> - <version>2.0.0</version> + <version>2.4.0</version> </dependency> <dependency> <groupId>org.apache.tika</groupId> http://git-wip-us.apache.org/repos/asf/nifi/blob/716587d0/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml index 1bb4878..3787dba 100755 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml @@ -49,7 +49,7 @@ <dependency> <groupId>com.jayway.jsonpath</groupId> <artifactId>json-path</artifactId> - <version>2.0.0</version> + <version>2.4.0</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId>
