This is an automated email from the ASF dual-hosted git repository. ijokarumawak pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push: new 6a06cd3 NIFI-6304 added trim, toLowerCase and toUpperCase to record path operations. 6a06cd3 is described below commit 6a06cd3094eccaa264d575906106504092f95cd6 Author: Mike Thomsen <mikerthom...@gmail.com> AuthorDate: Thu May 16 21:11:05 2019 -0400 NIFI-6304 added trim, toLowerCase and toUpperCase to record path operations. NIFI-6304 Updated code based on code review. NIFI-6304 Updated documentation. NIFI-6304 Refactored to make it simpler NIFI-6304 Reverted Concat to its last state. This closes #3478. Signed-off-by: Koji Kawamura <ijokaruma...@apache.org> --- .../apache/nifi/record/path/functions/Concat.java | 2 +- .../{Concat.java => NoArgStringFunction.java} | 46 +++++------ .../nifi/record/path/functions/ToLowerCase.java | 30 ++++++++ .../nifi/record/path/functions/ToUpperCase.java | 30 ++++++++ .../nifi/record/path/functions/TrimString.java | 30 ++++++++ .../nifi/record/path/paths/RecordPathCompiler.java | 26 +++++++ .../apache/nifi/record/path/TestRecordPath.java | 90 ++++++++++++++++++---- nifi-docs/src/main/asciidoc/record-path-guide.adoc | 90 +++++++++++++++++++++- 8 files changed, 306 insertions(+), 38 deletions(-) diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Concat.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Concat.java index daeee05..3b8a3cd 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Concat.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Concat.java @@ -52,4 +52,4 @@ public class Concat extends RecordPathSegment { return Stream.of(responseValue); } -} +} \ No newline at end of file diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Concat.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/NoArgStringFunction.java similarity index 51% copy from nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Concat.java copy to nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/NoArgStringFunction.java index daeee05..8cb801e 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Concat.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/NoArgStringFunction.java @@ -17,39 +17,39 @@ package org.apache.nifi.record.path.functions; -import java.util.stream.Stream; - import org.apache.nifi.record.path.FieldValue; import org.apache.nifi.record.path.RecordPathEvaluationContext; import org.apache.nifi.record.path.StandardFieldValue; import org.apache.nifi.record.path.paths.RecordPathSegment; -import org.apache.nifi.serialization.record.RecordField; -import org.apache.nifi.serialization.record.RecordFieldType; import org.apache.nifi.serialization.record.util.DataTypeUtils; -public class Concat extends RecordPathSegment { - private final RecordPathSegment[] valuePaths; +import java.util.stream.Stream; + +/** + * Abstract class for String functions without any argument. + */ +public abstract class NoArgStringFunction extends RecordPathSegment { + private final RecordPathSegment valuePath; - public Concat(final RecordPathSegment[] valuePaths, final boolean absolute) { - super("concat", null, absolute); - this.valuePaths = valuePaths; + public NoArgStringFunction(final String path, final RecordPathSegment valuePath, final boolean absolute) { + super(path, null, absolute); + this.valuePath = valuePath; } @Override - public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { - Stream<FieldValue> concatenated = Stream.empty(); - - for (final RecordPathSegment valuePath : valuePaths) { - final Stream<FieldValue> stream = valuePath.evaluate(context); - concatenated = Stream.concat(concatenated, stream); - } - - final StringBuilder sb = new StringBuilder(); - concatenated.forEach(fv -> sb.append(DataTypeUtils.toString(fv.getValue(), (String) null))); - - final RecordField field = new RecordField("concat", RecordFieldType.STRING.getDataType()); - final FieldValue responseValue = new StandardFieldValue(sb.toString(), field, null); - return Stream.of(responseValue); + public Stream<FieldValue> evaluate(RecordPathEvaluationContext context) { + return valuePath.evaluate(context).map(fv -> { + final String original = fv.getValue() == null ? "" : DataTypeUtils.toString(fv.getValue(), (String) null); + final String processed = apply(original); + return new StandardFieldValue(processed, fv.getField(), fv.getParent().orElse(null)); + }); } + /** + * Sub-classes apply its function to the given value and return the result. + * @param value possibly null + * @return the function result + */ + abstract String apply(String value); + } diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToLowerCase.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToLowerCase.java new file mode 100644 index 0000000..07a6fe5 --- /dev/null +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToLowerCase.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.record.path.functions; + +import org.apache.nifi.record.path.paths.RecordPathSegment; + +public class ToLowerCase extends NoArgStringFunction { + public ToLowerCase(RecordPathSegment valuePath, boolean absolute) { + super("toLowerCase", valuePath, absolute); + } + + @Override + String apply(String value) { + return value == null ? null : value.toLowerCase(); + } +} diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToUpperCase.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToUpperCase.java new file mode 100644 index 0000000..59cc302 --- /dev/null +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToUpperCase.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.record.path.functions; + +import org.apache.nifi.record.path.paths.RecordPathSegment; + +public class ToUpperCase extends NoArgStringFunction { + public ToUpperCase(final RecordPathSegment valuePath, final boolean absolute) { + super("toUpperCase", valuePath, absolute); + } + + @Override + String apply(String value) { + return value == null ? null : value.toUpperCase(); + } +} diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/TrimString.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/TrimString.java new file mode 100644 index 0000000..5ea26b7 --- /dev/null +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/TrimString.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.record.path.functions; + +import org.apache.nifi.record.path.paths.RecordPathSegment; + +public class TrimString extends NoArgStringFunction { + public TrimString(RecordPathSegment valuePath, boolean absolute) { + super("trim", valuePath, absolute); + } + + @Override + String apply(String value) { + return value == null ? null : value.trim(); + } +} diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java index 482c651..11ff55d 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java @@ -79,7 +79,10 @@ import org.apache.nifi.record.path.functions.SubstringBefore; import org.apache.nifi.record.path.functions.SubstringBeforeLast; import org.apache.nifi.record.path.functions.ToBytes; import org.apache.nifi.record.path.functions.ToDate; +import org.apache.nifi.record.path.functions.ToLowerCase; import org.apache.nifi.record.path.functions.ToString; +import org.apache.nifi.record.path.functions.ToUpperCase; +import org.apache.nifi.record.path.functions.TrimString; public class RecordPathCompiler { @@ -246,6 +249,18 @@ public class RecordPathCompiler { return new Concat(argPaths, absolute); } + case "toLowerCase": { + final RecordPathSegment[] args = getArgPaths(argumentListTree, 1, functionName, absolute); + return new ToLowerCase(args[0], absolute); + } + case "toUpperCase": { + final RecordPathSegment[] args = getArgPaths(argumentListTree, 1, functionName, absolute); + return new ToUpperCase(args[0], absolute); + } + case "trim": { + final RecordPathSegment[] args = getArgPaths(argumentListTree, 1, functionName, absolute); + return new TrimString(args[0], absolute); + } case "fieldName": { final RecordPathSegment[] args = getArgPaths(argumentListTree, 1, functionName, absolute); return new FieldName(args[0], absolute); @@ -299,6 +314,17 @@ public class RecordPathCompiler { throw new RecordPathException("Encountered unexpected token " + tree); } + private static RecordPathSegment[] getArgumentsForStringFunction(boolean absolute, Tree argumentListTree) { + final int numArgs = argumentListTree.getChildCount(); + + final RecordPathSegment[] argPaths = new RecordPathSegment[numArgs]; + for (int i = 0; i < numArgs; i++) { + argPaths[i] = buildPath(argumentListTree.getChild(i), null, absolute); + } + + return argPaths; + } + private static RecordPathFilter createFilter(final Tree operatorTree, final RecordPathSegment parent, final boolean absolute) { switch (operatorTree.getType()) { case EQUAL: diff --git a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java index 8714b33..2f4abd9 100644 --- a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java +++ b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java @@ -17,9 +17,17 @@ package org.apache.nifi.record.path; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.nifi.record.path.exception.RecordPathException; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.DataType; +import org.apache.nifi.serialization.record.MapRecord; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.serialization.record.type.ArrayDataType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; +import org.junit.Test; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.StandardCharsets; @@ -35,16 +43,10 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.apache.nifi.record.path.exception.RecordPathException; -import org.apache.nifi.serialization.SimpleRecordSchema; -import org.apache.nifi.serialization.record.DataType; -import org.apache.nifi.serialization.record.MapRecord; -import org.apache.nifi.serialization.record.Record; -import org.apache.nifi.serialization.record.RecordField; -import org.apache.nifi.serialization.record.RecordFieldType; -import org.apache.nifi.serialization.record.RecordSchema; -import org.apache.nifi.serialization.record.util.DataTypeUtils; -import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class TestRecordPath { @@ -1210,6 +1212,68 @@ public class TestRecordPath { assertEquals("John Doe: 48", RecordPath.compile("concat(/firstName, ' ', /lastName, ': ', 48)").evaluate(record).getSelectedFields().findFirst().get().getValue()); } + private Record getCaseTestRecord() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("middleName", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("lastName", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("firstName", RecordFieldType.STRING.getDataType())); + + final RecordSchema schema = new SimpleRecordSchema(fields); + + final Map<String, Object> values = new HashMap<>(); + values.put("lastName", "Doe"); + values.put("firstName", "John"); + values.put("middleName", "Smith"); + return new MapRecord(schema, values); + } + + @Test + public void testToUpperCase() { + final Record record = getCaseTestRecord(); + + assertEquals("JOHN SMITH DOE", RecordPath.compile("toUpperCase(concat(/firstName, ' ', /middleName, ' ', /lastName))").evaluate(record).getSelectedFields().findFirst().get().getValue()); + assertEquals("", RecordPath.compile("toLowerCase(/notDefined)").evaluate(record).getSelectedFields().findFirst().get().getValue()); + } + + @Test + public void testToLowerCase() { + final Record record = getCaseTestRecord(); + + assertEquals("john smith doe", RecordPath.compile("toLowerCase(concat(/firstName, ' ', /middleName, ' ', /lastName))").evaluate(record).getSelectedFields().findFirst().get().getValue()); + assertEquals("", RecordPath.compile("toLowerCase(/notDefined)").evaluate(record).getSelectedFields().findFirst().get().getValue()); + } + + @Test + public void testTrimString() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("fullName", RecordFieldType.STRING.getDataType())); + + final RecordSchema schema = new SimpleRecordSchema(fields); + + final Map<String, Object> values = new HashMap<>(); + values.put("fullName", " John Smith "); + final Record record = new MapRecord(schema, values); + + assertEquals("John Smith", RecordPath.compile("trim(/fullName)").evaluate(record).getSelectedFields().findFirst().get().getValue()); + assertEquals("", RecordPath.compile("trim(/missing)").evaluate(record).getSelectedFields().findFirst().get().getValue()); + } + + @Test + public void testTrimArray() { + final List<RecordField> fields = new ArrayList<>(); + final DataType dataType = new ArrayDataType(RecordFieldType.STRING.getDataType()); + fields.add(new RecordField("names", dataType)); + + final RecordSchema schema = new SimpleRecordSchema(fields); + + final Map<String, Object> values = new HashMap<>(); + values.put("names", new String[]{" John Smith ", " Jane Smith "}); + final Record record = new MapRecord(schema, values); + + final List<FieldValue> results = RecordPath.compile("trim(/names[*])").evaluate(record).getSelectedFields().collect(Collectors.toList()); + assertEquals("John Smith", results.get(0).getValue()); + assertEquals("Jane Smith", results.get(1).getValue()); + } @Test public void testFieldName() { final List<RecordField> fields = new ArrayList<>(); diff --git a/nifi-docs/src/main/asciidoc/record-path-guide.adoc b/nifi-docs/src/main/asciidoc/record-path-guide.adoc index daa19a0..976a35e 100644 --- a/nifi-docs/src/main/asciidoc/record-path-guide.adoc +++ b/nifi-docs/src/main/asciidoc/record-path-guide.adoc @@ -3,7 +3,7 @@ // contributor license agreements. See the NOTICE file distributed with // this work for additional information regarding copyright ownership. // The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with +// (the "License"); you may not use this file except ixn compliance with // the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 @@ -632,6 +632,94 @@ The following record path expression would re-format the date String: | `format( toDate(/eventDate, "yyyy-MM-dd'T'HH:mm:ss'Z'"), 'yyyy-MM-dd')` | 2017-10-20 |========================================================== +=== trim + +Removes whitespace from the start and end of a string. + +---- +{ + "type": "record", + "name": "events", + "fields": [ + { "name": "name", "type": "string" } + ] +} +---- + +and a record such as: + +---- +{ + "name" : " John Smith " +} +---- + +The following record path expression would remove extraneous whitespace: + +|========================================================== +| RecordPath | Return value +| `trim(/name)` | John Smith +|========================================================== + + +=== toUpperCase + +Change the entire String to upper case + +---- +{ + "type": "record", + "name": "events", + "fields": [ + { "name": "fullName", "type": "string" } + ] +} +---- + +and a record such as: + +---- +{ + "fullName" : "john smith" +} +---- + +The following record path expression would remove extraneous whitespace: + +|========================================================== +| RecordPath | Return value +| `toUpperCase(/name)` | JOHN SMITH +|========================================================== + +=== toLowerCase + +Changes the entire string to lower case. + +---- +{ + "type": "record", + "name": "events", + "fields": [ + { "name": "message", "type": "string" } + ] +} +---- + +and a record such as: + +---- +{ + "name" : "hEllO wORLd" +} +---- + +The following record path expression would remove extraneous whitespace: + +|========================================================== +| RecordPath | Return value +| `trim(/message)` | hello world +|========================================================== + === base64Encode Converts a String or byte[] using Base64 encoding, using the UTF-8 character set. For example, given a schema such as: