This is an automated email from the ASF dual-hosted git repository.
chriss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 2ead9e5494 NIFI-10970: Added a count RecordPath function
2ead9e5494 is described below
commit 2ead9e54947a56f59faead0e8a5f324056d47171
Author: Mark Payne <marka...@hotmail.com>
AuthorDate: Mon Dec 12 14:43:41 2022 -0500
NIFI-10970: Added a count RecordPath function
Signed-off-by: Chris Sampson <chris.sampso...@gmail.com>
This closes #6778
---
.../apache/nifi/record/path/functions/Count.java | 45 +++++++++++
.../nifi/record/path/paths/RecordPathCompiler.java | 5 ++
.../apache/nifi/record/path/TestRecordPath.java | 91 +++++++++++++++-------
nifi-docs/src/main/asciidoc/record-path-guide.adoc | 31 ++++++++
4 files changed, 142 insertions(+), 30 deletions(-)
diff --git
a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Count.java
b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Count.java
new file mode 100644
index 0000000000..2ed6825345
--- /dev/null
+++
b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Count.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.functions;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.StandardFieldValue;
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+
+import java.util.stream.Stream;
+
+public class Count extends RecordPathSegment {
+ private final RecordPathSegment recordPath;
+
+ public Count(final RecordPathSegment recordPath, final boolean absolute) {
+ super("count", null, absolute);
+ this.recordPath = recordPath;
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext
context) {
+ final Stream<FieldValue> stream = recordPath.evaluate(context);
+ final long count = stream.count();
+ final RecordField recordField = new RecordField("count",
RecordFieldType.LONG.getDataType());
+ final FieldValue fieldValue = new StandardFieldValue(count,
recordField, null);
+ return Stream.of(fieldValue);
+ }
+}
diff --git
a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java
b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java
index d966382569..7ae5b45dbc 100644
---
a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java
+++
b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java
@@ -39,6 +39,7 @@ import org.apache.nifi.record.path.functions.Base64Decode;
import org.apache.nifi.record.path.functions.Base64Encode;
import org.apache.nifi.record.path.functions.Coalesce;
import org.apache.nifi.record.path.functions.Concat;
+import org.apache.nifi.record.path.functions.Count;
import org.apache.nifi.record.path.functions.EscapeJson;
import org.apache.nifi.record.path.functions.FieldName;
import org.apache.nifi.record.path.functions.FilterFunction;
@@ -377,6 +378,10 @@ public class RecordPathCompiler {
return new Coalesce(argPaths, absolute);
}
+ case "count": {
+ final RecordPathSegment[] args =
getArgPaths(argumentListTree, 1, functionName, absolute);
+ return new Count(args[0], absolute);
+ }
case "not":
case "contains":
case "containsRegex":
diff --git
a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
index db3b13b786..41596e443d 100644
---
a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
+++
b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
@@ -53,12 +53,12 @@ import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestRecordPath {
@@ -1094,23 +1094,20 @@ public class TestRecordPath {
// Special character cases
values.put("name", "John Doe");
- assertEquals("Replacing whitespace to new line",
+ assertEquals(
"John\nDoe", RecordPath.compile("replaceRegex(/name, '[\\s]',
'\\n')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
values.put("name", "John\nDoe");
- assertEquals("Replacing new line to whitespace",
- "John Doe", RecordPath.compile("replaceRegex(/name, '\\n', '
')")
+ assertEquals("John Doe", RecordPath.compile("replaceRegex(/name,
'\\n', ' ')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
values.put("name", "John Doe");
- assertEquals("Replacing whitespace to tab",
- "John\tDoe", RecordPath.compile("replaceRegex(/name, '[\\s]',
'\\t')")
+ assertEquals("John\tDoe", RecordPath.compile("replaceRegex(/name,
'[\\s]', '\\t')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
values.put("name", "John\tDoe");
- assertEquals("Replacing tab to whitespace",
- "John Doe", RecordPath.compile("replaceRegex(/name, '\\t', '
')")
+ assertEquals("John Doe", RecordPath.compile("replaceRegex(/name,
'\\t', ' ')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
}
@@ -1132,23 +1129,19 @@ public class TestRecordPath {
// NOTE: At Java code, a single back-slash needs to be escaped with
another-back slash, but needn't to do so at NiFi UI.
// The test record path is equivalent to replaceRegex(/name,
'\'', '"')
values.put("name", "'John' 'Doe'");
- assertEquals("Replacing quote to double-quote",
- "\"John\" \"Doe\"", RecordPath.compile("replaceRegex(/name,
'\\'', '\"')")
+ assertEquals("\"John\" \"Doe\"",
RecordPath.compile("replaceRegex(/name, '\\'', '\"')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
values.put("name", "\"John\" \"Doe\"");
- assertEquals("Replacing double-quote to single-quote",
- "'John' 'Doe'", RecordPath.compile("replaceRegex(/name, '\"',
'\\'')")
+ assertEquals("'John' 'Doe'", RecordPath.compile("replaceRegex(/name,
'\"', '\\'')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
values.put("name", "'John' 'Doe'");
- assertEquals("Replacing quote to double-quote, the function arguments
are wrapped by double-quote",
- "\"John\" \"Doe\"", RecordPath.compile("replaceRegex(/name,
\"'\", \"\\\"\")")
+ assertEquals("\"John\" \"Doe\"",
RecordPath.compile("replaceRegex(/name, \"'\", \"\\\"\")")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
values.put("name", "\"John\" \"Doe\"");
- assertEquals("Replacing double-quote to single-quote, the function
arguments are wrapped by double-quote",
- "'John' 'Doe'", RecordPath.compile("replaceRegex(/name,
\"\\\"\", \"'\")")
+ assertEquals("'John' 'Doe'", RecordPath.compile("replaceRegex(/name,
\"\\\"\", \"'\")")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
}
@@ -1170,13 +1163,11 @@ public class TestRecordPath {
// NOTE: At Java code, a single back-slash needs to be escaped with
another-back slash, but needn't to do so at NiFi UI.
// The test record path is equivalent to replaceRegex(/name,
'\\', '/')
values.put("name", "John\\Doe");
- assertEquals("Replacing a back-slash to forward-slash",
- "John/Doe", RecordPath.compile("replaceRegex(/name, '\\\\',
'/')")
+ assertEquals("John/Doe", RecordPath.compile("replaceRegex(/name,
'\\\\', '/')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
values.put("name", "John/Doe");
- assertEquals("Replacing a forward-slash to back-slash",
- "John\\Doe", RecordPath.compile("replaceRegex(/name, '/',
'\\\\')")
+ assertEquals("John\\Doe", RecordPath.compile("replaceRegex(/name, '/',
'\\\\')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
}
@@ -1196,13 +1187,11 @@ public class TestRecordPath {
// Brackets
values.put("name", "J[o]hn Do[e]");
- assertEquals("Square brackets can be escaped with back-slash",
- "J(o)hn Do(e)",
RecordPath.compile("replaceRegex(replaceRegex(/name, '\\[', '('), '\\]', ')')")
+ assertEquals("J(o)hn Do(e)",
RecordPath.compile("replaceRegex(replaceRegex(/name, '\\[', '('), '\\]', ')')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
values.put("name", "J(o)hn Do(e)");
- assertEquals("Brackets can be escaped with back-slash",
- "J[o]hn Do[e]",
RecordPath.compile("replaceRegex(replaceRegex(/name, '\\(', '['), '\\)', ']')")
+ assertEquals("J[o]hn Do[e]",
RecordPath.compile("replaceRegex(replaceRegex(/name, '\\(', '['), '\\)', ']')")
.evaluate(record).getSelectedFields().findFirst().get().getValue());
}
@@ -1870,7 +1859,7 @@ public class TestRecordPath {
assertEquals("MyString", RecordPath.compile("padRight(/someString,
3)").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertEquals("MyString", RecordPath.compile("padRight(/someString,
-10)").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertEquals("@@@@@@@@@@", RecordPath.compile("padRight(/emptyString,
10, '@')").evaluate(record).getSelectedFields().findFirst().get().getValue());
- assertNull(null, RecordPath.compile("padRight(/nullString, 10,
'@')").evaluate(record).getSelectedFields().findFirst().get().getValue());
+ assertNull(RecordPath.compile("padRight(/nullString, 10,
'@')").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertEquals("MyStringxy", RecordPath.compile("padRight(/someString,
10,
\"xy\")").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertEquals("MyStringaV", RecordPath.compile("padRight(/someString,
10,
\"aVeryLongPadding\")").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertEquals("MyStringfewfewfewfew",
RecordPath.compile("padRight(/someString, 20,
\"few\")").evaluate(record).getSelectedFields().findFirst().get().getValue());
@@ -1940,6 +1929,48 @@ public class TestRecordPath {
assertEquals(Boolean.FALSE, RecordPath.compile("not(/id =
48)").evaluate(record).getSelectedFields().findFirst().get().getValue());
}
+ @Test
+ public void testCountArrayElements() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("numbers", new Object[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues =
RecordPath.compile("count(/numbers[*])").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+ assertEquals(10L, fieldValues.get(0).getValue());
+ }
+
+ @Test
+ public void testCountComparison() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("numbers", new Object[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues =
RecordPath.compile("count(/numbers[*]) >
9").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+ assertEquals(true, fieldValues.get(0).getValue());
+ }
+
+ @Test
+ public void testCountAsFilter() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("numbers", new Object[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues =
RecordPath.compile("/id[count(/numbers[*]) >
2]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+ assertEquals(48, fieldValues.get(0).getValue());
+ }
+
private List<RecordField> getDefaultFields() {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
diff --git a/nifi-docs/src/main/asciidoc/record-path-guide.adoc
b/nifi-docs/src/main/asciidoc/record-path-guide.adoc
index b59f78ade6..17fd585fba 100644
--- a/nifi-docs/src/main/asciidoc/record-path-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/record-path-guide.adoc
@@ -1071,6 +1071,37 @@ take the value of the supplied record path and use it as
the namespace.
Please note that the namespace must always be a valid UUID string. An empty
string, another data type, etc. will result
in an error. This is by design because the most common use case for UUID v5 is
to uniquely identify records across data sets.
+
+=== count
+
+Returns the count of the number of elements. This is commonly used in
conjunction with arrays. For example, if we have the following record:
+
+----
+{
+ "id": "1234",
+ "elements": [{
+ "name": "book",
+ "color": "red"
+ }, {
+ "name": "computer",
+ "color": "black"
+ }]
+}
+----
+
+We could determine the number of `elements` by using the `count` function.
Using:
+
+----
+count(/elements[*])
+----
+
+Would yield a value of `2`. We could also use this as a filter, such as:
+----
+/id[ count(/elements[*]) = 2 ]
+----
+Which will return the `id` element with a value of `1234`.
+
+
[[filter_functions]]
== Filter Functions
