(nifi) branch main updated: NIFI-9677 Fixed issue that an empty JSON array causes flow file to be considered unmatched even though it should be considered as a match.
This is an automated email from the ASF dual-hosted git repository.
mattyb149 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 59ff1b6561 NIFI-9677 Fixed issue that an empty JSON array causes flow
file to be considered unmatched even though it should be considered as a match.
59ff1b6561 is described below
commit 59ff1b6561863c500283d665e9675979b065e78b
Author: jsteinebrey
AuthorDate: Thu Jan 11 15:21:14 2024 -0600
NIFI-9677 Fixed issue that an empty JSON array causes flow file to be
considered unmatched even though it should be considered as a match.
Refactored to avoid streaming over the list twice.
Tweaked unit test, so it works in NiFi 1.x also.
Signed-off-by: Matt Burgess
This closes #8266
---
.../nifi/processors/standard/LookupRecord.java | 15 +++-
.../nifi/processors/standard/TestLookupRecord.java | 94 ++
.../lookup-array-input-empty-array.json| 40 +
.../lookup-array-input-missing.json| 14
.../lookup-array-output-empty-array.json | 1 +
.../lookup-array-output-missing-matched.json | 1 +
.../lookup-array-output-missing-unmatched.json | 1 +
7 files changed, 164 insertions(+), 2 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LookupRecord.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LookupRecord.java
index c50db9f2ce..0517f3d800 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LookupRecord.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LookupRecord.java
@@ -80,6 +80,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -487,9 +488,19 @@ public class LookupRecord extends AbstractProcessor {
final RecordPath recordPath = entry.getValue();
final RecordPathResult pathResult =
recordPath.evaluate(record);
+AtomicLong selectedFieldsCount = new AtomicLong(0);
final List lookupFieldValues =
pathResult.getSelectedFields()
-.filter(fieldVal -> fieldVal.getValue() != null)
-.collect(Collectors.toList());
+.filter(fieldVal -> {
+selectedFieldsCount.incrementAndGet();
+return fieldVal.getValue() != null;
+})
+.collect(Collectors.toList());
+
+if (selectedFieldsCount.get() == 0) {
+// When selectedFieldsCount == 0; then an empty array was
found which counts as a match.
+// Since the array is empty, no further processing is
needed, so continue to next recordPath.
+continue;
+}
if (lookupFieldValues.isEmpty()) {
final Set rels = routeToMatchedUnmatched ?
UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestLookupRecord.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestLookupRecord.java
index 66b498be46..f52b799383 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestLookupRecord.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestLookupRecord.java
@@ -527,6 +527,100 @@ public class TestLookupRecord {
out.assertContentEquals(new
File("src/test/resources/TestLookupRecord/lookup-array-output.json").toPath());
}
+@Test
+public void testLookupEmptyArray() throws InitializationException,
IOException {
+TestRunner runner = TestRunners.newTestRunner(LookupRecord.class);
+final MapLookup lookupService = new MapLookupForInPlaceReplacement();
+
+final JsonTreeReader jsonReader = new JsonTreeReader();
+runner.addControllerService("reader", jsonReader);
+runner.setProperty(jsonReader,
SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaInferenceUtil.INFER_SCHEMA);
+
+final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
+runner.addControllerService("writer", jsonWriter);
+runner.setProperty(jsonWriter,
SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY,
SchemaAccessUtils.INHERIT_RECORD_SCHEMA);
+
+runner.addC
(nifi) branch support/nifi-1.x updated: NIFI-9677 Fixed issue that an empty JSON array causes flow file to be considered unmatched even though it should be considered as a match.
This is an automated email from the ASF dual-hosted git repository.
mattyb149 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 4c1edbefa3 NIFI-9677 Fixed issue that an empty JSON array causes flow
file to be considered unmatched even though it should be considered as a match.
4c1edbefa3 is described below
commit 4c1edbefa36cda8ad19d4d1956a678558d50eca8
Author: jsteinebrey
AuthorDate: Thu Jan 11 15:21:14 2024 -0600
NIFI-9677 Fixed issue that an empty JSON array causes flow file to be
considered unmatched even though it should be considered as a match.
Refactored to avoid streaming over the list twice.
Tweaked unit test, so it works in NiFi 1.x also.
Signed-off-by: Matt Burgess
---
.../nifi/processors/standard/LookupRecord.java | 15 +++-
.../nifi/processors/standard/TestLookupRecord.java | 94 ++
.../lookup-array-input-empty-array.json| 40 +
.../lookup-array-input-missing.json| 14
.../lookup-array-output-empty-array.json | 1 +
.../lookup-array-output-missing-matched.json | 1 +
.../lookup-array-output-missing-unmatched.json | 1 +
7 files changed, 164 insertions(+), 2 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LookupRecord.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LookupRecord.java
index 7e4285ad2f..1340182d27 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LookupRecord.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LookupRecord.java
@@ -81,6 +81,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -489,9 +490,19 @@ public class LookupRecord extends AbstractProcessor {
final RecordPath recordPath = entry.getValue();
final RecordPathResult pathResult =
recordPath.evaluate(record);
+AtomicLong selectedFieldsCount = new AtomicLong(0);
final List lookupFieldValues =
pathResult.getSelectedFields()
-.filter(fieldVal -> fieldVal.getValue() != null)
-.collect(Collectors.toList());
+.filter(fieldVal -> {
+selectedFieldsCount.incrementAndGet();
+return fieldVal.getValue() != null;
+})
+.collect(Collectors.toList());
+
+if (selectedFieldsCount.get() == 0) {
+// When selectedFieldsCount == 0; then an empty array was
found which counts as a match.
+// Since the array is empty, no further processing is
needed, so continue to next recordPath.
+continue;
+}
if (lookupFieldValues.isEmpty()) {
final Set rels = routeToMatchedUnmatched ?
UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestLookupRecord.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestLookupRecord.java
index 66b498be46..f52b799383 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestLookupRecord.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestLookupRecord.java
@@ -527,6 +527,100 @@ public class TestLookupRecord {
out.assertContentEquals(new
File("src/test/resources/TestLookupRecord/lookup-array-output.json").toPath());
}
+@Test
+public void testLookupEmptyArray() throws InitializationException,
IOException {
+TestRunner runner = TestRunners.newTestRunner(LookupRecord.class);
+final MapLookup lookupService = new MapLookupForInPlaceReplacement();
+
+final JsonTreeReader jsonReader = new JsonTreeReader();
+runner.addControllerService("reader", jsonReader);
+runner.setProperty(jsonReader,
SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaInferenceUtil.INFER_SCHEMA);
+
+final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
+runner.addControllerService("writer", jsonWriter);
+runner.setProperty(jsonWriter,
SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY,
SchemaAccessUtils.INHERIT_RECORD_SCHEMA);
+
+runner.addCont
(nifi) branch main updated: NIFI-12643 Added support for FileResourceService in PutGCSObject
This is an automated email from the ASF dual-hosted git repository.
turcsanyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new bce14f573b NIFI-12643 Added support for FileResourceService in
PutGCSObject
bce14f573b is described below
commit bce14f573b57685637d776333029641e62730d26
Author: Balázs Gerner
AuthorDate: Fri Jan 19 09:49:30 2024 +0100
NIFI-12643 Added support for FileResourceService in PutGCSObject
This closes #8281.
Signed-off-by: Peter Turcsanyi
---
.../nifi-gcp-bundle/nifi-gcp-processors/pom.xml| 5 +
.../nifi/processors/gcp/storage/PutGCSObject.java | 375 +++--
.../processors/gcp/storage/PutGCSObjectTest.java | 55 ++-
3 files changed, 242 insertions(+), 193 deletions(-)
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml
b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml
index 70f104365e..897e710395 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml
@@ -69,6 +69,11 @@
2.0.0-SNAPSHOT
provided
+
+org.apache.nifi
+nifi-resource-transfer
+2.0.0-SNAPSHOT
+
org.apache.nifi
nifi-file-resource-service-api
diff --git
a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/PutGCSObject.java
b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/PutGCSObject.java
index 07546637da..438fac19b8 100644
---
a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/PutGCSObject.java
+++
b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/PutGCSObject.java
@@ -22,15 +22,6 @@ import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageException;
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.ReadsAttribute;
@@ -43,13 +34,23 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.fileresource.service.api.FileResource;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.transfer.ResourceTransferSource;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
import static
com.google.cloud.storage.Storage.PredefinedAcl.ALL_AUTHENTICATED_USERS;
import static
com.google.cloud.storage.Storage.PredefinedAcl.AUTHENTICATED_READ;
@@ -102,6 +103,9 @@ import static
org.apache.nifi.processors.gcp.storage.StorageAttributes.UPDATE_TI
import static
org.apache.nifi.processors.gcp.storage.StorageAttributes.UPDATE_TIME_DESC;
import static
org.apache.nifi.processors.gcp.storage.StorageAttributes.URI_ATTR;
import static
org.apache.nifi.processors.gcp.storage.StorageAttributes.URI_DESC;
+import static
org.apache.nifi.processors.transfer.ResourceTransferProperties.FILE_RESOURCE_SERVICE;
+import static
org.apache.nifi.processors.transfer.ResourceTransferProperties.RESOURCE_TRANSFER_SOURCE;
+import static
org.apache.nifi.processors.transfer.ResourceTransferUtils.getFileResource;
@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
@@ -290,6 +294,8 @@ public class PutGCSObject extends AbstractGCSProcessor {
final List descriptors = new
ArrayList<>(super.getSupportedPropertyDescriptors());
descriptors.add(BUCKET);
descriptors.add(KEY);
+descriptors.add(RESOURCE_TRANSFER_SOURCE);
+descriptors.add(FILE_RESOURCE_SERVICE);
descriptors.add(CONTENT_TYPE);
descriptors.add(CRC32C);
descriptors.add(ACL);
@@ -322,199
(nifi-minifi-cpp) branch MINIFICPP-2276 created (now ee23065b9)
This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a change to branch MINIFICPP-2276 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git at ee23065b9 Review update No new revisions were added by this update.
