(nifi) branch main updated: NIFI-8134 allow unescapeJson Record Path function to recursively convert Maps to Records (#7745)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 2d112871db NIFI-8134 allow unescapeJson Record Path function to
recursively convert Maps to Records (#7745)
2d112871db is described below
commit 2d112871db40c8f599d31974d14dd4a2227bf7da
Author: Chris Sampson <12159006+chrissamo...@users.noreply.github.com>
AuthorDate: Tue May 14 22:11:28 2024 +0100
NIFI-8134 allow unescapeJson Record Path function to recursively convert
Maps to Records (#7745)
* NIFI-8134 recursively convert Java Objects to NiFi Records
---
.../nifi/record/path/functions/UnescapeJson.java | 17 +-
.../nifi/record/path/paths/RecordPathCompiler.java | 12 +-
.../apache/nifi/record/path/TestRecordPath.java| 183 ++---
.../serialization/record/util/DataTypeUtils.java | 73 ++--
nifi-docs/src/main/asciidoc/record-path-guide.adoc | 30 +++-
.../nifi-standard-processors/pom.xml | 4 +
.../nifi/processors/standard/TestUpdateRecord.java | 64 +++
.../TestUpdateRecord/input/organisation.json | 3 +
.../TestUpdateRecord/output/organisation.json | 15 ++
.../organisation-with-departments-string.avsc | 8 +
.../schema/organisation-with-departments.avsc | 78 +
11 files changed, 400 insertions(+), 87 deletions(-)
diff --git
a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/UnescapeJson.java
b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/UnescapeJson.java
index 35f7d93d3f..d5e821a44b 100644
---
a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/UnescapeJson.java
+++
b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/UnescapeJson.java
@@ -44,18 +44,23 @@ public class UnescapeJson extends RecordPathSegment {
private final RecordPathSegment convertToRecordRecordPath;
+private final RecordPathSegment recursiveRecordConversion;
+
private final ObjectMapper objectMapper = new ObjectMapper();
-public UnescapeJson(final RecordPathSegment recordPath, final
RecordPathSegment convertToRecordRecordPath, final boolean absolute) {
+public UnescapeJson(final RecordPathSegment recordPath, final
RecordPathSegment convertToRecordRecordPath, final RecordPathSegment
recursiveRecordConversion, final boolean absolute) {
super("unescapeJson", null, absolute);
this.recordPath = recordPath;
this.convertToRecordRecordPath = convertToRecordRecordPath;
+this.recursiveRecordConversion = recursiveRecordConversion;
}
@Override
public Stream evaluate(final RecordPathEvaluationContext
context) {
final boolean convertMapToRecord = convertToRecordRecordPath != null
&&
Boolean.parseBoolean(RecordPathUtils.getFirstStringValue(convertToRecordRecordPath,
context));
+final boolean recursiveMapToRecord = recursiveRecordConversion != null
+&&
Boolean.parseBoolean(RecordPathUtils.getFirstStringValue(recursiveRecordConversion,
context));
final Stream fieldValues = recordPath.evaluate(context);
return fieldValues.filter(fv -> fv.getValue() != null)
@@ -70,7 +75,7 @@ public class UnescapeJson extends RecordPathSegment {
}
return new StandardFieldValue(
-convertFieldValue(value,
fv.getField().getFieldName(), dataType, convertMapToRecord),
+convertFieldValue(value,
fv.getField().getFieldName(), dataType, convertMapToRecord,
recursiveMapToRecord),
fv.getField(), fv.getParent().orElse(null)
);
} catch (IOException e) {
@@ -83,7 +88,7 @@ public class UnescapeJson extends RecordPathSegment {
}
@SuppressWarnings("unchecked")
-private Object convertFieldValue(final Object value, final String
fieldName, final DataType dataType, final boolean convertMapToRecord) throws
IOException {
+private Object convertFieldValue(final Object value, final String
fieldName, final DataType dataType, final boolean convertMapToRecord, final
boolean recursiveMapToRecord) throws IOException {
if (dataType instanceof RecordDataType) {
// convert Maps to Records
final Map map =
objectMapper.readValue(value.toString(), Map.class);
@@ -102,13 +107,13 @@ public class UnescapeJson extends RecordPathSegment {
final Object parsed = objectMapper.readValue(value.toString(),
Object.class);
if (convertMapToRecord) {
if (DataTypeUtils.isCompatibleDataTyp
(nifi) branch main updated (326df914bc -> 5a3b47353e)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 326df914bc NIFI-13175: Updating dialog layout and sizes to prevent a double scroll bar on smaller screen sizes (#8779) add 5a3b47353e NIFI-13146 ConsumeSlack rate limit error mitigation (#8748) No new revisions were added by this update. Summary of changes: .../apache/nifi/processors/slack/ConsumeSlack.java | 90 +++--- .../slack/consume/ConsumeSlackClient.java | 2 + .../nifi/processors/slack/TestConsumeSlack.java| 17 +++- 3 files changed, 79 insertions(+), 30 deletions(-)
(nifi) branch main updated: NIFI-13076: reduce enum array allocation (#8679)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 31e1ce8f4c NIFI-13076: reduce enum array allocation (#8679)
31e1ce8f4c is described below
commit 31e1ce8f4cec10e291adeb31116194e6f445b751
Author: sullis
AuthorDate: Mon Apr 22 10:17:15 2024 -0500
NIFI-13076: reduce enum array allocation (#8679)
---
.../processors/opentelemetry/server/HttpRequestHandler.java| 10 +++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-opentelemetry-bundle/nifi-opentelemetry-processors/src/main/java/org/apache/nifi/processors/opentelemetry/server/HttpRequestHandler.java
b/nifi-nar-bundles/nifi-opentelemetry-bundle/nifi-opentelemetry-processors/src/main/java/org/apache/nifi/processors/opentelemetry/server/HttpRequestHandler.java
index ef0efd3bc2..740df36af3 100644
---
a/nifi-nar-bundles/nifi-opentelemetry-bundle/nifi-opentelemetry-processors/src/main/java/org/apache/nifi/processors/opentelemetry/server/HttpRequestHandler.java
+++
b/nifi-nar-bundles/nifi-opentelemetry-bundle/nifi-opentelemetry-processors/src/main/java/org/apache/nifi/processors/opentelemetry/server/HttpRequestHandler.java
@@ -55,6 +55,10 @@ import java.util.Objects;
* HTTP Handler for OTLP Export Service Requests over gGRPC or encoded as JSON
or Protobuf over HTTP
*/
public class HttpRequestHandler extends
SimpleChannelInboundHandler {
+private static final TelemetryContentEncoding[]
TELEMETRY_CONTENT_ENCODING_VALUES = TelemetryContentEncoding.values();
+private static final TelemetryRequestType[] TELEMETRY_REQUEST_TYPE_VALUES
= TelemetryRequestType.values();
+private static final TelemetryContentType[] TELEMETRY_CONTENT_TYPE_VALUES
= TelemetryContentType.values();
+
private final ResponseBodyWriter responseBodyWriter = new
StandardResponseBodyWriter();
private final ComponentLog log;
@@ -138,7 +142,7 @@ public class HttpRequestHandler extends
SimpleChannelInboundHandler
(nifi) branch main updated: NIFI-11443 Route Python Framework Logging to SLF4J (#8407)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 26f5fa2be0 NIFI-11443 Route Python Framework Logging to SLF4J (#8407)
26f5fa2be0 is described below
commit 26f5fa2be044accc6ae8403daf62be621e47f33e
Author: David Handermann
AuthorDate: Tue Mar 5 15:55:04 2024 -0600
NIFI-11443 Route Python Framework Logging to SLF4J (#8407)
* NIFI-11443 Routed Python Framework Logging to SLF4J
- Changed Python logging to use standard output stream
- Adjusted Python logging format for simplified processing
- Updated PythonProcess to pipe standard error and standard output streams
to reader
- Added Log Reader command with Virtual Thread for each Python Process
- Removed Python log properties from NiFi Properties configuration
---
.../java/org/apache/nifi/util/NiFiProperties.java | 2 -
.../src/main/asciidoc/administration-guide.adoc| 3 -
.../org/apache/nifi/controller/FlowController.java | 4 -
.../nifi-framework/nifi-resources/pom.xml | 1 -
.../src/main/resources/conf/logback.xml| 3 +
.../src/main/resources/conf/nifi.properties| 1 -
.../nifi-py4j-bundle/nifi-py4j-bridge/pom.xml | 6 +
.../java/org/apache/nifi/py4j/PythonProcess.java | 46 -
.../apache/nifi/py4j/PythonProcessLogReader.java | 176 +
.../org/apache/nifi/py4j/StandardPythonBridge.java | 6 +
.../nifi/py4j/logback/LevelChangeListener.java | 145 ++
.../nifi/py4j/logging/LogLevelChangeHandler.java | 37
.../nifi/py4j/logging/LogLevelChangeListener.java | 32 +++
.../apache/nifi/py4j/logging/PythonLogLevel.java | 67 +++
.../logging/StandardLogLevelChangeHandler.java | 100 ++
.../nifi/py4j/PythonProcessLogReaderTest.java | 220 +
.../nifi/py4j/logback/LevelChangeListenerTest.java | 67 +++
.../PythonControllerInteractionIT.java | 1 -
.../org/apache/nifi/python/PythonController.java | 8 +
.../apache/nifi/python/PythonProcessConfig.java| 25 ---
.../src/main/python/framework/Controller.py| 19 +-
.../resources/conf/clustered/node1/nifi.properties | 1 -
.../resources/conf/clustered/node2/nifi.properties | 1 -
.../test/resources/conf/default/nifi.properties| 1 -
.../src/test/resources/conf/pythonic/logback.xml | 3 +
.../test/resources/conf/pythonic/nifi.properties | 1 -
26 files changed, 919 insertions(+), 57 deletions(-)
diff --git
a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
index 7109600631..01dd887190 100644
---
a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
+++
b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
@@ -319,7 +319,6 @@ public class NiFiProperties extends ApplicationProperties {
public static final String PYTHON_FRAMEWORK_SOURCE_DIRECTORY =
"nifi.python.framework.source.directory";
public static final String PYTHON_EXTENSION_DIRECTORY_PREFIX =
"nifi.python.extensions.source.directory.";
public static final String PYTHON_WORKING_DIRECTORY =
"nifi.python.working.directory";
-public static final String PYTHON_LOGS_DIRECTORY =
"nifi.python.logs.directory";
public static final String PYTHON_MAX_PROCESSES =
"nifi.python.max.processes";
public static final String PYTHON_MAX_PROCESSES_PER_TYPE =
"nifi.python.max.processes.per.extension.type";
public static final String PYTHON_COMMS_TIMEOUT =
"nifi.python.comms.timeout";
@@ -327,7 +326,6 @@ public class NiFiProperties extends ApplicationProperties {
public static final String PYTHON_CONTROLLER_DEBUGPY_ENABLED =
"nifi.python.controller.debugpy.enabled";
public static final String PYTHON_CONTROLLER_DEBUGPY_PORT =
"nifi.python.controller.debugpy.port";
public static final String PYTHON_CONTROLLER_DEBUGPY_HOST =
"nifi.python.controller.debugpy.host";
-public static final String PYTHON_CONTROLLER_DEBUGPY_LOGS_DIR =
"nifi.python.controller.debugpy.logs.directory";
// kubernetes properties
public static final String CLUSTER_LEADER_ELECTION_KUBERNETES_LEASE_PREFIX
= "nifi.cluster.leader.election.kubernetes.lease.prefix";
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc
b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index db31ffa366..72e073c652 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -247,9 +247,6 @@ located as a sibling of this directory. For example
(nifi) branch main updated (ecea18f796 -> 01ca19eccc)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from ecea18f796 NIFI-12733 Make Apicurio's groupId optional and configurable and use artifactId instead of name as key add 01ca19eccc NIFI-12498 Adjust docs on default behaviour of prioritizers (#8451) No new revisions were added by this update. Summary of changes: nifi-docs/src/main/asciidoc/user-guide.adoc | 5 - 1 file changed, 4 insertions(+), 1 deletion(-)
(nifi) branch main updated: NIFI-12739 - Import ProcessPoolExecutor to fix bug in python 3.9+ (#8357)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/main by this push: new e03329e01f NIFI-12739 - Import ProcessPoolExecutor to fix bug in python 3.9+ (#8357) e03329e01f is described below commit e03329e01f491b88f6f90e5285cb034e204d9363 Author: Alex Ethier <39684038+alexeth...@users.noreply.github.com> AuthorDate: Wed Feb 14 12:50:01 2024 -0500 NIFI-12739 - Import ProcessPoolExecutor to fix bug in python 3.9+ (#8357) * NIFI-12739 Import ProcessPoolExecutor to fix bug in python 3.9+ that causes Exceptions to be raised incorrectly in multi-threaded applications (https://bugs.python.org/issue42647) * Removed extraneous whitespace. --- .../src/main/python/framework/Controller.py | 9 + 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py index 4425fd26b3..8919924c06 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py @@ -15,18 +15,19 @@ import logging import os -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor from py4j.java_gateway import JavaGateway, CallbackServerParameters, GatewayParameters import ExtensionManager -# We do not use ThreadPoolExecutor, but it must be kept here. Python introduced a bug in 3.9 that causes Exceptions to be raised +# We do not use ThreadPoolExecutor or ProcessPoolExecutor, but they must be kept here. Python introduced a bug in 3.9 that causes Exceptions to be raised # incorrectly in multi-threaded applications (https://bugs.python.org/issue42647). This works around the bug. -# What is actually necessary is to import ThreadPoolExecutor. +# What is actually necessary is to import ThreadPoolExecutor and ProcessPoolExecutor. # Unfortunately, IntelliJ often likes to cleanup the unused import. So we assign a bogus variable just so -# that we have some reference to ThreadPoolExecutor in order to prevent the IDE from cleaning up the import +# that we have some reference to ThreadPoolExecutor and ProcessPoolExecutor in order to prevent the IDE from cleaning up the import threadpool_attrs = dir(ThreadPoolExecutor) +processpool_attrs = dir(ProcessPoolExecutor) # Initialize logging
(nifi) branch main updated: NIFI-12768 Removed unstable filename assertion in TestListFile (#8383)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new e93fb17b66 NIFI-12768 Removed unstable filename assertion in
TestListFile (#8383)
e93fb17b66 is described below
commit e93fb17b66aea4940c076724b9ba8702172847fc
Author: David Handermann
AuthorDate: Fri Feb 9 15:00:22 2024 -0600
NIFI-12768 Removed unstable filename assertion in TestListFile (#8383)
---
.../src/test/java/org/apache/nifi/processors/standard/TestListFile.java | 1 -
1 file changed, 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListFile.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListFile.java
index 99b13b1afa..2f5a9a8c09 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListFile.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListFile.java
@@ -328,7 +328,6 @@ public class TestListFile {
runner.assertAllFlowFilesTransferred(ListFile.REL_SUCCESS);
final List successFiles4 =
runner.getFlowFilesForRelationship(ListFile.REL_SUCCESS);
assertEquals(1, successFiles4.size());
-assertEquals(file2.getName(),
successFiles4.get(0).getAttribute("filename"));
assertVerificationOutcome(Outcome.SUCCESSFUL, "Successfully listed .*
Found 3 objects. Of those, 1 matches the filter.");
}
(nifi) branch main updated: NIFI-12764 Removed Commons Codec and Lang3 from security-utils (#8380)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 2ea4838157 NIFI-12764 Removed Commons Codec and Lang3 from
security-utils (#8380)
2ea4838157 is described below
commit 2ea4838157748de8b597357ac7b7af4ff504d61e
Author: David Handermann
AuthorDate: Fri Feb 9 15:00:06 2024 -0600
NIFI-12764 Removed Commons Codec and Lang3 from security-utils (#8380)
---
nifi-commons/nifi-security-utils/pom.xml | 8 --
.../apache/nifi/security/util/KeyStoreUtils.java | 27 ++
.../security/util/StandardTlsConfiguration.java| 17 ++-
.../nifi/security/util/crypto/HashAlgorithm.java | 13 ++---
.../nifi/security/util/crypto/HashService.java | 33 --
.../nifi/security/util/KeyStoreUtilsTest.java | 4 +--
.../nifi/security/util/SslSocketFactoryTest.java | 7 +++--
.../nifi/security/util/crypto/HashServiceTest.java | 15 --
.../nifi-web/nifi-web-security/pom.xml | 6
nifi-system-tests/nifi-system-test-suite/pom.xml | 4 +++
10 files changed, 72 insertions(+), 62 deletions(-)
diff --git a/nifi-commons/nifi-security-utils/pom.xml
b/nifi-commons/nifi-security-utils/pom.xml
index 69f93ddcb3..b9d1e66a85 100644
--- a/nifi-commons/nifi-security-utils/pom.xml
+++ b/nifi-commons/nifi-security-utils/pom.xml
@@ -44,14 +44,6 @@
nifi-security-cert-builder
2.0.0-SNAPSHOT
-
-org.apache.commons
-commons-lang3
-
-
-commons-codec
-commons-codec
-
org.bouncycastle
bcprov-jdk18on
diff --git
a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeyStoreUtils.java
b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeyStoreUtils.java
index 4b541c0ce6..34e8c5e057 100644
---
a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeyStoreUtils.java
+++
b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeyStoreUtils.java
@@ -41,6 +41,7 @@ import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HexFormat;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -48,8 +49,6 @@ import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.TrustManagerFactory;
import javax.security.auth.x500.X500Principal;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.security.cert.builder.StandardCertificateBuilder;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
@@ -171,11 +170,11 @@ public class KeyStoreUtils {
public static TlsConfiguration
createTlsConfigAndNewKeystoreTruststore(final TlsConfiguration
tlsConfiguration, int certDurationDays,
String[] dnsSubjectAlternativeNames) throws IOException,
GeneralSecurityException {
final Path keyStorePath;
-final String keystorePassword =
StringUtils.isNotBlank(tlsConfiguration.getKeystorePassword()) ?
tlsConfiguration.getKeystorePassword() : generatePassword();
+final String keystorePassword =
isNotBlank(tlsConfiguration.getKeystorePassword()) ?
tlsConfiguration.getKeystorePassword() : generatePassword();
final KeystoreType keystoreType = tlsConfiguration.getKeystoreType()
!= null ? tlsConfiguration.getKeystoreType() : KeystoreType.PKCS12;
-final String keyPassword =
StringUtils.isNotBlank(tlsConfiguration.getKeyPassword()) ?
tlsConfiguration.getKeyPassword() : keystorePassword;
+final String keyPassword =
isNotBlank(tlsConfiguration.getKeyPassword()) ?
tlsConfiguration.getKeyPassword() : keystorePassword;
final Path trustStorePath;
-final String truststorePassword =
StringUtils.isNotBlank(tlsConfiguration.getTruststorePassword()) ?
tlsConfiguration.getTruststorePassword() : generatePassword();
+final String truststorePassword =
isNotBlank(tlsConfiguration.getTruststorePassword()) ?
tlsConfiguration.getTruststorePassword() : generatePassword();
final KeystoreType truststoreType =
tlsConfiguration.getTruststoreType() != null ?
tlsConfiguration.getTruststoreType() : KeystoreType.PKCS12;
// Create temporary Keystore file
@@ -259,11 +258,11 @@ public class KeyStoreUtils {
* @throws TlsException if there is a problem initializing or reading from
the keystore
*/
public static KeyManagerFactory loadKeyManagerFactory(String keystorePath,
String keystorePassword, String keyPassword, String keystoreType) throws
TlsException {
-if (StringUtils.isEmpty
(nifi) branch main updated: NIFI-12629 - adding metadata filtering to QueryPinecone (#8264)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new f402970132 NIFI-12629 - adding metadata filtering to QueryPinecone
(#8264)
f402970132 is described below
commit f402970132adb9e22840d5aae8117cf4d44262d7
Author: Pierre Villard
AuthorDate: Fri Jan 19 22:03:38 2024 +0400
NIFI-12629 - adding metadata filtering to QueryPinecone (#8264)
* NIFI-12629 - adding metadata filtering to QueryPinecone
---
.../src/main/python/vectorstores/QueryPinecone.py | 13 +++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git
a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/QueryPinecone.py
b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/QueryPinecone.py
index e12b7a6e77..28f6756fb8 100644
---
a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/QueryPinecone.py
+++
b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/QueryPinecone.py
@@ -18,6 +18,7 @@ from nifiapi.flowfiletransform import FlowFileTransform,
FlowFileTransformResult
from nifiapi.properties import PropertyDescriptor, StandardValidators,
ExpressionLanguageScope, PropertyDependency
import QueryUtils
import pinecone
+import json
from EmbeddingUtils import OPENAI, HUGGING_FACE, EMBEDDING_MODEL,
create_embedding_service
@@ -109,7 +110,14 @@ class QueryPinecone(FlowFileTransform):
)
NAMESPACE = PropertyDescriptor(
name="Namespace",
-description="The name of the Pinecone Namespace to put the documents
to.",
+description="The name of the Pinecone Namespace to query into.",
+required=False,
+validators=[StandardValidators.NON_EMPTY_VALIDATOR],
+expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES
+)
+FILTER = PropertyDescriptor(
+name="Metadata Filter",
+description="Optional metadata filter to apply with the query. For
example: { \"author\": {\"$eq\": \"john.doe\"} }",
required=False,
validators=[StandardValidators.NON_EMPTY_VALIDATOR],
expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES
@@ -164,8 +172,9 @@ class QueryPinecone(FlowFileTransform):
index = pinecone.Index(index_name)
text_key =
context.getProperty(self.TEXT_KEY).evaluateAttributeExpressions().getValue()
+filter =
context.getProperty(self.FILTER).evaluateAttributeExpressions(flowfile).getValue()
vectorstore = Pinecone(index, self.embeddings.embed_query, text_key,
namespace=namespace)
-results = vectorstore.similarity_search_with_score(query, num_results)
+results = vectorstore.similarity_search_with_score(query, num_results,
filter=None if filter is None else json.loads(filter))
documents = []
for result in results:
(nifi) branch main updated: NIFI-12634 Ignored Blank Prefix Values in Kubernetes Components (#8268)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 806d8d6165 NIFI-12634 Ignored Blank Prefix Values in Kubernetes
Components (#8268)
806d8d6165 is described below
commit 806d8d61658c03537a38132eb5764900cb7b4e8c
Author: David Handermann
AuthorDate: Fri Jan 19 07:30:45 2024 -0600
NIFI-12634 Ignored Blank Prefix Values in Kubernetes Components (#8268)
- Updated KubernetesConfigMapStateProvider and
KubernetesLeaderElectionManager to ignore blank prefix values as provided in
default configuration files
---
.../election/KubernetesLeaderElectionManager.java | 3 +-
.../KubernetesLeaderElectionManagerTest.java | 35 +-
.../pom.xml| 4 +++
.../provider/KubernetesConfigMapStateProvider.java | 7 +++--
.../KubernetesConfigMapStateProviderTest.java | 26 +++-
5 files changed, 56 insertions(+), 19 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-kubernetes-bundle/nifi-framework-kubernetes-leader-election/src/main/java/org/apache/nifi/kubernetes/leader/election/KubernetesLeaderElectionManager.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-kubernetes-bundle/nifi-framework-kubernetes-leader-election/src/main/java/org/apache/nifi/kubernetes/leader/election/KubernetesLeaderElectionManager.java
index ecefed0323..0272149470 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-kubernetes-bundle/nifi-framework-kubernetes-leader-election/src/main/java/org/apache/nifi/kubernetes/leader/election/KubernetesLeaderElectionManager.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-kubernetes-bundle/nifi-framework-kubernetes-leader-election/src/main/java/org/apache/nifi/kubernetes/leader/election/KubernetesLeaderElectionManager.java
@@ -80,7 +80,8 @@ public class KubernetesLeaderElectionManager extends
TrackedLeaderElectionManage
* Kubernetes Leader Election Manager constructor with NiFi Properties
*/
public KubernetesLeaderElectionManager(final NiFiProperties
nifiProperties) {
-this.roleIdPrefix =
nifiProperties.getProperty(NiFiProperties.CLUSTER_LEADER_ELECTION_KUBERNETES_LEASE_PREFIX);
+final String leasePrefix =
nifiProperties.getProperty(NiFiProperties.CLUSTER_LEADER_ELECTION_KUBERNETES_LEASE_PREFIX);
+this.roleIdPrefix = leasePrefix == null || leasePrefix.isBlank() ?
null : leasePrefix;
executorService = createExecutorService();
leaderElectionCommandProvider = createLeaderElectionCommandProvider();
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-kubernetes-bundle/nifi-framework-kubernetes-leader-election/src/test/java/org/apache/nifi/kubernetes/leader/election/KubernetesLeaderElectionManagerTest.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-kubernetes-bundle/nifi-framework-kubernetes-leader-election/src/test/java/org/apache/nifi/kubernetes/leader/election/KubernetesLeaderElectionManagerTest.java
index 3457332127..05c6f636c5 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-kubernetes-bundle/nifi-framework-kubernetes-leader-election/src/test/java/org/apache/nifi/kubernetes/leader/election/KubernetesLeaderElectionManagerTest.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-kubernetes-bundle/nifi-framework-kubernetes-leader-election/src/test/java/org/apache/nifi/kubernetes/leader/election/KubernetesLeaderElectionManagerTest.java
@@ -53,6 +53,8 @@ class KubernetesLeaderElectionManagerTest {
private static final String PREFIX = "label";
+private static final String EMPTY_PREFIX = "";
+
@Mock
LeaderElectionStateChangeListener changeListener;
@@ -68,16 +70,11 @@ class KubernetesLeaderElectionManagerTest {
ManagedLeaderElectionCommandProvider leaderElectionCommandProvider;
KubernetesLeaderElectionManager manager;
-KubernetesLeaderElectionManager managerWithProperties;
@BeforeEach
void setManager() {
leaderElectionCommandProvider = new
ManagedLeaderElectionCommandProvider();
manager = new MockKubernetesLeaderElectionManager(new
NiFiProperties());
-
-final Properties properties = new Properties();
-
properties.setProperty(NiFiProperties.CLUSTER_LEADER_ELECTION_KUBERNETES_LEASE_PREFIX,
PREFIX);
-managerWithProperties = new MockKubernetesLeaderElectionManager(new
NiFiProperties(properties));
}
@Test
@@ -195,15 +192,37 @@ class KubernetesLeaderElectionManagerTest {
@Test
void testRoleIdWithPrefix() {
-managerWithProperties.start();
+final Propert
(nifi) branch main updated: NIFI-9464 Fixed race condition between "Timer-Driven" threads when running SiteToSiteProvenanceReportingTask.onTrigger and "Compress Provenance Log" threads running EventFi
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 1c26b39fcd NIFI-9464 Fixed race condition between "Timer-Driven"
threads when running SiteToSiteProvenanceReportingTask.onTrigger and "Compress
Provenance Log" threads running EventFileCompressos.run that can cause the
SiteToSiteProvenanceReportingTask.onTrigger to pair an already compressed
.prov.gz file with a .toc file that corresponds to the uncompressed .prov file.
(#8157)
1c26b39fcd is described below
commit 1c26b39fcdae58e9e49a52e694a7f39fd8ad17b1
Author: tpalfy <53442425+tpa...@users.noreply.github.com>
AuthorDate: Mon Jan 8 20:30:28 2024 +0100
NIFI-9464 Fixed race condition between "Timer-Driven" threads when running
SiteToSiteProvenanceReportingTask.onTrigger and "Compress Provenance Log"
threads running EventFileCompressos.run that can cause the
SiteToSiteProvenanceReportingTask.onTrigger to pair an already compressed
.prov.gz file with a .toc file that corresponds to the uncompressed .prov file.
(#8157)
Co-authored-by: Tamas Palfy
---
.../nifi/provenance/EncryptedWriteAheadProvenanceRepository.java | 2 +-
.../org/apache/nifi/provenance/WriteAheadProvenanceRepository.java | 5 ++---
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java
b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java
index 7b028fd94d..052b799be2 100644
---
a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java
+++
b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java
@@ -106,6 +106,6 @@ public class EncryptedWriteAheadProvenanceRepository
extends WriteAheadProvenanc
};
// Delegate the init to the parent impl
-super.init(recordWriterFactory, recordReaderFactory, eventReporter,
authorizer, resourceFactory);
+super.init(recordWriterFactory, recordReaderFactory, eventReporter,
authorizer, resourceFactory, fileManager);
}
}
diff --git
a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
index 08cfc3f017..d486641ba6 100644
---
a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
+++
b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
@@ -137,13 +137,12 @@ public class WriteAheadProvenanceRepository implements
ProvenanceRepository {
}
};
- init(recordWriterFactory, recordReaderFactory, eventReporter,
authorizer, resourceFactory);
+ init(recordWriterFactory, recordReaderFactory, eventReporter,
authorizer, resourceFactory, fileManager);
}
synchronized void init(RecordWriterFactory recordWriterFactory,
RecordReaderFactory recordReaderFactory,
final EventReporter eventReporter, final Authorizer
authorizer,
- final ProvenanceAuthorizableFactory
resourceFactory) throws IOException {
-final EventFileManager fileManager = new EventFileManager();
+ final ProvenanceAuthorizableFactory
resourceFactory, final EventFileManager fileManager) throws IOException {
eventStore = new PartitionedWriteAheadEventStore(config,
recordWriterFactory, recordReaderFactory, eventReporter, fileManager);
(nifi) branch main updated: NIFI-12478 Return Message Type as body for JMS Object Messages (#8131)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new f73888e7dd NIFI-12478 Return Message Type as body for JMS Object
Messages (#8131)
f73888e7dd is described below
commit f73888e7dd4b0df37f5845bc318a7ad85f22d75d
Author: David Handermann
AuthorDate: Thu Dec 7 15:41:34 2023 -0600
NIFI-12478 Return Message Type as body for JMS Object Messages (#8131)
---
.../java/org/apache/nifi/jms/processors/JMSConsumer.java | 4 +++-
.../nifi/jms/processors/MessageBodyToBytesConverter.java | 15 ---
.../nifi/jms/processors/JMSPublisherConsumerIT.java | 13 ++---
pom.xml | 1 -
4 files changed, 9 insertions(+), 24 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSConsumer.java
b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSConsumer.java
index 3c721764ad..d39e625d7a 100644
---
a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSConsumer.java
+++
b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSConsumer.java
@@ -38,6 +38,7 @@ import javax.jms.StreamMessage;
import javax.jms.TextMessage;
import javax.jms.Topic;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
@@ -184,7 +185,8 @@ class JMSConsumer extends JMSWorker {
messageBody =
MessageBodyToBytesConverter.toBytes((BytesMessage) message);
} else if (message instanceof ObjectMessage) {
messageType = ObjectMessage.class.getSimpleName();
-messageBody =
MessageBodyToBytesConverter.toBytes((ObjectMessage) message);
+// Return Message Type as body to avoid unsupported class
references
+messageBody = messageType.getBytes(StandardCharsets.UTF_8);
} else if (message instanceof StreamMessage) {
messageType = StreamMessage.class.getSimpleName();
messageBody =
MessageBodyToBytesConverter.toBytes((StreamMessage) message);
diff --git
a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/MessageBodyToBytesConverter.java
b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/MessageBodyToBytesConverter.java
index c8cd9adbc9..e6c0d3e9e4 100644
---
a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/MessageBodyToBytesConverter.java
+++
b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/MessageBodyToBytesConverter.java
@@ -31,13 +31,11 @@ import javax.jms.BytesMessage;
import javax.jms.JMSException;
import javax.jms.MapMessage;
import javax.jms.MessageEOFException;
-import javax.jms.ObjectMessage;
import javax.jms.StreamMessage;
import javax.jms.TextMessage;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.SerializationUtils;
/**
*
@@ -88,19 +86,6 @@ abstract class MessageBodyToBytesConverter {
}
}
-/**
- *
- * @param message instance of {@link ObjectMessage}
- * @return byte array representing the {@link ObjectMessage}
- */
-public static byte[] toBytes(ObjectMessage message) {
-try {
-return SerializationUtils.serialize(message.getObject());
-} catch (Exception e) {
-throw new MessageConversionException("Failed to convert " +
ObjectMessage.class.getSimpleName() + " to byte[]", e);
-}
-}
-
/**
* @param message instance of {@link StreamMessage}
* @return byte array representing the {@link StreamMessage}
diff --git
a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java
b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java
index 36dd3828e7..a2d9f6e2f1 100644
---
a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java
+++
b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java
@@ -18,7 +18,6 @@ package org.apache.nifi.jms.processors;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang3.Seriali
(nifi) branch main updated: NIFI-12332 Removed nifi-toolkit-flowfile-repo Module (#7996)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 57ed9ad675 NIFI-12332 Removed nifi-toolkit-flowfile-repo Module (#7996)
57ed9ad675 is described below
commit 57ed9ad675682d74c99cb6c54f5d36dc67282830
Author: David Handermann
AuthorDate: Fri Nov 10 11:28:00 2023 -0600
NIFI-12332 Removed nifi-toolkit-flowfile-repo Module (#7996)
---
nifi-toolkit/nifi-toolkit-flowfile-repo/pom.xml| 32 ---
.../repos/flowfile/RepairCorruptedFileEndings.java | 287 -
.../flowfile/TestRepairCorruptedFileEndings.java | 171
nifi-toolkit/pom.xml | 10 -
4 files changed, 500 deletions(-)
diff --git a/nifi-toolkit/nifi-toolkit-flowfile-repo/pom.xml
b/nifi-toolkit/nifi-toolkit-flowfile-repo/pom.xml
deleted file mode 100644
index 95c2ba91b8..00
--- a/nifi-toolkit/nifi-toolkit-flowfile-repo/pom.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-
-
-http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/xsd/maven-4.0.0.xsd;>
-4.0.0
-
-org.apache.nifi
-nifi-toolkit
-2.0.0-SNAPSHOT
-
-nifi-toolkit-flowfile-repo
-
-
-org.apache.nifi
-nifi-utils
-2.0.0-SNAPSHOT
-
-
-org.apache.commons
-commons-lang3
-test
-
-
-
diff --git
a/nifi-toolkit/nifi-toolkit-flowfile-repo/src/main/java/org/apache/nifi/toolkit/repos/flowfile/RepairCorruptedFileEndings.java
b/nifi-toolkit/nifi-toolkit-flowfile-repo/src/main/java/org/apache/nifi/toolkit/repos/flowfile/RepairCorruptedFileEndings.java
deleted file mode 100644
index d911a9d3a1..00
---
a/nifi-toolkit/nifi-toolkit-flowfile-repo/src/main/java/org/apache/nifi/toolkit/repos/flowfile/RepairCorruptedFileEndings.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.toolkit.repos.flowfile;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.RandomAccessFile;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
-import java.util.List;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.nifi.stream.io.LimitingInputStream;
-import org.apache.nifi.stream.io.StreamUtils;
-
-public class RepairCorruptedFileEndings {
-private static final Pattern PARTITION_FILE_PATTERN =
Pattern.compile("partition\\-\\d+");
-
-private static void printUsage() {
-System.out.println("Whenever a sudden power loss occurs, it is common
with some operating systems for files that are being written to ");
-System.out.println("to contain many NUL characters (hex 0) at the end
of the file upon restart. If this happens to the FlowFile repository, ");
-System.out.println("NiFi will be unable to recover, because it cannot
properly read the repository. This utility attempts to read the FlowFile ");
-System.out.println("Repository and write out a new copy of the
repository, where the new copy does not contain the trailing NUL characters so
");
-System.out.println("NiFi can be restarted by pointing at the new
FlowFile Repository.");
-System.out.println("Typically, this problem can be identified by
seeing an error in the NiFi logs at startup, indicating either:");
-System.out.println();
-System.out.println("Caused by: java.io.IOException: Expected to read a
Sentinel Byte of '1' but got a value of '0' instead");
-System.out.println();
-System.out.println("or:");
-System.out.println();
-System.out.println("Caused by: java.lan
(nifi) branch main updated: NIFI-12339 Fixed Property Decryption for Migrated Components (#8002)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new dabdf94bf1 NIFI-12339 Fixed Property Decryption for Migrated
Components (#8002)
dabdf94bf1 is described below
commit dabdf94bf1d09ed9e3c299cbab33939610e2fc5f
Author: David Handermann
AuthorDate: Fri Nov 10 10:50:03 2023 -0600
NIFI-12339 Fixed Property Decryption for Migrated Components (#8002)
- Updated StandardVersionedComponentSynchronizer to decrypt properties when
creating extension references for subsequent migration
---
.../StandardVersionedComponentSynchronizer.java| 23 ++-
...StandardVersionedComponentSynchronizerTest.java | 163 +
.../nifi/controller/AbstractComponentNode.java | 5 +
3 files changed, 123 insertions(+), 68 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
index d9b6d81424..af3cf4931e 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
@@ -126,6 +126,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -1202,7 +1203,8 @@ public class StandardVersionedComponentSynchronizer
implements VersionedComponen
}
updateControllerService(newService, proposed, topLevelGroup);
-createdExtensions.add(new CreatedExtension(newService,
proposed.getProperties()));
+final Map decryptedProperties =
getDecryptedProperties(proposed.getProperties());
+createdExtensions.add(new CreatedExtension(newService,
decryptedProperties));
return newService;
}
@@ -1466,6 +1468,18 @@ public class StandardVersionedComponentSynchronizer
implements VersionedComponen
return fullPropertyMap;
}
+private Map getDecryptedProperties(final Map properties) {
+final Map decryptedProperties = new LinkedHashMap<>();
+
+final PropertyDecryptor decryptor = syncOptions.getPropertyDecryptor();
+properties.forEach((propertyName, propertyValue) -> {
+final String propertyValueDecrypted = decrypt(propertyValue,
decryptor);
+decryptedProperties.put(propertyName, propertyValueDecrypted);
+});
+
+return decryptedProperties;
+}
+
private static String decrypt(final String value, final PropertyDecryptor
decryptor) {
if (isValueEncrypted(value)) {
try {
@@ -2388,7 +2402,8 @@ public class StandardVersionedComponentSynchronizer
implements VersionedComponen
destination.addProcessor(procNode);
updateProcessor(procNode, proposed, topLevelGroup);
-createdExtensions.add(new CreatedExtension(procNode,
proposed.getProperties()));
+final Map decryptedProperties =
getDecryptedProperties(proposed.getProperties());
+createdExtensions.add(new CreatedExtension(procNode,
decryptedProperties));
// Notify the processor node that the configuration (properties, e.g.)
has been restored
final ProcessContext processContext =
context.getProcessContextFactory().apply(procNode);
@@ -3468,7 +3483,9 @@ public class StandardVersionedComponentSynchronizer
implements VersionedComponen
final BundleCoordinate coordinate =
toCoordinate(reportingTask.getBundle());
final ReportingTaskNode taskNode =
context.getFlowManager().createReportingTask(reportingTask.getType(),
reportingTask.getInstanceIdentifier(), coordinate, false);
updateReportingTask(taskNode, reportingTask);
-createdExtensions.add(new CreatedExtension(taskNode,
reportingTask.getProperties()));
+
+final Map decryptedProperties =
getDecryptedProperties(reportingTask.getProperties());
+createdExtensions.add(new CreatedExtension(taskNode,
decryptedProperties));
return taskNode;
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/
(nifi) branch main updated: NIFI-12341: Fixed typo in expression language guide
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 3b6c482db3 NIFI-12341: Fixed typo in expression language guide
3b6c482db3 is described below
commit 3b6c482db3f53a7d9090ff2e4375e4a8de0643f1
Author: Mark Payne
AuthorDate: Thu Nov 9 09:52:34 2023 -0500
NIFI-12341: Fixed typo in expression language guide
---
nifi-docs/src/main/asciidoc/expression-language-guide.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
index f545c33272..5dacdb00fd 100644
--- a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
@@ -2215,7 +2215,7 @@ In order to run the correct method, the parameter types
must be correct. The Exp
- ${literal(2):toDecimal:math("pow", 2.5)} runs Math.pow(2D,2.5D).
- - ${literal(64):toDouble():math("cbrt"):toNumber():math("max", 5)} runs
Math.max((Double.valueOf(Math.cbrt(64D))).longValue(), 5L). Note that the
toDecimal() is needed because "cbrt" takes a "double" as input and the "64"
will get interpreted as a long. The "toDecimal()" call is necessary to
correctly call the method. that the "toNumber()" call is necessary because
"cbrt" returns a double and the "max" method is must have parameters of the
same type and "5" is interpreted as a long.
+ - ${literal(64):toDecimal():math("cbrt"):toNumber():math("max", 5)}
runs Math.max((Double.valueOf(Math.cbrt(64D))).longValue(), 5L). Note that the
toDecimal() is needed because "cbrt" takes a "double" as input and the "64"
will get interpreted as a long. The "toDecimal()" call is necessary to
correctly call the method. that the "toNumber()" call is necessary because
"cbrt" returns a double and the "max" method is must have parameters of the
same type and "5" is interpreted as a long.
- ${literal(5.4):math("scalb", 2)} runs Math.scalb(5.4, 2). This
example is important because NiFi EL treats all whole numbers as "longs" and
there is no concept of an "int". "scalb" takes a second parameter of an "int"
and it is not overloaded to accept longs so it could not be run without special
type handling. In the instance where the Java method cannot be found using
parameters of type "double" and "long" the "math()" EL function will attempt to
find a Java method with the same nam [...]
(nifi) branch main updated: NIFI-12321 Avoid Exceptions when removing Python Processors (#7984)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new b50557b854 NIFI-12321 Avoid Exceptions when removing Python Processors
(#7984)
b50557b854 is described below
commit b50557b854a8a3ba4509ab9652ff0ab494b33cc7
Author: exceptionfactory
AuthorDate: Mon Nov 6 15:17:41 2023 -0600
NIFI-12321 Avoid Exceptions when removing Python Processors (#7984)
- Updated StandardPythonBridge.onProcessorRemoved to avoid throwing
exceptions when called with a Processor Type and Version that is not registered
- Updated system-tests workflow to run on changes in the nifi-py4j-bundle
---
.github/workflows/system-tests.yml | 2 +
.../org/apache/nifi/py4j/StandardPythonBridge.java | 51 --
2 files changed, 30 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/system-tests.yml
b/.github/workflows/system-tests.yml
index e9c07148a9..ec82be7278 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -26,6 +26,7 @@ on:
- 'nifi-api/**'
- 'nifi-framework-api/**'
- 'nifi-nar-bundles/nifi-framework-bundle/**'
+ - 'nifi-nar-bundles/nifi-py4j-bundle/**'
- 'nifi-stateless/**'
pull_request:
paths:
@@ -34,6 +35,7 @@ on:
- 'nifi-api/**'
- 'nifi-framework-api/**'
- 'nifi-nar-bundles/nifi-framework-bundle/**'
+ - 'nifi-nar-bundles/nifi-py4j-bundle/**'
- 'nifi-stateless/**'
env:
diff --git
a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java
b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java
index 67877fadcf..578d3b887e 100644
---
a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java
+++
b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java
@@ -96,7 +96,8 @@ public class StandardPythonBridge implements PythonBridge {
private PythonProcessorBridge createProcessorBridge(final String
identifier, final String type, final String version, final boolean
preferIsolatedProcess) {
ensureStarted();
-final ExtensionId extensionId = getExtensionId(type, version);
+final Optional extensionIdFound = findExtensionId(type,
version);
+final ExtensionId extensionId = extensionIdFound.orElseThrow(() -> new
IllegalArgumentException("Processor Type [%s] Version [%s] not
found".formatted(type, version)));
logger.debug("Creating Python Processor Type [{}] Version [{}]",
extensionId.type(), extensionId.version());
final PythonProcess pythonProcess =
getProcessForNextComponent(extensionId, identifier, preferIsolatedProcess);
@@ -151,26 +152,32 @@ public class StandardPythonBridge implements PythonBridge
{
@Override
public synchronized void onProcessorRemoved(final String identifier, final
String type, final String version) {
-final ExtensionId extensionId = getExtensionId(type, version);
-final List processes =
processesByProcessorType.get(extensionId);
-if (processes == null) {
-return;
-}
+final Optional extensionIdFound = findExtensionId(type,
version);
-// Find the Python Process that has the Processor, if any, and remove
it.
-// If there are no additional Processors in the Python Process, remove
it from our list and shut down the process.
-final Iterator processItr = processes.iterator(); //
Use iterator so we can call remove()
-while (processItr.hasNext()) {
-final PythonProcess process = processItr.next();
-final boolean removed = process.removeProcessor(identifier);
-if (removed && process.getProcessorCount() == 0) {
-processItr.remove();
-process.shutdown();
-break;
+if (extensionIdFound.isPresent()) {
+final ExtensionId extensionId = extensionIdFound.get();
+final List processes =
processesByProcessorType.get(extensionId);
+if (processes == null) {
+return;
+}
+
+// Find the Python Process that has the Processor, if any, and
remove it.
+// If there are no additional Processors in the Python Process,
remove it from our list and shut down the process.
+final Iterator processItr = processes.iterator();
// Use iterator so we can call remove()
+while (processItr.hasNext()) {
+final PythonProcess process = processItr.next();
+final boolean removed = process.removeProcessor(identifier);
+if (removed &
(nifi) branch main updated (dcc4b8590f -> 309061ca2d)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from dcc4b8590f NIFI-8673 Documented Active Threads versus Cluster Summary details add 309061ca2d NIFI-12310 Validate Python Component Type for Processes (#7972) No new revisions were added by this update. Summary of changes: .../org/apache/nifi/py4j/StandardPythonBridge.java | 79 -- 1 file changed, 27 insertions(+), 52 deletions(-)
(nifi) branch main updated: NIFI-12297 Standardized File Path resolution in Persistence Providers (#7975)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new c706877147 NIFI-12297 Standardized File Path resolution in Persistence
Providers (#7975)
c706877147 is described below
commit c706877147d89b7947f48cea68b3577bbfe3798d
Author: exceptionfactory
AuthorDate: Fri Nov 3 11:26:11 2023 -0500
NIFI-12297 Standardized File Path resolution in Persistence Providers
(#7975)
---
.../FileSystemBundlePersistenceProvider.java | 53 ++--
.../flow/FileSystemFlowPersistenceProvider.java| 72 ---
.../TestFileSystemBundlePersistenceProvider.java | 117 --
.../TestFileSystemFlowPersistenceProvider.java | 137 ++---
.../registry/FileSystemFlowRegistryClient.java | 122 +++---
.../tests/system/registry/RegistryClientIT.java| 30 ++---
.../tests/system/stateless/StatelessBasicsIT.java | 2 +-
7 files changed, 306 insertions(+), 227 deletions(-)
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/extension/FileSystemBundlePersistenceProvider.java
b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/extension/FileSystemBundlePersistenceProvider.java
index 555260a227..96bbceaffa 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/extension/FileSystemBundlePersistenceProvider.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/extension/FileSystemBundlePersistenceProvider.java
@@ -39,7 +39,12 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Map;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* An {@link BundlePersistenceProvider} that uses local file-system for
storage.
@@ -53,6 +58,8 @@ public class FileSystemBundlePersistenceProvider implements
BundlePersistencePro
static final String NAR_EXTENSION = ".nar";
static final String CPP_EXTENSION = ".cpp";
+private static final Pattern NUMBER_PATTERN = Pattern.compile("\\d+");
+
private File bundleStorageDir;
@Override
@@ -71,8 +78,7 @@ public class FileSystemBundlePersistenceProvider implements
BundlePersistencePro
try {
bundleStorageDir = new File(bundleStorageDirValue);
FileUtils.ensureDirectoryExistAndCanReadAndWrite(bundleStorageDir);
-LOGGER.info("Configured BundlePersistenceProvider with Extension
Bundle Storage Directory {}",
-new Object[] {bundleStorageDir.getAbsolutePath()});
+LOGGER.info("Configured BundlePersistenceProvider with Extension
Bundle Storage Directory {}", bundleStorageDir.getAbsolutePath());
} catch (IOException e) {
throw new ProviderCreationException(e);
}
@@ -107,7 +113,7 @@ public class FileSystemBundlePersistenceProvider implements
BundlePersistencePro
}
if (LOGGER.isDebugEnabled()) {
-LOGGER.debug("Writing extension bundle to {}", new
Object[]{bundleFile.getAbsolutePath()});
+LOGGER.debug("Writing extension bundle to {}",
bundleFile.getAbsolutePath());
}
try (final OutputStream out = new FileOutputStream(bundleFile)) {
@@ -124,7 +130,7 @@ public class FileSystemBundlePersistenceProvider implements
BundlePersistencePro
final File bundleFile = getBundleFile(versionCoordinate);
if (LOGGER.isDebugEnabled()) {
-LOGGER.debug("Reading extension bundle from {}", new
Object[]{bundleFile.getAbsolutePath()});
+LOGGER.debug("Reading extension bundle from {}",
bundleFile.getAbsolutePath());
}
try (final InputStream in = new FileInputStream(bundleFile);
@@ -142,7 +148,7 @@ public class FileSystemBundlePersistenceProvider implements
BundlePersistencePro
public synchronized void deleteBundleVersion(final BundleVersionCoordinate
versionCoordinate) throws BundlePersistenceException {
final File bundleFile = getBundleFile(versionCoordinate);
if (!bundleFile.exists()) {
-LOGGER.warn("Extension bundle content does not exist at {}", new
Object[] {bundleFile.getAbsolutePath()});
+LOGGER.warn("Extension bundle content does not exist at {}",
bundleFile.getAbsolutePath());
return;
}
@@ -152,7 +158,7 @@ public class FileSystemBundlePersistenceProvider implements
BundlePersistencePro
(nifi) branch support/nifi-1.x updated: NIFI-12294 Standardized NAR Entry Loading (#7958)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 6c0a82ebd7 NIFI-12294 Standardized NAR Entry Loading (#7958)
6c0a82ebd7 is described below
commit 6c0a82ebd7c24357ae9d184f8a0bbc725dfe2988
Author: exceptionfactory
AuthorDate: Wed Nov 1 13:10:00 2023 -0500
NIFI-12294 Standardized NAR Entry Loading (#7958)
- Consolidated duplicative NAR file entry normalization
---
.../java/org/apache/nifi/nar/NarUnpackerTest.java | 49 +--
.../main/java/org/apache/nifi/nar/NarUnpacker.java | 99 +++---
2 files changed, 72 insertions(+), 76 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
index beb673f0a8..5733d1bb60 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
@@ -33,6 +33,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
@@ -44,17 +45,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class NarUnpackerTest {
+private static final String PROPERTIES_PATH =
"/NarUnpacker/conf/nifi.properties";
+
@BeforeAll
public static void copyResources() throws IOException {
-
final Path sourcePath = Paths.get("./src/test/resources");
final Path targetPath = Paths.get("./target");
-Files.walkFileTree(sourcePath, new SimpleFileVisitor() {
+Files.walkFileTree(sourcePath, new SimpleFileVisitor<>() {
@Override
-public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs)
-throws IOException {
+public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) throws IOException {
Path relativeSource = sourcePath.relativize(dir);
Path target = targetPath.resolve(relativeSource);
@@ -62,12 +63,10 @@ public class NarUnpackerTest {
Files.createDirectories(target);
return FileVisitResult.CONTINUE;
-
}
@Override
-public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs)
-throws IOException {
+public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) throws IOException {
Path relativeSource = sourcePath.relativize(file);
Path target = targetPath.resolve(relativeSource);
@@ -81,8 +80,7 @@ public class NarUnpackerTest {
@Test
public void testUnpackNars() {
-
-NiFiProperties properties =
loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties",
Collections.EMPTY_MAP);
+NiFiProperties properties =
loadSpecifiedProperties(Collections.emptyMap());
assertEquals("./target/NarUnpacker/lib/",
properties.getProperty("nifi.nar.library.directory"));
@@ -93,10 +91,10 @@ public class NarUnpackerTest {
assertEquals(2, extensionMapping.getAllExtensionNames().size());
-
assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.one"));
-
assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.two"));
+
assertTrue(extensionMapping.getAllExtensionNames().containsKey("org.apache.nifi.processors.dummy.one"));
+
assertTrue(extensionMapping.getAllExtensionNames().containsKey("org.apache.nifi.processors.dummy.two"));
final File extensionsWorkingDir =
properties.getExtensionsWorkingDirectory();
-File[] extensionFiles = extensionsWorkingDir.listFiles();
+File[] extensionFiles =
Objects.requireNonNull(extensionsWorkingDir.listFiles());
Set expectedNars = new HashSet<>();
expectedNars.add("dummy-one.nar-unpacked");
@@ -110,24 +108,22 @@ public class NarUnpackerTest {
}
@Test
-public void testUnpackNarsFromEmptyDir() throws IOException {
-
+public void testUnpackNarsFromEmptyDir() {
final File emptyDir = new File("./target/empty/dir");
-emptyDir.delete();
(nifi) branch main updated: NIFI-12294 Standardized NAR Entry Loading (#7958)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 945d8b54bc NIFI-12294 Standardized NAR Entry Loading (#7958)
945d8b54bc is described below
commit 945d8b54bce63d7454238e11e08d073b88748378
Author: exceptionfactory
AuthorDate: Wed Nov 1 13:10:00 2023 -0500
NIFI-12294 Standardized NAR Entry Loading (#7958)
- Consolidated duplicative NAR file entry normalization
---
.../java/org/apache/nifi/nar/NarUnpackerTest.java | 49 +--
.../main/java/org/apache/nifi/nar/NarUnpacker.java | 99 +++---
2 files changed, 72 insertions(+), 76 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
index beb673f0a8..5733d1bb60 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
@@ -33,6 +33,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
@@ -44,17 +45,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class NarUnpackerTest {
+private static final String PROPERTIES_PATH =
"/NarUnpacker/conf/nifi.properties";
+
@BeforeAll
public static void copyResources() throws IOException {
-
final Path sourcePath = Paths.get("./src/test/resources");
final Path targetPath = Paths.get("./target");
-Files.walkFileTree(sourcePath, new SimpleFileVisitor() {
+Files.walkFileTree(sourcePath, new SimpleFileVisitor<>() {
@Override
-public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs)
-throws IOException {
+public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) throws IOException {
Path relativeSource = sourcePath.relativize(dir);
Path target = targetPath.resolve(relativeSource);
@@ -62,12 +63,10 @@ public class NarUnpackerTest {
Files.createDirectories(target);
return FileVisitResult.CONTINUE;
-
}
@Override
-public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs)
-throws IOException {
+public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) throws IOException {
Path relativeSource = sourcePath.relativize(file);
Path target = targetPath.resolve(relativeSource);
@@ -81,8 +80,7 @@ public class NarUnpackerTest {
@Test
public void testUnpackNars() {
-
-NiFiProperties properties =
loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties",
Collections.EMPTY_MAP);
+NiFiProperties properties =
loadSpecifiedProperties(Collections.emptyMap());
assertEquals("./target/NarUnpacker/lib/",
properties.getProperty("nifi.nar.library.directory"));
@@ -93,10 +91,10 @@ public class NarUnpackerTest {
assertEquals(2, extensionMapping.getAllExtensionNames().size());
-
assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.one"));
-
assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.two"));
+
assertTrue(extensionMapping.getAllExtensionNames().containsKey("org.apache.nifi.processors.dummy.one"));
+
assertTrue(extensionMapping.getAllExtensionNames().containsKey("org.apache.nifi.processors.dummy.two"));
final File extensionsWorkingDir =
properties.getExtensionsWorkingDirectory();
-File[] extensionFiles = extensionsWorkingDir.listFiles();
+File[] extensionFiles =
Objects.requireNonNull(extensionsWorkingDir.listFiles());
Set expectedNars = new HashSet<>();
expectedNars.add("dummy-one.nar-unpacked");
@@ -110,24 +108,22 @@ public class NarUnpackerTest {
}
@Test
-public void testUnpackNarsFromEmptyDir() throws IOException {
-
+public void testUnpackNarsFromEmptyDir() {
final File emptyDir = new File("./target/empty/dir");
-emptyDir.delete();
emptyDir.deleteOnExit();
assertTrue(emptyD
[nifi] branch main updated: NIFI-12266 Added Standard Shared NAR and BOM (#7925)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/main by this push: new bd4ba34339 NIFI-12266 Added Standard Shared NAR and BOM (#7925) bd4ba34339 is described below commit bd4ba343399ddd9f2b722d48b21f3258f34f8401 Author: exceptionfactory AuthorDate: Wed Oct 25 12:22:26 2023 -0500 NIFI-12266 Added Standard Shared NAR and BOM (#7925) - Added nifi-standard-shared-nar with common dependencies - Added nifi-standard-shared-bom with provided scope for parent module references --- minifi/minifi-assembly/pom.xml | 5 + .../minifi-nar-bundles/minifi-standard-nar/pom.xml | 2 +- .../minifi-update-attribute-nar/pom.xml| 2 +- minifi/pom.xml | 6 + nifi-assembly/pom.xml | 6 + nifi-bom/pom.xml | 6 + .../nifi-airtable-bundle/nifi-airtable-nar/pom.xml | 8 +- .../nifi-airtable-processors/pom.xml | 3 +- nifi-nar-bundles/nifi-airtable-bundle/pom.xml | 3 +- .../nifi-avro-bundle/nifi-avro-nar/pom.xml | 6 +- nifi-nar-bundles/nifi-avro-bundle/pom.xml | 3 +- .../nifi-aws-abstract-processors/pom.xml | 1 - .../nifi-aws-bundle/nifi-aws-nar/pom.xml | 4 - .../nifi-aws-bundle/nifi-aws-processors/pom.xml| 4 + .../nifi-aws-service-api-nar/pom.xml | 7 +- nifi-nar-bundles/nifi-aws-bundle/pom.xml | 3 +- .../nifi-azure-bundle/nifi-azure-nar/pom.xml | 4 - .../nifi-azure-reporting-task/pom.xml | 3 - .../nifi-azure-services-api-nar/pom.xml| 8 +- nifi-nar-bundles/nifi-azure-bundle/pom.xml | 3 +- .../nifi-box-bundle/nifi-box-nar/pom.xml | 4 - .../nifi-box-services-api-nar/pom.xml | 12 +- nifi-nar-bundles/nifi-box-bundle/pom.xml | 3 +- .../nifi-cassandra-nar/pom.xml | 4 - .../nifi-cassandra-services-api-nar/pom.xml| 3 +- nifi-nar-bundles/nifi-cassandra-bundle/pom.xml | 3 +- .../nifi-cdc-mysql-nar/pom.xml | 3 +- nifi-nar-bundles/nifi-cdc/pom.xml | 3 +- .../nifi-cipher-bundle/nifi-cipher-nar/pom.xml | 7 +- nifi-nar-bundles/nifi-cipher-bundle/pom.xml| 3 +- .../nifi-confluent-platform-nar/pom.xml| 3 +- .../nifi-confluent-platform-bundle/pom.xml | 3 +- .../nifi-datadog-bundle/nifi-datadog-nar/pom.xml | 7 +- nifi-nar-bundles/nifi-datadog-bundle/pom.xml | 3 +- .../nifi-dropbox-processors-nar/pom.xml| 4 - .../nifi-dropbox-services-api-nar/pom.xml | 7 +- .../nifi-dropbox-services-nar/pom.xml | 4 - nifi-nar-bundles/nifi-dropbox-bundle/pom.xml | 3 +- .../pom.xml| 7 +- .../nifi-elasticsearch-client-service-nar/pom.xml | 4 - nifi-nar-bundles/nifi-elasticsearch-bundle/pom.xml | 3 +- .../nifi-enrich-bundle/nifi-enrich-nar/pom.xml | 7 +- .../nifi-enrich-processors/pom.xml | 5 - nifi-nar-bundles/nifi-enrich-bundle/pom.xml| 3 +- .../nifi-flow-registry-client-nar/pom.xml | 8 +- .../nifi-flow-registry-client-bundle/pom.xml | 8 +- .../nifi-gcp-bundle/nifi-gcp-nar/pom.xml | 4 - .../nifi-gcp-services-api-nar/pom.xml | 7 +- nifi-nar-bundles/nifi-gcp-bundle/pom.xml | 3 +- .../nifi-geohash-bundle/nifi-geohash-nar/pom.xml | 7 +- .../nifi-geohash-processors/pom.xml| 3 - nifi-nar-bundles/nifi-geohash-bundle/pom.xml | 3 +- .../nifi-groovyx-bundle/nifi-groovyx-nar/pom.xml | 5 +- .../nifi-groovyx-processors/pom.xml| 5 - nifi-nar-bundles/nifi-groovyx-bundle/pom.xml | 3 +- .../nifi-hadoop-libraries-nar/pom.xml | 7 +- .../nifi-hadoop-libraries-bundle/pom.xml | 3 +- .../pom.xml| 7 +- .../nifi-hashicorp-vault-nar/pom.xml | 4 - .../nifi-hashicorp-vault-bundle/pom.xml| 3 +- .../nifi-hbase-bundle/nifi-hbase-nar/pom.xml | 7 +- nifi-nar-bundles/nifi-hbase-bundle/pom.xml | 3 +- .../nifi-hubspot-bundle/nifi-hubspot-nar/pom.xml | 8 +- nifi-nar-bundles/nifi-hubspot-bundle/pom.xml | 3 +- .../nifi-jms-cf-service-nar/pom.xml| 7 +- .../nifi-jms-processors-nar/pom.xml| 4 - nifi-nar-bundles/nifi-jms-bundle/pom.xml | 3 +- .../nifi-jolt-record-nar/pom.xml | 7 +- nifi-nar-bundles/nifi-jolt-record-bundle/pom.xml | 8 +- .../nifi-jslt-bundle/nifi-jslt-nar/pom.xml | 2 +- nifi-nar-bundles/nifi-jslt-bundle/pom.xml | 3 +- .../nifi
[nifi] branch main updated: NIFI-12269 Resolved Maven Build Warnings for Plugins (#7928)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 28b75c78a6 NIFI-12269 Resolved Maven Build Warnings for Plugins (#7928)
28b75c78a6 is described below
commit 28b75c78a6fd4557e8997c2eeb2bef782d89fc4f
Author: exceptionfactory
AuthorDate: Wed Oct 25 10:31:48 2023 -0500
NIFI-12269 Resolved Maven Build Warnings for Plugins (#7928)
- Set plugin versions in root configuration for plugin management across
modules
---
pom.xml | 18 ++
1 file changed, 18 insertions(+)
diff --git a/pom.xml b/pom.xml
index d4be371f8b..34fa9aca1a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -845,14 +845,17 @@
org.codehaus.mojo
buildnumber-maven-plugin
+3.2.0
org.antlr
antlr3-maven-plugin
+3.5.3
org.apache.maven.plugins
maven-checkstyle-plugin
+3.3.1
com.puppycrawl.tools
@@ -869,6 +872,21 @@
org.apache.rat
apache-rat-plugin
+
+org.apache.nifi
+nifi-nar-maven-plugin
+${nifi.nar.maven.plugin.version}
+
+
+org.sonatype.plugins
+nexus-staging-maven-plugin
+1.6.13
+
+
+org.jvnet.jaxb2.maven2
+maven-jaxb2-plugin
+0.15.3
+
[nifi] branch main updated: NIFI-12198 Add API and CLI commands to import reporting task snapshots (#7875)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new fd2de5a151 NIFI-12198 Add API and CLI commands to import reporting
task snapshots (#7875)
fd2de5a151 is described below
commit fd2de5a1515458ccc73df3e9a85fed9d6c4c3ce1
Author: Bryan Bende
AuthorDate: Fri Oct 20 10:45:48 2023 -0400
NIFI-12198 Add API and CLI commands to import reporting task snapshots
(#7875)
* NIFI-12198 Add API and CLI commands to import reporting task snapshots
---
.../VersionedReportingTaskImportRequestEntity.java | 47 +++
...VersionedReportingTaskImportResponseEntity.java | 47 +++
.../java/org/apache/nifi/util/BundleUtils.java | 35 +++--
.../reporting/ReportingTaskProvider.java | 9 +-
.../org/apache/nifi/controller/FlowController.java | 9 +-
.../serialization/FlowSynchronizationUtils.java| 111
.../StandardVersionedReportingTaskImporter.java| 145 +
.../serialization/VersionedFlowSynchronizer.java | 84 ++--
.../VersionedReportingTaskImportResult.java| 44 +++
.../VersionedReportingTaskImporter.java| 34 +
.../nifi/registry/flow/FlowRegistryUtils.java | 27 +++-
.../org/apache/nifi/web/NiFiServiceFacade.java | 25
.../apache/nifi/web/StandardNiFiServiceFacade.java | 72 ++
.../apache/nifi/web/api/ControllerResource.java| 120 +
.../nifi/web/controller/ControllerFacade.java | 6 +
.../web/dao/impl/StandardReportingTaskDAO.java | 24 ++--
.../nifi/web/StandardNiFiServiceFacadeTest.java| 43 ++
.../cli/impl/client/nifi/ControllerClient.java | 5 +
.../client/nifi/impl/JerseyControllerClient.java | 19 +++
.../cli/impl/command/nifi/NiFiCommandGroup.java| 2 +
.../command/nifi/flow/ImportReportingTasks.java| 69 ++
.../result/nifi/ImportReportingTasksResult.java| 90 +
22 files changed, 928 insertions(+), 139 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedReportingTaskImportRequestEntity.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedReportingTaskImportRequestEntity.java
new file mode 100644
index 00..0b8e1dc605
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedReportingTaskImportRequestEntity.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.entity;
+
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.nifi.flow.VersionedReportingTaskSnapshot;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "versionedReportingTaskImportRequestEntity")
+public class VersionedReportingTaskImportRequestEntity extends Entity {
+
+private VersionedReportingTaskSnapshot reportingTaskSnapshot;
+private Boolean disconnectedNodeAcknowledged;
+
+@ApiModelProperty("The snapshot to import")
+public VersionedReportingTaskSnapshot getReportingTaskSnapshot() {
+return reportingTaskSnapshot;
+}
+
+public void setReportingTaskSnapshot(VersionedReportingTaskSnapshot
reportingTaskSnapshot) {
+this.reportingTaskSnapshot = reportingTaskSnapshot;
+}
+
+@ApiModelProperty("The disconnected node acknowledged flag")
+public Boolean getDisconnectedNodeAcknowledged() {
+return disconnectedNodeAcknowledged;
+}
+
+public void setDisconnectedNodeAcknowledged(Boolean
disconnectedNodeAcknowledged) {
+this.disconnectedNodeAcknowledged = disconnectedNodeAcknowledged;
+}
+}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedReportingTaskImportResponseEntity.java
b/nifi-nar-bundles/nifi-framew
[nifi] branch main updated: NIFI-12250: (#7908)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 98eef74dd8 NIFI-12250: (#7908)
98eef74dd8 is described below
commit 98eef74dd84952f2a6ae7793277d80985ab16b99
Author: Matt Gilman
AuthorDate: Thu Oct 19 15:53:07 2023 -0400
NIFI-12250: (#7908)
- Fixing issue where the registry configuration dialog was being
initialized twice.
---
.../src/main/webapp/js/nf/canvas/nf-settings.js| 39 --
1 file changed, 13 insertions(+), 26 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js
index f68448f127..0b71333757 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js
@@ -1192,19 +1192,6 @@
}
}
});
-
-// initialize the registry configuration dialog
-$('#registry-configuration-dialog').modal({
-scrollableContentStyle: 'scrollable',
-handler: {
-close: function () {
-$('#registry-id-config').text('');
-$('#registry-name-config').val('');
-$('#registry-type-config').val('');
-$('#registry-description-config').val('');
-}
-}
-});
};
/**
@@ -2504,19 +2491,6 @@
}
}
});
-
-// initialize the registry configuration dialog
-$('#registry-configuration-dialog').modal({
-scrollableContentStyle: 'scrollable',
-handler: {
-close: function () {
-$('#registry-id').text('');
-$('#registry-name').val('');
-$('#registry-location').val('');
-$('#registry-description').val('');
-}
-}
-});
};
/**
@@ -3902,6 +3876,19 @@
}
});
+// initialize the registry configuration dialog
+$('#registry-configuration-dialog').modal({
+scrollableContentStyle: 'scrollable',
+handler: {
+close: function () {
+$('#registry-id-config').text('');
+$('#registry-name-config').val('');
+$('#registry-type-config').val('');
+$('#registry-description-config').val('');
+}
+}
+});
+
// initialize each tab
initGeneral();
nfControllerServices.init(getControllerServicesTable(),
nfSettings.showSettings);
[nifi] branch main updated: NIFI-12247: (#7905)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new f2822a6bfb NIFI-12247: (#7905)
f2822a6bfb is described below
commit f2822a6bfb0fa009eeea001fdfd7e3606d7e0700
Author: Matt Gilman
AuthorDate: Thu Oct 19 14:25:06 2023 -0400
NIFI-12247: (#7905)
- Reverting to the initial value if there is no selected parameter in the
combo editor.
---
.../src/main/webapp/js/jquery/propertytable/jquery.propertytable.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js
index 5fa3b224b4..355d7a3048 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js
@@ -768,8 +768,8 @@
if (parametersSupported && _.isUndefined(selectedValue)) {
selectedOption = parameterCombo.combo('getSelectedOption');
-// if the parameters are still loading, revert to the
initial value, otherwise use the selected parameter
-if (selectedOption === LOADING_PARAMETERS_OPTION) {
+// if there are no parameters, or they are still loading,
revert to the initial value otherwise use the selected parameter
+if (_.isUndefined(selectedOption) || selectedOption ===
LOADING_PARAMETERS_OPTION) {
selectedValue = initialValue;
} else {
selectedValue = selectedOption.value;
[nifi] branch main updated: NIFI-12248: (#7906)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new dc76d3896d NIFI-12248: (#7906)
dc76d3896d is described below
commit dc76d3896dcd2f8a862885b4bfbfcb4542fbab97
Author: Matt Gilman
AuthorDate: Thu Oct 19 13:56:55 2023 -0400
NIFI-12248: (#7906)
- Fixing arrow alignment in combo editor.
---
.../nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css| 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css
index a1902e2168..2dd7b351cd 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css
@@ -52,13 +52,13 @@
box-shadow:0 3px 6px rgba(0,0,0,0.3);
}
-.combo-arrow {
+.combo .combo-arrow {
float: right;
width: 10px;
-margin-top: 4px;
+line-height: 30px;
}
-.combo-text {
+.combo .combo-text {
float: left;
overflow: hidden;
white-space: nowrap;
[nifi] branch main updated: NIFI-12244 - UpdateRecord invalid because of migrateProperties (#7898)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 814e94b2ac NIFI-12244 - UpdateRecord invalid because of
migrateProperties (#7898)
814e94b2ac is described below
commit 814e94b2ac27becc0835f95d4670f1e34f23993a
Author: Pierre Villard
AuthorDate: Thu Oct 19 15:34:55 2023 +0200
NIFI-12244 - UpdateRecord invalid because of migrateProperties (#7898)
---
.../src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java
index 1c19be5869..168dfe59ac 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java
@@ -172,6 +172,7 @@ public class UpdateRecord extends AbstractRecordProcessor {
@Override
public void migrateProperties(final PropertyConfiguration config) {
+super.migrateProperties(config);
config.renameProperty("replacement-value-strategy",
REPLACEMENT_VALUE_STRATEGY.getName());
}
[nifi] branch support/nifi-1.x updated: NIFI-12237 change label height and width from POSITION to SIZE difference (#7889)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 33773c772e NIFI-12237 change label height and width from POSITION to
SIZE difference (#7889)
33773c772e is described below
commit 33773c772ee693e8fe52c76c86011a05eb8a03eb
Author: Michael Moser
AuthorDate: Wed Oct 18 12:56:19 2023 -0400
NIFI-12237 change label height and width from POSITION to SIZE difference
(#7889)
---
.../nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java | 3 +++
.../main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java | 5 +
.../org/apache/nifi/registry/flow/diff/StandardFlowComparator.java | 4 ++--
3 files changed, 10 insertions(+), 2 deletions(-)
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
index 3fc4e4b1bf..b0f1bc6ea7 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
@@ -71,6 +71,9 @@ public class ConciseEvolvingDifferenceDescriptor implements
DifferenceDescriptor
case POSITION_CHANGED:
description = "Position was changed";
break;
+case SIZE_CHANGED:
+description = "Size was changed";
+break;
case BENDPOINTS_CHANGED:
description = "Connection Bendpoints changed";
break;
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java
index 219aff4642..b54fd5dd00 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java
@@ -205,6 +205,11 @@ public enum DifferenceType {
*/
POSITION_CHANGED("Position Changed"),
+/**
+ * The height and/or width size of the component is different in each of
the flows
+ */
+SIZE_CHANGED("Size Changed"),
+
/**
* The stylistic configuration of the component is different in each of
the flows
*/
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java
index be21d97069..4762c03bf5 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java
@@ -441,8 +441,8 @@ public class StandardFlowComparator implements
FlowComparator {
}
addIfDifferent(differences, DifferenceType.LABEL_VALUE_CHANGED,
labelA, labelB, VersionedLabel::getLabel);
-addIfDifferent(differences, DifferenceType.POSITION_CHANGED, labelA,
labelB, VersionedLabel::getHeight);
-addIfDifferent(differences, DifferenceType.POSITION_CHANGED, labelA,
labelB, VersionedLabel::getWidth);
+addIfDifferent(differences, DifferenceType.SIZE_CHANGED, labelA,
labelB, VersionedLabel::getHeight);
+addIfDifferent(differences, DifferenceType.SIZE_CHANGED, labelA,
labelB, VersionedLabel::getWidth);
addIfDifferent(differences, DifferenceType.STYLE_CHANGED, labelA,
labelB, VersionedLabel::getStyle);
addIfDifferent(differences, DifferenceType.ZINDEX_CHANGED, labelA,
labelB, VersionedLabel::getzIndex);
}
[nifi] branch main updated: NIFI-12237 change label height and width from POSITION to SIZE difference (#7889)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 229199dab7 NIFI-12237 change label height and width from POSITION to
SIZE difference (#7889)
229199dab7 is described below
commit 229199dab79f37a82ca2ab61cc2a4677996cc7ea
Author: Michael Moser
AuthorDate: Wed Oct 18 12:56:19 2023 -0400
NIFI-12237 change label height and width from POSITION to SIZE difference
(#7889)
---
.../nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java | 3 +++
.../main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java | 5 +
.../org/apache/nifi/registry/flow/diff/StandardFlowComparator.java | 4 ++--
3 files changed, 10 insertions(+), 2 deletions(-)
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
index 07db3efe50..e1e71b49b7 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
@@ -65,6 +65,9 @@ public class ConciseEvolvingDifferenceDescriptor implements
DifferenceDescriptor
case POSITION_CHANGED:
description = "Position was changed";
break;
+case SIZE_CHANGED:
+description = "Size was changed";
+break;
case BENDPOINTS_CHANGED:
description = "Connection Bendpoints changed";
break;
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java
index c6f527390c..23d02391c2 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java
@@ -220,6 +220,11 @@ public enum DifferenceType {
*/
POSITION_CHANGED("Position Changed"),
+/**
+ * The height and/or width size of the component is different in each of
the flows
+ */
+SIZE_CHANGED("Size Changed"),
+
/**
* The stylistic configuration of the component is different in each of
the flows
*/
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java
index 30475e2696..e306765771 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java
@@ -456,8 +456,8 @@ public class StandardFlowComparator implements
FlowComparator {
}
addIfDifferent(differences, DifferenceType.LABEL_VALUE_CHANGED,
labelA, labelB, VersionedLabel::getLabel);
-addIfDifferent(differences, DifferenceType.POSITION_CHANGED, labelA,
labelB, VersionedLabel::getHeight);
-addIfDifferent(differences, DifferenceType.POSITION_CHANGED, labelA,
labelB, VersionedLabel::getWidth);
+addIfDifferent(differences, DifferenceType.SIZE_CHANGED, labelA,
labelB, VersionedLabel::getHeight);
+addIfDifferent(differences, DifferenceType.SIZE_CHANGED, labelA,
labelB, VersionedLabel::getWidth);
addIfDifferent(differences, DifferenceType.STYLE_CHANGED, labelA,
labelB, VersionedLabel::getStyle);
addIfDifferent(differences, DifferenceType.ZINDEX_CHANGED, labelA,
labelB, VersionedLabel::getzIndex);
}
[nifi] branch support/nifi-1.x updated: NIFI-12238 Fix SplitText endline trimming with max fragment size (#7892)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 142abb07c6 NIFI-12238 Fix SplitText endline trimming with max fragment
size (#7892)
142abb07c6 is described below
commit 142abb07c6a4d585b0335a7e6628ffba82147a68
Author: Gabor Gyimesi
AuthorDate: Wed Oct 18 18:40:52 2023 +0200
NIFI-12238 Fix SplitText endline trimming with max fragment size (#7892)
---
.../apache/nifi/processors/standard/SplitText.java | 26 +++---
.../nifi/processors/standard/TestSplitText.java| 21 +
2 files changed, 34 insertions(+), 13 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
index d16bc4be5f..c1015df7b7 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
@@ -461,13 +461,6 @@ public class SplitText extends AbstractProcessor {
while ((offsetInfo = demarcator.nextOffsetInfo()) != null) {
lastCrlfLength = offsetInfo.getCrlfLength();
-if (offsetInfo.getLength() == offsetInfo.getCrlfLength()) {
-trailingCrlfLength += offsetInfo.getCrlfLength();
-trailingLineCount++;
-} else if (offsetInfo.getLength() > offsetInfo.getCrlfLength()) {
-trailingCrlfLength = 0; // non-empty line came in, thus
resetting counter
-}
-
if (length + offsetInfo.getLength() + startingLength >
this.maxSplitSize) {
if (length == 0) { // single line per split
length += offsetInfo.getLength();
@@ -476,12 +469,19 @@ public class SplitText extends AbstractProcessor {
remaningOffsetInfo = offsetInfo;
}
break;
-} else {
-length += offsetInfo.getLength();
-actualLineCount++;
-if (splitMaxLineCount > 0 && actualLineCount >=
splitMaxLineCount) {
-break;
-}
+}
+
+if (offsetInfo.getLength() == offsetInfo.getCrlfLength()) {
+trailingCrlfLength += offsetInfo.getCrlfLength();
+trailingLineCount++;
+} else if (offsetInfo.getLength() > offsetInfo.getCrlfLength()) {
+trailingCrlfLength = 0; // non-empty line came in, thus
resetting counter
+}
+
+length += offsetInfo.getLength();
+actualLineCount++;
+if (splitMaxLineCount > 0 && actualLineCount >= splitMaxLineCount)
{
+break;
}
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitText.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitText.java
index c76a49fc97..0f1ec059e7 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitText.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitText.java
@@ -890,4 +890,25 @@ public class TestSplitText {
splits.get(1).assertContentEquals("\n");
}
+@Test
+public void testMaxFragmentSizeWithTrimmedEndlines() {
+final TestRunner splitRunner = TestRunners.newTestRunner(new
SplitText());
+splitRunner.setProperty(SplitText.HEADER_LINE_COUNT, "2");
+splitRunner.setProperty(SplitText.LINE_SPLIT_COUNT, "0");
+splitRunner.setProperty(SplitText.FRAGMENT_MAX_SIZE, "30 B");
+splitRunner.setProperty(SplitText.REMOVE_TRAILING_NEWLINES, "true");
+
+splitRunner.enqueue("header1\nheader2\nline1 longer than
limit\nline2\nline3\n\n\n\n\n");
+
+splitRunner.run();
+splitRunner.assertTransferCount(SplitText.REL_SPLITS, 3);
+splitRunner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
+splitRunner.assertTransferCount(SplitText.REL_FAILURE, 0);
+
+final List splits =
splitRunner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
+splits.get(0).assertContentEquals("header1\nheader2\nline1 longer than
limit");
+splits.get(1).assertContentEquals("header1\nheader2\nline2\nline3");
+splits.get(2).assertContentEquals("header1\nheader2");
+}
+
}
[nifi] branch main updated: NIFI-12238 Fix SplitText endline trimming with max fragment size (#7892)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 91e4b453b4 NIFI-12238 Fix SplitText endline trimming with max fragment
size (#7892)
91e4b453b4 is described below
commit 91e4b453b44f45efee8da838c02226d9b972d7b0
Author: Gabor Gyimesi
AuthorDate: Wed Oct 18 18:40:52 2023 +0200
NIFI-12238 Fix SplitText endline trimming with max fragment size (#7892)
---
.../apache/nifi/processors/standard/SplitText.java | 26 +++---
.../nifi/processors/standard/TestSplitText.java| 21 +
2 files changed, 34 insertions(+), 13 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
index ca33b4046b..5797a205af 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
@@ -459,13 +459,6 @@ public class SplitText extends AbstractProcessor {
while ((offsetInfo = demarcator.nextOffsetInfo()) != null) {
lastCrlfLength = offsetInfo.getCrlfLength();
-if (offsetInfo.getLength() == offsetInfo.getCrlfLength()) {
-trailingCrlfLength += offsetInfo.getCrlfLength();
-trailingLineCount++;
-} else if (offsetInfo.getLength() > offsetInfo.getCrlfLength()) {
-trailingCrlfLength = 0; // non-empty line came in, thus
resetting counter
-}
-
if (length + offsetInfo.getLength() + startingLength >
this.maxSplitSize) {
if (length == 0) { // single line per split
length += offsetInfo.getLength();
@@ -474,12 +467,19 @@ public class SplitText extends AbstractProcessor {
remaningOffsetInfo = offsetInfo;
}
break;
-} else {
-length += offsetInfo.getLength();
-actualLineCount++;
-if (splitMaxLineCount > 0 && actualLineCount >=
splitMaxLineCount) {
-break;
-}
+}
+
+if (offsetInfo.getLength() == offsetInfo.getCrlfLength()) {
+trailingCrlfLength += offsetInfo.getCrlfLength();
+trailingLineCount++;
+} else if (offsetInfo.getLength() > offsetInfo.getCrlfLength()) {
+trailingCrlfLength = 0; // non-empty line came in, thus
resetting counter
+}
+
+length += offsetInfo.getLength();
+actualLineCount++;
+if (splitMaxLineCount > 0 && actualLineCount >= splitMaxLineCount)
{
+break;
}
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitText.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitText.java
index c76a49fc97..0f1ec059e7 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitText.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitText.java
@@ -890,4 +890,25 @@ public class TestSplitText {
splits.get(1).assertContentEquals("\n");
}
+@Test
+public void testMaxFragmentSizeWithTrimmedEndlines() {
+final TestRunner splitRunner = TestRunners.newTestRunner(new
SplitText());
+splitRunner.setProperty(SplitText.HEADER_LINE_COUNT, "2");
+splitRunner.setProperty(SplitText.LINE_SPLIT_COUNT, "0");
+splitRunner.setProperty(SplitText.FRAGMENT_MAX_SIZE, "30 B");
+splitRunner.setProperty(SplitText.REMOVE_TRAILING_NEWLINES, "true");
+
+splitRunner.enqueue("header1\nheader2\nline1 longer than
limit\nline2\nline3\n\n\n\n\n");
+
+splitRunner.run();
+splitRunner.assertTransferCount(SplitText.REL_SPLITS, 3);
+splitRunner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
+splitRunner.assertTransferCount(SplitText.REL_FAILURE, 0);
+
+final List splits =
splitRunner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
+splits.get(0).assertContentEquals("header1\nheader2\nline1 longer than
limit");
+splits.get(1).assertContentEquals("header1\nheader2\nline2\nline3");
+splits.get(2).assertContentEquals("header1\nheader2");
+}
+
}
[nifi] branch support/nifi-1.x updated: NIFI-12219 Added Xodus Flow History Storage with Migration (#7885)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 1a2d769e82 NIFI-12219 Added Xodus Flow History Storage with Migration
(#7885)
1a2d769e82 is described below
commit 1a2d769e829385b8ed443949ede25810e76cf304
Author: exceptionfactory
AuthorDate: Wed Oct 18 09:48:05 2023 -0500
NIFI-12219 Added Xodus Flow History Storage with Migration (#7885)
- Added EntityStoreAuditService implementation using JetBrains Xodus
libraries
- Added FlowConfigurationHistoryMigrator for checking and migrating H2
databases containing Flow Configuration History to Xodus persistent store
implementation
---
.../src/main/asciidoc/administration-guide.adoc| 5 +-
.../nifi-framework/nifi-administration/pom.xml | 13 +
.../nifi/admin/AuditDataSourceFactoryBean.java | 233
.../admin/service/EntityStoreAuditService.java | 639 +
.../service/entity/ActionEntity.java} | 42 +-
.../service/entity/ActionLink.java}| 34 +-
.../service/entity/ConfigureDetailsEntity.java}| 34 +-
.../service/entity/ConnectDetailsEntity.java} | 42 +-
.../service/entity/EntityProperty.java}| 36 +-
.../service/entity/EntityType.java}| 37 +-
.../service/entity/ExtensionDetailsEntity.java}| 32 +-
.../service/entity/MoveDetailsEntity.java} | 36 +-
.../service/entity/PurgeDetailsEntity.java}| 32 +-
.../main/resources/nifi-administration-context.xml | 36 --
.../admin/service/EntityStoreAuditServiceTest.java | 370
.../apache/nifi/headless/HeadlessAuditService.java | 68 +++
.../apache/nifi/headless/HeadlessNiFiServer.java | 3 +-
.../nifi-framework/nifi-resources/pom.xml | 2 -
.../src/main/resources/conf/logback.xml| 3 +
.../src/main/resources/conf/nifi.properties| 3 +-
.../nifi-framework/nifi-web/nifi-web-api/pom.xml | 11 +
.../apache/nifi/web/NiFiWebApiConfiguration.java | 23 +-
.../FlowConfigurationHistoryMigrator.java | 116
.../FlowConfigurationHistoryMigratorTest.java | 64 +++
.../test/resources/migration/nifi-flow-audit.mv.db | Bin 0 -> 36864 bytes
.../resources/conf/clustered/node1/logback.xml | 3 +
.../resources/conf/clustered/node1/nifi.properties | 3 +-
.../resources/conf/clustered/node2/logback.xml | 3 +
.../resources/conf/clustered/node2/nifi.properties | 3 +-
.../src/test/resources/conf/default/logback.xml| 3 +
.../test/resources/conf/default/nifi.properties| 3 +-
31 files changed, 1457 insertions(+), 475 deletions(-)
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc
b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 530e36781a..c50eea6c03 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -3449,14 +3449,13 @@ for components to persist state. See the
<> section for more i
|
-=== H2 Settings
+=== Database Settings
-The H2 Settings section defines the settings for the H2 database, which keeps
track of user access and flow controller history.
+The Database Settings section defines the settings for the internal database,
which tracks flow configuration history.
|
|*Property*|*Description*
|`nifi.database.directory`*|The location of the H2 database directory. The
default value is `./database_repository`.
-|`nifi.h2.url.append`|This property specifies additional arguments to add to
the connection string for the H2 database. The default value should be used and
should not be changed. It is:
`;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE`.
|
[[repository-encryption-properties]]
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
index ee64ba554d..b6a973331a 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
@@ -21,6 +21,9 @@
1.24.0-SNAPSHOT
nifi-administration
+
+2.0.1
+
org.apache.nifi
@@ -83,5 +86,15 @@
org.apache.commons
commons-collections4
+
+org.jetbrains.xodus
+xodus-openAPI
+${xodus.version}
+
+
+org.jetbrains.xodus
+xodus-entity-store
+${xodus.version}
+
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java
b/nifi-nar-bun
[nifi] branch main updated (22ad7d542d -> 96eb1d825a)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 22ad7d542d NIFI-12206 Refactor Flow History using JetBrains Xodus (#7870) add 96eb1d825a NIFI-1 Protect against missing parameter context when syncing a PG in component synchronizer (#7877) No new revisions were added by this update. Summary of changes: .../StandardVersionedComponentSynchronizer.java| 41 - ...StandardVersionedComponentSynchronizerTest.java | 53 ++ 2 files changed, 71 insertions(+), 23 deletions(-)
[nifi] branch main updated (d2aec89738 -> ce2a156b9c)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from d2aec89738 NIFI-12166 Add repo usage to the monitoring endpoint (#7836) add ce2a156b9c NIFI-12140: Updated dependency to include nifi-property-utils in nifi-redis-utils module as a result of merging NIFI-12185 No new revisions were added by this update. Summary of changes: nifi-nar-bundles/nifi-redis-bundle/nifi-redis-utils/pom.xml | 5 - 1 file changed, 4 insertions(+), 1 deletion(-)
[nifi] branch support/nifi-1.x updated: NIFI-12140 Add optional properties to RedisConnectionPoolService for Username and Sentinel Username (#7862)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new c24271fb43 NIFI-12140 Add optional properties to
RedisConnectionPoolService for Username and Sentinel Username (#7862)
c24271fb43 is described below
commit c24271fb432aeaf93082ca4ab7469948891dc753
Author: Bryan Bende
AuthorDate: Tue Oct 10 14:39:41 2023 -0400
NIFI-12140 Add optional properties to RedisConnectionPoolService for
Username and Sentinel Username (#7862)
---
.../ITRedisDistributedMapCacheClientService.java | 87 +-
.../nifi/redis/testcontainers/RedisContainer.java | 11 +++
.../testcontainers/RedisReplicaContainer.java | 5 ++
.../testcontainers/RedisSentinelContainer.java | 17 +
.../org/apache/nifi/redis/util/RedisConfig.java| 19 +
.../org/apache/nifi/redis/util/RedisUtils.java | 30 +++-
6 files changed, 149 insertions(+), 20 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-redis-bundle/nifi-redis-extensions/src/test/java/org/apache/nifi/redis/service/ITRedisDistributedMapCacheClientService.java
b/nifi-nar-bundles/nifi-redis-bundle/nifi-redis-extensions/src/test/java/org/apache/nifi/redis/service/ITRedisDistributedMapCacheClientService.java
index 15b3c41792..a83b46f915 100644
---
a/nifi-nar-bundles/nifi-redis-bundle/nifi-redis-extensions/src/test/java/org/apache/nifi/redis/service/ITRedisDistributedMapCacheClientService.java
+++
b/nifi-nar-bundles/nifi-redis-bundle/nifi-redis-extensions/src/test/java/org/apache/nifi/redis/service/ITRedisDistributedMapCacheClientService.java
@@ -87,7 +87,7 @@ public class ITRedisDistributedMapCacheClientService {
@Test
public void testStandaloneRedis() throws InitializationException,
IOException {
-int redisPort = setupStandaloneRedis(null).port;
+int redisPort = setupStandaloneRedis(null,null).port;
setUpRedisConnectionPool(portsToConnectionString(redisPort), pool -> {
// uncomment this to test using a different database index than
the default 0
// testRunner.setProperty(pool, RedisUtils.DATABASE, "1");
@@ -98,9 +98,24 @@ public class ITRedisDistributedMapCacheClientService {
}
@Test
-public void testStandaloneRedisWithAuthentication() throws
InitializationException, IOException {
+public void testStandaloneRedisWithUsernameAndPasswordAuthentication()
throws InitializationException, IOException {
+final String redisUsername = "foo";
final String redisPassword = "foobared";
-final int redisPort = setupStandaloneRedis(redisPassword).port;
+final int redisPort = setupStandaloneRedis(redisUsername,
redisPassword).port;
+setUpRedisConnectionPool(portsToConnectionString(redisPort), pool -> {
+testRunner.setProperty(redisConnectionPool, RedisUtils.USERNAME,
redisUsername);
+testRunner.setProperty(redisConnectionPool, RedisUtils.PASSWORD,
redisPassword);
+});
+setupRedisMapCacheClientService();
+
+executeProcessor();
+}
+
+
+@Test
+public void testStandaloneRedisWithDefaultUserAuthentication() throws
InitializationException, IOException {
+final String redisPassword = "foobared";
+final int redisPort = setupStandaloneRedis(null, redisPassword).port;
setUpRedisConnectionPool(portsToConnectionString(redisPort), pool -> {
testRunner.setProperty(redisConnectionPool, RedisUtils.PASSWORD,
redisPassword);
});
@@ -111,15 +126,15 @@ public class ITRedisDistributedMapCacheClientService {
@Test
public void testSentinelRedis() throws InitializationException,
IOException {
-RedisContainer redisMasterContainer = setupStandaloneRedis(null);
+RedisContainer redisMasterContainer = setupStandaloneRedis(null,null);
String masterHost = "127.0.0.1";
int masterPort = redisMasterContainer.port;
-setUpRedisReplica(masterHost, masterPort, null);
-setUpRedisReplica(masterHost, masterPort, null);
+setUpRedisReplica(masterHost, masterPort, null, null);
+setUpRedisReplica(masterHost, masterPort, null, null);
-int sentinelAPort = setUpSentinel(masterHost, masterPort, null, 2,
null).port;
-int sentinelBPort = setUpSentinel(masterHost, masterPort, null, 2,
null).port;
-int sentinelCPort = setUpSentinel(masterHost, masterPort, null, 2,
null).port;
+int sentinelAPort = setUpSentinel(masterHost, masterPort, null, null,
2, null, null).port;
+int sentinelBPort = setUpSentinel(masterHost, masterPort, null, null,
2, null, null).port;
+int sentinelCPort = setUpSentinel(masterHost, masterPort, null, null,
2, null, null).port;
set
[nifi] branch main updated: NIFI-12166 Add repo usage to the monitoring endpoint (#7836)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new d2aec89738 NIFI-12166 Add repo usage to the monitoring endpoint (#7836)
d2aec89738 is described below
commit d2aec89738409c731328b9a1e5faad626c0b8e65
Author: timeabarna <38653567+timeaba...@users.noreply.github.com>
AuthorDate: Tue Oct 10 20:55:33 2023 +0200
NIFI-12166 Add repo usage to the monitoring endpoint (#7836)
* NIFI-12166 Add repo usage to the monitoring endpoint
---
.../org/apache/nifi/diagnostics/StorageUsage.java | 0
.../org/apache/nifi/reporting/EventAccess.java | 21 +
.../java/org/apache/nifi/util/MockEventAccess.java | 18 +
.../nifi/prometheus/util/NiFiMetricsRegistry.java | 45 +++
.../prometheus/util/PrometheusMetricsUtil.java | 32
.../org/apache/nifi/controller/FlowController.java | 3 +-
.../apache/nifi/reporting/StandardEventAccess.java | 90 +-
.../apache/nifi/web/StandardNiFiServiceFacade.java | 9 +++
.../nifi/web/controller/ControllerFacade.java | 25 ++
.../prometheus/TestPrometheusMetricsUtil.java | 58 ++
.../controller/reporting/StatelessEventAccess.java | 17
11 files changed, 316 insertions(+), 2 deletions(-)
diff --git
a/nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java
b/nifi-api/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java
similarity index 100%
rename from
nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java
rename to nifi-api/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/EventAccess.java
b/nifi-api/src/main/java/org/apache/nifi/reporting/EventAccess.java
index 0f19fe6452..43ef613d95 100644
--- a/nifi-api/src/main/java/org/apache/nifi/reporting/EventAccess.java
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/EventAccess.java
@@ -18,8 +18,11 @@ package org.apache.nifi.reporting;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+
import org.apache.nifi.action.Action;
import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.diagnostics.StorageUsage;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventRepository;
@@ -91,4 +94,22 @@ public interface EventAccess {
*/
long getTotalBytesReceived();
+/**
+ * Returns the storage usage of all provenance repositories
+ * @return the map of all the storage usage
+ */
+Map getProvenanceRepositoryStorageUsage();
+
+/**
+ * Returns the storage usage of all content repositories
+ * @return the map of all the storage usage
+ */
+Map getContentRepositoryStorageUsage();
+
+/**
+ * Returns the storage usage of the flow file repository
+ * @return the storage usage
+ */
+StorageUsage getFlowFileRepositoryStorageUsage();
+
}
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockEventAccess.java
b/nifi-mock/src/main/java/org/apache/nifi/util/MockEventAccess.java
index 05a70f8f6f..4fcdf05bef 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockEventAccess.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockEventAccess.java
@@ -18,11 +18,14 @@ package org.apache.nifi.util;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+
import org.apache.nifi.action.Action;
import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.diagnostics.StorageUsage;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventRepository;
import org.apache.nifi.reporting.EventAccess;
@@ -119,4 +122,19 @@ public class MockEventAccess implements EventAccess {
public long getTotalBytesReceived() {
return -1;
}
+
+@Override
+public Map getProvenanceRepositoryStorageUsage() {
+return Collections.emptyMap();
+}
+
+@Override
+public Map getContentRepositoryStorageUsage() {
+return Collections.emptyMap();
+}
+
+@Override
+public StorageUsage getFlowFileRepositoryStorageUsage() {
+return null;
+}
}
diff --git
a/nifi-nar-bundles/nifi-extension-utils/nifi-prometheus-utils/src/main/java/org/apache/nifi/prometheus/util/NiFiMetricsRegistry.java
b/nifi-nar-bundles/nifi-extension-utils/nifi-prometheus-utils/src/main/java/org/apache/nifi/prometheus/util/NiFiMetricsRegistry.java
index 97eca61a8c..cc3ff65eaa 100644
---
a/nifi-nar-bundles/nifi-extension-utils/nifi-prometheus-utils/src/main/java/org/apache/nifi/prometheus/util/NiFiMetricsRegistry.java
+++
b/nifi-nar-bundles/nifi-extension-util
[nifi] branch main updated: NIFI-12140 Add optional properties to RedisConnectionPoolService for Username and Sentinel Username (#7862)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 5a42d7245b NIFI-12140 Add optional properties to
RedisConnectionPoolService for Username and Sentinel Username (#7862)
5a42d7245b is described below
commit 5a42d7245b6e852c99c068fa34f991f0c6d80b3b
Author: Bryan Bende
AuthorDate: Tue Oct 10 14:39:41 2023 -0400
NIFI-12140 Add optional properties to RedisConnectionPoolService for
Username and Sentinel Username (#7862)
---
.../ITRedisDistributedMapCacheClientService.java | 87 +-
.../nifi/redis/testcontainers/RedisContainer.java | 11 +++
.../testcontainers/RedisReplicaContainer.java | 5 ++
.../testcontainers/RedisSentinelContainer.java | 17 +
.../org/apache/nifi/redis/util/RedisConfig.java| 19 +
.../org/apache/nifi/redis/util/RedisUtils.java | 30 +++-
6 files changed, 149 insertions(+), 20 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-redis-bundle/nifi-redis-extensions/src/test/java/org/apache/nifi/redis/service/ITRedisDistributedMapCacheClientService.java
b/nifi-nar-bundles/nifi-redis-bundle/nifi-redis-extensions/src/test/java/org/apache/nifi/redis/service/ITRedisDistributedMapCacheClientService.java
index f14187f6ed..bc3961b7b1 100644
---
a/nifi-nar-bundles/nifi-redis-bundle/nifi-redis-extensions/src/test/java/org/apache/nifi/redis/service/ITRedisDistributedMapCacheClientService.java
+++
b/nifi-nar-bundles/nifi-redis-bundle/nifi-redis-extensions/src/test/java/org/apache/nifi/redis/service/ITRedisDistributedMapCacheClientService.java
@@ -88,7 +88,7 @@ public class ITRedisDistributedMapCacheClientService {
@Test
public void testStandaloneRedis() throws InitializationException,
IOException {
-int redisPort = setupStandaloneRedis(null).port;
+int redisPort = setupStandaloneRedis(null,null).port;
setUpRedisConnectionPool(portsToConnectionString(redisPort), pool -> {
// uncomment this to test using a different database index than
the default 0
// testRunner.setProperty(pool, RedisUtils.DATABASE, "1");
@@ -99,9 +99,24 @@ public class ITRedisDistributedMapCacheClientService {
}
@Test
-public void testStandaloneRedisWithAuthentication() throws
InitializationException, IOException {
+public void testStandaloneRedisWithUsernameAndPasswordAuthentication()
throws InitializationException, IOException {
+final String redisUsername = "foo";
final String redisPassword = "foobared";
-final int redisPort = setupStandaloneRedis(redisPassword).port;
+final int redisPort = setupStandaloneRedis(redisUsername,
redisPassword).port;
+setUpRedisConnectionPool(portsToConnectionString(redisPort), pool -> {
+testRunner.setProperty(redisConnectionPool, RedisUtils.USERNAME,
redisUsername);
+testRunner.setProperty(redisConnectionPool, RedisUtils.PASSWORD,
redisPassword);
+});
+setupRedisMapCacheClientService();
+
+executeProcessor();
+}
+
+
+@Test
+public void testStandaloneRedisWithDefaultUserAuthentication() throws
InitializationException, IOException {
+final String redisPassword = "foobared";
+final int redisPort = setupStandaloneRedis(null, redisPassword).port;
setUpRedisConnectionPool(portsToConnectionString(redisPort), pool -> {
testRunner.setProperty(redisConnectionPool, RedisUtils.PASSWORD,
redisPassword);
});
@@ -112,15 +127,15 @@ public class ITRedisDistributedMapCacheClientService {
@Test
public void testSentinelRedis() throws InitializationException,
IOException {
-RedisContainer redisMasterContainer = setupStandaloneRedis(null);
+RedisContainer redisMasterContainer = setupStandaloneRedis(null,null);
String masterHost = "127.0.0.1";
int masterPort = redisMasterContainer.port;
-setUpRedisReplica(masterHost, masterPort, null);
-setUpRedisReplica(masterHost, masterPort, null);
+setUpRedisReplica(masterHost, masterPort, null, null);
+setUpRedisReplica(masterHost, masterPort, null, null);
-int sentinelAPort = setUpSentinel(masterHost, masterPort, null, 2,
null).port;
-int sentinelBPort = setUpSentinel(masterHost, masterPort, null, 2,
null).port;
-int sentinelCPort = setUpSentinel(masterHost, masterPort, null, 2,
null).port;
+int sentinelAPort = setUpSentinel(masterHost, masterPort, null, null,
2, null, null).port;
+int sentinelBPort = setUpSentinel(masterHost, masterPort, null, null,
2, null, null).port;
+int sentinelCPort = setUpSentinel(masterHost, masterPort, null, null,
2, null, null).port;
setUpRedisConnectio
[nifi] branch main updated: NIFI-12186 Add ability to export a versioned reporting task snapshot (#7853)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new ae61ebb5ed NIFI-12186 Add ability to export a versioned reporting task
snapshot (#7853)
ae61ebb5ed is described below
commit ae61ebb5eda7076c6a18ba0419b614b8516faf14
Author: Bryan Bende
AuthorDate: Mon Oct 9 10:50:36 2023 -0400
NIFI-12186 Add ability to export a versioned reporting task snapshot (#7853)
* NIFI-12186 Add ability to export a versioned reporting task snapshot
- Add CLI commands and optional query param to specify specific reporting
task
---
.../nifi/flow/VersionedReportingTaskSnapshot.java | 48 +++
.../flow/mapping/NiFiRegistryFlowMapper.java | 3 +-
.../VersionedReportingTaskSnapshotMapper.java | 72 ++
.../org/apache/nifi/web/NiFiServiceFacade.java | 16 +++
.../apache/nifi/web/StandardNiFiServiceFacade.java | 49 +++
.../java/org/apache/nifi/web/api/FlowResource.java | 151 -
.../toolkit/cli/impl/client/nifi/FlowClient.java | 15 ++
.../impl/client/nifi/impl/JerseyFlowClient.java| 18 +++
.../cli/impl/command/nifi/NiFiCommandGroup.java| 4 +
.../command/nifi/flow/ExportReportingTask.java | 67 +
.../command/nifi/flow/ExportReportingTasks.java| 65 +
.../nifi/VersionedReportingTaskSnapshotResult.java | 63 +
12 files changed, 538 insertions(+), 33 deletions(-)
diff --git
a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedReportingTaskSnapshot.java
b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedReportingTaskSnapshot.java
new file mode 100644
index 00..3da3f8c302
--- /dev/null
+++
b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedReportingTaskSnapshot.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.flow;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.List;
+
+@ApiModel
+public class VersionedReportingTaskSnapshot {
+
+private List reportingTasks;
+private List controllerServices;
+
+@ApiModelProperty(value = "The controller services")
+public List getControllerServices() {
+return controllerServices;
+}
+
+public void setControllerServices(List
controllerServices) {
+this.controllerServices = controllerServices;
+}
+
+@ApiModelProperty(value = "The reporting tasks")
+public List getReportingTasks() {
+return reportingTasks;
+}
+
+public void setReportingTasks(List reportingTasks)
{
+this.reportingTasks = reportingTasks;
+}
+
+}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java
index f1dfa857fb..b0ac1b8909 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java
@@ -615,8 +615,9 @@ public class NiFiRegistryFlowMapper {
continue;
}
+// if mapping a reporting task, serviceGroupId will be
null and we don't want to produce external service references
final String serviceGroupId =
serviceNode.getProcessGroupIdentifier();
-if (!includedGroupIds.contains(serviceGroupId)) {
+if (serviceGroupId != null &&
!includedGroupIds.contains(serviceGroupId)) {
final String serviceId =
getId(serviceNode.getVersionedComponentId(), serviceNode.getIdentifier());
[nifi] branch support/nifi-1.x updated: [NIFI-12067] mock process session keeps track of flowfiles created during the session and removes them on rollback rather than putting them on the input queue (
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 6929ddecb6 [NIFI-12067] mock process session keeps track of flowfiles
created during the session and removes them on rollback rather than putting
them on the input queue (#7827)
6929ddecb6 is described below
commit 6929ddecb62b247451fc12ca7a82ca33dea7628e
Author: Eric Secules
AuthorDate: Wed Oct 4 07:48:50 2023 -0700
[NIFI-12067] mock process session keeps track of flowfiles created during
the session and removes them on rollback rather than putting them on the input
queue (#7827)
rename method to be more descriptive, fix checkstyle error for trailing
whitespace in TestMockProcessSession.java
added session.transfer call to unit tests so that they fail without the
fixes.
Co-authored-by: Eric Secules
---
.../org/apache/nifi/util/MockProcessSession.java | 48 ++
.../apache/nifi/util/TestMockProcessSession.java | 34 +++
2 files changed, 66 insertions(+), 16 deletions(-)
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
index ce0d93094e..c408c44e5b 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -69,6 +69,7 @@ public class MockProcessSession implements ProcessSession {
private final Map> transferMap = new
ConcurrentHashMap<>();
private final MockFlowFileQueue processorQueue;
private final Set beingProcessed = new HashSet<>();
+private final Set created = new HashSet<>();
private final List penalized = new ArrayList<>();
private final Processor processor;
@@ -213,6 +214,10 @@ public class MockProcessSession implements ProcessSession {
if (removedFlowFiles.remove(flowFile.getId())) {
newOwner.removedFlowFiles.add(flowFile.getId());
}
+
+if (created.remove(flowFile.getId())) {
+newOwner.created.add(flowFile.getId());
+}
}
final Set flowFileIds = flowFiles.stream()
@@ -226,8 +231,7 @@ public class MockProcessSession implements ProcessSession {
public MockFlowFile clone(FlowFile flowFile) {
flowFile = validateState(flowFile);
final MockFlowFile newFlowFile = new
MockFlowFile(sharedState.nextFlowFileId(), flowFile);
-currentVersions.put(newFlowFile.getId(), newFlowFile);
-beingProcessed.add(newFlowFile.getId());
+updateStateWithNewFlowFile(newFlowFile);
return newFlowFile;
}
@@ -242,8 +246,7 @@ public class MockProcessSession implements ProcessSession {
final byte[] newContent = Arrays.copyOfRange(((MockFlowFile)
flowFile).getData(), (int) offset, (int) (offset + size));
newFlowFile.setData(newContent);
-currentVersions.put(newFlowFile.getId(), newFlowFile);
-beingProcessed.add(newFlowFile.getId());
+updateStateWithNewFlowFile(newFlowFile);
return newFlowFile;
}
@@ -290,6 +293,7 @@ public class MockProcessSession implements ProcessSession {
beingProcessed.clear();
currentVersions.clear();
originalVersions.clear();
+created.clear();
for (final Map.Entry entry : counterMap.entrySet()) {
sharedState.adjustCounter(entry.getKey(), entry.getValue());
@@ -339,8 +343,7 @@ public class MockProcessSession implements ProcessSession {
@Override
public MockFlowFile create() {
final MockFlowFile flowFile = new
MockFlowFile(sharedState.nextFlowFileId());
-currentVersions.put(flowFile.getId(), flowFile);
-beingProcessed.add(flowFile.getId());
+updateStateWithNewFlowFile(flowFile);
return flowFile;
}
@@ -348,8 +351,7 @@ public class MockProcessSession implements ProcessSession {
public MockFlowFile create(final FlowFile flowFile) {
MockFlowFile newFlowFile = create();
newFlowFile = (MockFlowFile) inheritAttributes(flowFile, newFlowFile);
-currentVersions.put(newFlowFile.getId(), newFlowFile);
-beingProcessed.add(newFlowFile.getId());
+updateStateWithNewFlowFile(newFlowFile);
return newFlowFile;
}
@@ -357,8 +359,7 @@ public class MockProcessSession implements ProcessSession {
public MockFlowFile create(final Collection flowFiles) {
MockFlowFile newFlowFile = create();
newFlowFile = (MockFlowFile) inheritAttributes(flowFiles, newFlowFile);
-currentVersions.put(newFlowFile.getId(), newFlowFile);
-beingProcessed.add(newFlowFile.getId());
+updateStateWithNewFlowFile(ne
[nifi] branch main updated: [NIFI-12067] mock process session keeps track of flowfiles created during the session and removes them on rollback rather than putting them on the input queue (#7827)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 746dad7f46 [NIFI-12067] mock process session keeps track of flowfiles
created during the session and removes them on rollback rather than putting
them on the input queue (#7827)
746dad7f46 is described below
commit 746dad7f4692581cd11020e8414cc4e3f8fc74fe
Author: Eric Secules
AuthorDate: Wed Oct 4 07:48:50 2023 -0700
[NIFI-12067] mock process session keeps track of flowfiles created during
the session and removes them on rollback rather than putting them on the input
queue (#7827)
rename method to be more descriptive, fix checkstyle error for trailing
whitespace in TestMockProcessSession.java
added session.transfer call to unit tests so that they fail without the
fixes.
Co-authored-by: Eric Secules
---
.../org/apache/nifi/util/MockProcessSession.java | 48 ++
.../apache/nifi/util/TestMockProcessSession.java | 34 +++
2 files changed, 66 insertions(+), 16 deletions(-)
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
index 09cc1d3a5f..c04bab70fd 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -68,6 +68,7 @@ public class MockProcessSession implements ProcessSession {
private final Map> transferMap = new
ConcurrentHashMap<>();
private final MockFlowFileQueue processorQueue;
private final Set beingProcessed = new HashSet<>();
+private final Set created = new HashSet<>();
private final List penalized = new ArrayList<>();
private final Processor processor;
@@ -212,6 +213,10 @@ public class MockProcessSession implements ProcessSession {
if (removedFlowFiles.remove(flowFile.getId())) {
newOwner.removedFlowFiles.add(flowFile.getId());
}
+
+if (created.remove(flowFile.getId())) {
+newOwner.created.add(flowFile.getId());
+}
}
final Set flowFileIds = flowFiles.stream()
@@ -225,8 +230,7 @@ public class MockProcessSession implements ProcessSession {
public MockFlowFile clone(FlowFile flowFile) {
flowFile = validateState(flowFile);
final MockFlowFile newFlowFile = new
MockFlowFile(sharedState.nextFlowFileId(), flowFile);
-currentVersions.put(newFlowFile.getId(), newFlowFile);
-beingProcessed.add(newFlowFile.getId());
+updateStateWithNewFlowFile(newFlowFile);
return newFlowFile;
}
@@ -241,8 +245,7 @@ public class MockProcessSession implements ProcessSession {
final byte[] newContent = Arrays.copyOfRange(((MockFlowFile)
flowFile).getData(), (int) offset, (int) (offset + size));
newFlowFile.setData(newContent);
-currentVersions.put(newFlowFile.getId(), newFlowFile);
-beingProcessed.add(newFlowFile.getId());
+updateStateWithNewFlowFile(newFlowFile);
return newFlowFile;
}
@@ -289,6 +292,7 @@ public class MockProcessSession implements ProcessSession {
beingProcessed.clear();
currentVersions.clear();
originalVersions.clear();
+created.clear();
for (final Map.Entry entry : counterMap.entrySet()) {
sharedState.adjustCounter(entry.getKey(), entry.getValue());
@@ -338,8 +342,7 @@ public class MockProcessSession implements ProcessSession {
@Override
public MockFlowFile create() {
final MockFlowFile flowFile = new
MockFlowFile(sharedState.nextFlowFileId());
-currentVersions.put(flowFile.getId(), flowFile);
-beingProcessed.add(flowFile.getId());
+updateStateWithNewFlowFile(flowFile);
return flowFile;
}
@@ -347,8 +350,7 @@ public class MockProcessSession implements ProcessSession {
public MockFlowFile create(final FlowFile flowFile) {
MockFlowFile newFlowFile = create();
newFlowFile = (MockFlowFile) inheritAttributes(flowFile, newFlowFile);
-currentVersions.put(newFlowFile.getId(), newFlowFile);
-beingProcessed.add(newFlowFile.getId());
+updateStateWithNewFlowFile(newFlowFile);
return newFlowFile;
}
@@ -356,8 +358,7 @@ public class MockProcessSession implements ProcessSession {
public MockFlowFile create(final Collection flowFiles) {
MockFlowFile newFlowFile = create();
newFlowFile = (MockFlowFile) inheritAttributes(flowFiles, newFlowFile);
-currentVersions.put(newFlowFile.getId(), newFlowFile);
-beingProcessed.add(newFlowFile.getId());
+updateStateWithNewFlowFile(newFlowFile);
return newF
[nifi] branch support/nifi-1.x updated: NIFI-12158 MockProcessSession write methods preserves attributes (#7828)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 4354072b4a NIFI-12158 MockProcessSession write methods preserves
attributes (#7828)
4354072b4a is described below
commit 4354072b4a623f9e565887bef423f9cd13d62814
Author: Eric Secules
AuthorDate: Tue Oct 3 11:40:31 2023 -0700
NIFI-12158 MockProcessSession write methods preserves attributes (#7828)
Co-authored-by: Eric Secules
---
.../java/org/apache/nifi/util/MockProcessSession.java | 13 ++---
.../java/org/apache/nifi/util/TestMockProcessSession.java | 15 +++
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
index 2b8326ed50..ce0d93094e 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -927,16 +927,15 @@ public class MockProcessSession implements ProcessSession
{
if (!(flowFile instanceof MockFlowFile)) {
throw new IllegalArgumentException("Cannot export a flow file that
I did not create");
}
-
final MockFlowFile mockFlowFile = validateState(flowFile);
-writeRecursionSet.add(flowFile);
+writeRecursionSet.add(mockFlowFile);
final ByteArrayOutputStream baos = new ByteArrayOutputStream() {
@Override
public void close() throws IOException {
super.close();
writeRecursionSet.remove(mockFlowFile);
-final MockFlowFile newFlowFile = new
MockFlowFile(mockFlowFile.getId(), flowFile);
+final MockFlowFile newFlowFile = new
MockFlowFile(mockFlowFile.getId(), mockFlowFile);
currentVersions.put(newFlowFile.getId(), newFlowFile);
newFlowFile.setData(toByteArray());
@@ -969,12 +968,12 @@ public class MockProcessSession implements ProcessSession
{
}
@Override
-public MockFlowFile write(final FlowFile flowFile, final StreamCallback
callback) {
+public MockFlowFile write(FlowFile flowFile, final StreamCallback
callback) {
+flowFile = validateState(flowFile);
if (callback == null || flowFile == null) {
throw new IllegalArgumentException("argument cannot be null");
}
-final MockFlowFile mock = validateState(flowFile);
-
+final MockFlowFile mock = (MockFlowFile) flowFile;
final ByteArrayInputStream in = new
ByteArrayInputStream(mock.getData());
final ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -987,7 +986,7 @@ public class MockProcessSession implements ProcessSession {
writeRecursionSet.remove(flowFile);
}
-final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(),
flowFile);
+final MockFlowFile newFlowFile = new MockFlowFile(flowFile.getId(),
flowFile);
currentVersions.put(newFlowFile.getId(), newFlowFile);
newFlowFile.setData(out.toByteArray());
diff --git
a/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
b/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
index eefd4a39dd..775bc2f5ed 100644
--- a/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
+++ b/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -133,6 +134,20 @@ public class TestMockProcessSession {
assertFalse(ff1.isPenalized());
}
+@Test
+public void testAttributePreservedAfterWrite() throws IOException {
+final Processor processor = new PoorlyBehavedProcessor();
+final MockProcessSession session = new MockProcessSession(new
SharedSessionState(processor, new AtomicLong(0L)), processor, new
MockStateManager(processor));
+FlowFile ff1 = session.createFlowFile("hello, world".getBytes());
+session.putAttribute(ff1, "key1", "val1");
+session.write(ff1).close();
+session.transfer(ff1, PoorlyBehavedProcessor.REL_FAILURE);
+session.commitAsync();
+List output =
session.getFlowFilesForRelationship(PoorlyBehavedProcessor.REL_FAILURE);
+assertEquals(1, output.size());
+output.get(0).assertAttributeEquals("key1", "val1");
+}
+
protected static class PoorlyBehavedProcessor extends AbstractProcessor {
private static final Relationship REL_FAILURE = new
Relationship.Builder()
[nifi] branch main updated: NIFI-12158 MockProcessSession write methods preserves attributes (#7828)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 721628eb95 NIFI-12158 MockProcessSession write methods preserves
attributes (#7828)
721628eb95 is described below
commit 721628eb955f0f52d8ece0db2afe784c0a144008
Author: Eric Secules
AuthorDate: Tue Oct 3 11:40:31 2023 -0700
NIFI-12158 MockProcessSession write methods preserves attributes (#7828)
Co-authored-by: Eric Secules
---
.../java/org/apache/nifi/util/MockProcessSession.java | 13 ++---
.../java/org/apache/nifi/util/TestMockProcessSession.java | 15 +++
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
index 4ab99220a2..09cc1d3a5f 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -919,16 +919,15 @@ public class MockProcessSession implements ProcessSession
{
if (!(flowFile instanceof MockFlowFile)) {
throw new IllegalArgumentException("Cannot export a flow file that
I did not create");
}
-
final MockFlowFile mockFlowFile = validateState(flowFile);
-writeRecursionSet.add(flowFile);
+writeRecursionSet.add(mockFlowFile);
final ByteArrayOutputStream baos = new ByteArrayOutputStream() {
@Override
public void close() throws IOException {
super.close();
writeRecursionSet.remove(mockFlowFile);
-final MockFlowFile newFlowFile = new
MockFlowFile(mockFlowFile.getId(), flowFile);
+final MockFlowFile newFlowFile = new
MockFlowFile(mockFlowFile.getId(), mockFlowFile);
currentVersions.put(newFlowFile.getId(), newFlowFile);
newFlowFile.setData(toByteArray());
@@ -961,12 +960,12 @@ public class MockProcessSession implements ProcessSession
{
}
@Override
-public MockFlowFile write(final FlowFile flowFile, final StreamCallback
callback) {
+public MockFlowFile write(FlowFile flowFile, final StreamCallback
callback) {
+flowFile = validateState(flowFile);
if (callback == null || flowFile == null) {
throw new IllegalArgumentException("argument cannot be null");
}
-final MockFlowFile mock = validateState(flowFile);
-
+final MockFlowFile mock = (MockFlowFile) flowFile;
final ByteArrayInputStream in = new
ByteArrayInputStream(mock.getData());
final ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -979,7 +978,7 @@ public class MockProcessSession implements ProcessSession {
writeRecursionSet.remove(flowFile);
}
-final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(),
flowFile);
+final MockFlowFile newFlowFile = new MockFlowFile(flowFile.getId(),
flowFile);
currentVersions.put(newFlowFile.getId(), newFlowFile);
newFlowFile.setData(out.toByteArray());
diff --git
a/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
b/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
index eefd4a39dd..775bc2f5ed 100644
--- a/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
+++ b/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -133,6 +134,20 @@ public class TestMockProcessSession {
assertFalse(ff1.isPenalized());
}
+@Test
+public void testAttributePreservedAfterWrite() throws IOException {
+final Processor processor = new PoorlyBehavedProcessor();
+final MockProcessSession session = new MockProcessSession(new
SharedSessionState(processor, new AtomicLong(0L)), processor, new
MockStateManager(processor));
+FlowFile ff1 = session.createFlowFile("hello, world".getBytes());
+session.putAttribute(ff1, "key1", "val1");
+session.write(ff1).close();
+session.transfer(ff1, PoorlyBehavedProcessor.REL_FAILURE);
+session.commitAsync();
+List output =
session.getFlowFilesForRelationship(PoorlyBehavedProcessor.REL_FAILURE);
+assertEquals(1, output.size());
+output.get(0).assertAttributeEquals("key1", "val1");
+}
+
protected static class PoorlyBehavedProcessor extends AbstractProcessor {
private static final Relationship REL_FAILURE = new
Relationship.Builder()
[nifi] branch main updated: NIFI-12132 change default content repo archive percentage and age to 90 percent and 3 hours to better align to default user needs (#7797)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/main by this push: new 476f30813d NIFI-12132 change default content repo archive percentage and age to 90 percent and 3 hours to better align to default user needs (#7797) 476f30813d is described below commit 476f30813d0ba046539d51078381b615ed780952 Author: Joe Witt AuthorDate: Thu Sep 28 14:32:21 2023 -0700 NIFI-12132 change default content repo archive percentage and age to 90 percent and 3 hours to better align to default user needs (#7797) --- .../nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index 7766f7ba60..b916372014 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -66,8 +66,8 @@ org.apache.nifi.controller.repository.FileSystemRepository 50 KB ./content_repository -7 days - 50% +3 hours + 90% true false ../nifi-content-viewer/
[nifi] branch main updated (1046fdf79d -> b58f3aee4d)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 1046fdf79d NIFI-11938: This closes #7673. Created ConsumeSlack Processor to consume conversation history message events from Slack add b58f3aee4d NIFI-12138 added volume for nar extensions (#7804) No new revisions were added by this update. Summary of changes: nifi-docker/dockerhub/Dockerfile | 2 ++ nifi-docker/dockerhub/sh/start.sh | 3 ++- nifi-docker/dockermaven/Dockerfile | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-)
[nifi] branch main updated: NIFI-12074 updated images to base on Java 21 and added Python3 (#7785)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new ff05a5d158 NIFI-12074 updated images to base on Java 21 and added
Python3 (#7785)
ff05a5d158 is described below
commit ff05a5d158137c83034368a031b251ab066e9397
Author: Joe Witt
AuthorDate: Tue Sep 26 07:08:32 2023 -0700
NIFI-12074 updated images to base on Java 21 and added Python3 (#7785)
* NIFI-12074 updated images to base on Java 21 and added Python3
* NIFI-12074 ensure python env included, enable python extension mount
---
nifi-docker/dockerhub/DockerRun.sh | 2 +-
nifi-docker/dockerhub/Dockerfile| 11 +++---
nifi-docker/dockerhub/README.md | 5 -
nifi-docker/dockerhub/sh/start.sh | 32 ++---
nifi-docker/dockermaven/Dockerfile | 17 +--
nifi-docker/dockermaven/integration-test.sh | 6 --
nifi-docker/dockermaven/pom.xml | 1 +
nifi-docker/pom.xml | 4 ++--
8 files changed, 37 insertions(+), 41 deletions(-)
diff --git a/nifi-docker/dockerhub/DockerRun.sh
b/nifi-docker/dockerhub/DockerRun.sh
index 97d0092cfc..7554bfaa78 100755
--- a/nifi-docker/dockerhub/DockerRun.sh
+++ b/nifi-docker/dockerhub/DockerRun.sh
@@ -22,4 +22,4 @@ DOCKER_IMAGE="$(grep -Ev '(^#|^\s*$|^\s*\t*#)'
DockerImage.txt)"
NIFI_IMAGE_VERSION="$(echo "${DOCKER_IMAGE}" | cut -d : -f 2)"
echo "Running Docker Image: ${DOCKER_IMAGE}"
-docker run -d --name "nifi-${NIFI_IMAGE_VERSION}" -p 8080:8080 -p 8443:8443 -p
1:1 -p 8000:8000 -p 8181:8181 "${DOCKER_IMAGE}"
+docker run -d --name "nifi-${NIFI_IMAGE_VERSION}" -p 8443:8443 -p 1:1
-p 8000:8000 "${DOCKER_IMAGE}"
diff --git a/nifi-docker/dockerhub/Dockerfile b/nifi-docker/dockerhub/Dockerfile
index 61db651783..d77d57d349 100644
--- a/nifi-docker/dockerhub/Dockerfile
+++ b/nifi-docker/dockerhub/Dockerfile
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,8 +17,8 @@
# under the License.
#
-ARG IMAGE_NAME=eclipse-temurin
-ARG IMAGE_TAG=11-jre
+ARG IMAGE_NAME=bellsoft/liberica-openjdk-alpine
+ARG IMAGE_TAG=21
FROM ${IMAGE_NAME}:${IMAGE_TAG}
ARG MAINTAINER="Apache NiFi "
LABEL maintainer="${MAINTAINER}"
@@ -48,6 +49,8 @@ RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group
${GID} | cut -d: -
&& chown -R nifi:nifi ${NIFI_BASE_DIR} \
&& apt-get update \
&& apt-get install -y jq xmlstarlet procps unzip \
+&& apt-get install -y python3 \
+&& apt-get install -y python3-pip \
&& apt-get -y autoremove \
&& apt-get clean autoclean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
@@ -73,6 +76,7 @@ RUN curl -fSL ${MIRROR_BASE_URL}/${NIFI_BINARY_PATH} -o
${NIFI_BASE_DIR}/nifi-${
&& mkdir -p ${NIFI_HOME}/flowfile_repository \
&& mkdir -p ${NIFI_HOME}/content_repository \
&& mkdir -p ${NIFI_HOME}/provenance_repository \
+&& mkdir -p ${NIFI_HOME}/python_extensions \
&& mkdir -p ${NIFI_HOME}/state \
&& mkdir -p ${NIFI_LOG_DIR} \
&& ln -s ${NIFI_HOME} ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}
@@ -83,13 +87,14 @@ VOLUME ${NIFI_LOG_DIR} \
${NIFI_HOME}/flowfile_repository \
${NIFI_HOME}/content_repository \
${NIFI_HOME}/provenance_repository \
+ ${NIFI_HOME}/python_extensions \
${NIFI_HOME}/state
# Clear nifi-env.sh in favour of configuring all environment variables in the
Dockerfile
RUN echo "#!/bin/sh\n" > $NIFI_HOME/bin/nifi-env.sh
# Web HTTP(s) & Socket Site-to-Site Ports
-EXPOSE 8080 8443 1 8000
+EXPOSE 8443/tcp 1/tcp 8000/tcp
WORKDIR ${NIFI_HOME}
diff --git a/nifi-docker/dockerhub/README.md b/nifi-docker/dockerhub/README.md
index 6a4c75e9cc..f7113a0340 100644
--- a/nifi-docker/dockerhub/README.md
+++ b/nifi-docker/dockerhub/README.md
@@ -15,6 +15,10 @@
## Latest changes
+### 2.0.0
+
+- Changed base image to bellsoft/liberica-openjdk-debian:21 as NiFi 2.0.0
requires Java 21
+
### 1.19.0
- Changed base image to eclipse-temurin:11-jre as openjdk:8-jre is no longer
maintained
@@ -215,7 +219,6 @@ can be published to the host.
| Function | Property | Port |
|--|---|---|
-| HTTP Port| nifi.web.http.port| 8080 |
| HTTPS Port | nifi.web.https.port | 8443 |
| Remote Input
[nifi] branch main updated (520840d93b -> 017b16254b)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 520840d93b NIFI-12076 Removed ComponentLog methods with Object[] and Throwable add 017b16254b NIFI-11903 disable if windows (#7566) No new revisions were added by this update. Summary of changes: .../apache/nifi/controller/state/server/TestZooKeeperStateServer.java | 3 +++ 1 file changed, 3 insertions(+)
[nifi] branch NIFI-11903 created (now 490810452c)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch NIFI-11903 in repository https://gitbox.apache.org/repos/asf/nifi.git at 490810452c NIFI-11903 disable if windows No new revisions were added by this update.
[nifi] branch main updated (cb03d6de74 -> cbf5bb98f9)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from cb03d6de74 NIFI-11857 - CLI - recursively change version of Processors add cbf5bb98f9 NIFI-12072 moved to StackWalker API elimnating usage of SecurityManager (#7742) No new revisions were added by this update. Summary of changes: .../nifi/nar/NarThreadContextClassLoader.java | 55 +- 1 file changed, 23 insertions(+), 32 deletions(-)
[nifi] branch main updated: NIFI-12025: Fixed duplicated bulletin messages (#7670)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 6cffc78ad2 NIFI-12025: Fixed duplicated bulletin messages (#7670)
6cffc78ad2 is described below
commit 6cffc78ad225c4b6228cbc78564d6029fb470ed2
Author: Peter Turcsanyi <35004384+turcsan...@users.noreply.github.com>
AuthorDate: Fri Sep 8 19:16:44 2023 +0200
NIFI-12025: Fixed duplicated bulletin messages (#7670)
---
.../nifi/logging/repository/StandardLogRepository.java | 1 +
.../apache/nifi/logging/TestStandardLogRepository.java | 18 +-
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/repository/StandardLogRepository.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/repository/StandardLogRepository.java
index f7cefdc452..127e14756f 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/repository/StandardLogRepository.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/repository/StandardLogRepository.java
@@ -127,6 +127,7 @@ public class StandardLogRepository implements LogRepository
{
try {
final Set observersCopy = new HashSet<>(observers);
observers.clear();
+observersPerLogLevel.clear();
for (final LogObserver observer : observersCopy) {
addObserver(level, observer);
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/logging/TestStandardLogRepository.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/logging/TestStandardLogRepository.java
index 84e936acd5..3ebf090e99 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/logging/TestStandardLogRepository.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/logging/TestStandardLogRepository.java
@@ -66,11 +66,27 @@ public class TestStandardLogRepository {
MockFlowFile mockFlowFile1 = new MockFlowFile(1L);
MockFlowFile mockFlowFile2 = new MockFlowFile(2L);
-repo.addLogMessage(LogLevel.INFO, "Testing {} {} flowfiles are not
being shown in exception message", new Object[]{mockFlowFile1, mockFlowFile2});
+repo.addLogMessage(LogLevel.INFO, "Testing {} {} FlowFiles are not
being shown in exception message", new Object[]{mockFlowFile1, mockFlowFile2});
assertNull(observer.getMessages().get(0).getFlowFileUuid());
}
+@Test
+public void testLogRepositoryAfterLogLevelChange() {
+StandardLogRepository repo = new StandardLogRepository();
+MockLogObserver observer = new MockLogObserver();
+repo.addObserver(LogLevel.ERROR, observer);
+
+repo.setObservationLevel(LogLevel.ERROR);
+
+IOException exception = new IOException("exception");
+
+repo.addLogMessage(LogLevel.ERROR, "Testing {} to get exception
message <{}>", new Object[]{observer.getClass().getName(), exception});
+
+assertEquals(1, observer.getMessages().size());
+assertEquals("Testing
org.apache.nifi.logging.TestStandardLogRepository$MockLogObserver to get
exception message ", observer.getMessages().get(0).getMessage());
+}
+
private static class MockLogObserver implements LogObserver {
private final List messages = new ArrayList<>();
[nifi] branch support/nifi-1.x updated: NIFI-11434 added support for optionally allowing zip entries with dat… (#7462)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 7943c684d2 NIFI-11434 added support for optionally allowing zip
entries with dat… (#7462)
7943c684d2 is described below
commit 7943c684d226505fd3cecf2f95a7117466976e1c
Author: Joe Witt
AuthorDate: Wed Jul 12 14:09:03 2023 -0700
NIFI-11434 added support for optionally allowing zip entries with dat…
(#7462)
* NIFI-11434 added support for optionally allowing zip entries with data
descriptors
* set default value and ensured all zip tests use various configs
---
.../nifi/processors/standard/UnpackContent.java| 34 ++
.../processors/standard/TestUnpackContent.java | 8 -
2 files changed, 36 insertions(+), 6 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
index 1c7629cb97..257ba00cd1 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
@@ -162,6 +162,20 @@ public class UnpackContent extends AbstractProcessor {
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.build();
+public static final PropertyDescriptor
ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR = new PropertyDescriptor.Builder()
+.name("allow-stored-entries-wdd")
+.displayName("Allow Stored Entries With Data Descriptor")
+.description("Some zip archives contain stored entries with data
descriptors which by spec should not " +
+"happen. If this property is true they will be read
anyway. If false and such an entry is discovered " +
+"the zip will fail to process.")
+.required(true)
+.defaultValue("false")
+.sensitive(false)
+.allowableValues("true", "false")
+.dependsOn(PACKAGING_FORMAT, PackageFormat.ZIP_FORMAT.toString())
+.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+.build();
+
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("Unpacked FlowFiles are sent to this relationship")
@@ -195,6 +209,7 @@ public class UnpackContent extends AbstractProcessor {
properties.add(PACKAGING_FORMAT);
properties.add(FILE_FILTER);
properties.add(PASSWORD);
+properties.add(ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR);
this.properties = Collections.unmodifiableList(properties);
}
@@ -224,7 +239,9 @@ public class UnpackContent extends AbstractProcessor {
if (passwordProperty.isSet()) {
password = passwordProperty.getValue().toCharArray();
}
-zipUnpacker = new ZipUnpacker(fileFilter, password);
+final PropertyValue allowStoredEntriesWithDataDescriptorVal =
context.getProperty(ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR);
+final boolean allowStoredEntriesWithDataDescriptor =
allowStoredEntriesWithDataDescriptorVal.isSet() ?
allowStoredEntriesWithDataDescriptorVal.asBoolean() : false;
+zipUnpacker = new ZipUnpacker(fileFilter, password,
allowStoredEntriesWithDataDescriptor);
}
}
@@ -390,17 +407,19 @@ public class UnpackContent extends AbstractProcessor {
private static class ZipUnpacker extends Unpacker {
private final char[] password;
+private final boolean allowStoredEntriesWithDataDescriptor;
-public ZipUnpacker(final Pattern fileFilter, final char[] password) {
+public ZipUnpacker(final Pattern fileFilter, final char[] password,
final boolean allowStoredEntriesWithDataDescriptor) {
super(fileFilter);
this.password = password;
+this.allowStoredEntriesWithDataDescriptor =
allowStoredEntriesWithDataDescriptor;
}
@Override
public void unpack(final ProcessSession session, final FlowFile
source, final List unpacked) {
final String fragmentId = UUID.randomUUID().toString();
if (password == null) {
-session.read(source, new
CompressedZipInputStreamCallback(fileFilter, session, source, unpacked,
fragmentId));
+session.read(source, new
CompressedZipInputStreamCallback(fileFilter,
[nifi] branch main updated: NIFI-11434 added support for optionally allowing zip entries with dat… (#7462)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new d2c70d1d2f NIFI-11434 added support for optionally allowing zip
entries with dat… (#7462)
d2c70d1d2f is described below
commit d2c70d1d2fdd8cf3e211676e0a6cc57d811cd2b2
Author: Joe Witt
AuthorDate: Wed Jul 12 14:09:03 2023 -0700
NIFI-11434 added support for optionally allowing zip entries with dat…
(#7462)
* NIFI-11434 added support for optionally allowing zip entries with data
descriptors
* set default value and ensured all zip tests use various configs
---
.../nifi/processors/standard/UnpackContent.java| 34 ++
.../processors/standard/TestUnpackContent.java | 8 -
2 files changed, 36 insertions(+), 6 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
index 697b84ccd6..32f5e4df87 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
@@ -162,6 +162,20 @@ public class UnpackContent extends AbstractProcessor {
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.build();
+public static final PropertyDescriptor
ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR = new PropertyDescriptor.Builder()
+.name("allow-stored-entries-wdd")
+.displayName("Allow Stored Entries With Data Descriptor")
+.description("Some zip archives contain stored entries with data
descriptors which by spec should not " +
+"happen. If this property is true they will be read
anyway. If false and such an entry is discovered " +
+"the zip will fail to process.")
+.required(true)
+.defaultValue("false")
+.sensitive(false)
+.allowableValues("true", "false")
+.dependsOn(PACKAGING_FORMAT, PackageFormat.ZIP_FORMAT.toString())
+.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+.build();
+
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("Unpacked FlowFiles are sent to this relationship")
@@ -195,6 +209,7 @@ public class UnpackContent extends AbstractProcessor {
properties.add(PACKAGING_FORMAT);
properties.add(FILE_FILTER);
properties.add(PASSWORD);
+properties.add(ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR);
this.properties = Collections.unmodifiableList(properties);
}
@@ -224,7 +239,9 @@ public class UnpackContent extends AbstractProcessor {
if (passwordProperty.isSet()) {
password = passwordProperty.getValue().toCharArray();
}
-zipUnpacker = new ZipUnpacker(fileFilter, password);
+final PropertyValue allowStoredEntriesWithDataDescriptorVal =
context.getProperty(ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR);
+final boolean allowStoredEntriesWithDataDescriptor =
allowStoredEntriesWithDataDescriptorVal.isSet() ?
allowStoredEntriesWithDataDescriptorVal.asBoolean() : false;
+zipUnpacker = new ZipUnpacker(fileFilter, password,
allowStoredEntriesWithDataDescriptor);
}
}
@@ -390,17 +407,19 @@ public class UnpackContent extends AbstractProcessor {
private static class ZipUnpacker extends Unpacker {
private final char[] password;
+private final boolean allowStoredEntriesWithDataDescriptor;
-public ZipUnpacker(final Pattern fileFilter, final char[] password) {
+public ZipUnpacker(final Pattern fileFilter, final char[] password,
final boolean allowStoredEntriesWithDataDescriptor) {
super(fileFilter);
this.password = password;
+this.allowStoredEntriesWithDataDescriptor =
allowStoredEntriesWithDataDescriptor;
}
@Override
public void unpack(final ProcessSession session, final FlowFile
source, final List unpacked) {
final String fragmentId = UUID.randomUUID().toString();
if (password == null) {
-session.read(source, new
CompressedZipInputStreamCallback(fileFilter, session, source, unpacked,
fragmentId));
+session.read(source, new
CompressedZipInputStreamCallback(fileFilter,
[nifi] branch main updated: NIFI-11689 Added Maven Wrapper with Maven 3.9.2 (#7378)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 63452da617 NIFI-11689 Added Maven Wrapper with Maven 3.9.2 (#7378)
63452da617 is described below
commit 63452da61785d5a1f1ac19806aeb067d76c2fd89
Author: exceptionfactory
AuthorDate: Wed Jun 14 14:42:27 2023 -0500
NIFI-11689 Added Maven Wrapper with Maven 3.9.2 (#7378)
---
.github/workflows/ci-workflow.yml | 17 ++-
.github/workflows/system-tests.yml| 2 +-
.mvn/wrapper/maven-wrapper.properties | 18 +++
README.md | 26 ++--
mvnw | 239 ++
mvnw.cmd | 145 +
6 files changed, 426 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/ci-workflow.yml
b/.github/workflows/ci-workflow.yml
index 660234d82c..915d5b31c3 100644
--- a/.github/workflows/ci-workflow.yml
+++ b/.github/workflows/ci-workflow.yml
@@ -18,6 +18,8 @@ name: ci-workflow
on: [push, pull_request]
env:
+ MAVEN_COMMAND: ./mvnw
+ MAVEN_COMMAND_WINDOWS: ./mvnw.cmd
DEFAULT_MAVEN_OPTS: >-
-Xmx3g
-XX:ReservedCodeCacheSize=1g
@@ -36,7 +38,7 @@ env:
-Dmaven.wagon.httpconnectionManager.maxPerRoute=5
-Dmaven.wagon.httpconnectionManager.ttlSeconds=30
MAVEN_COMPILE_COMMAND: >-
-mvn test-compile
+test-compile
--show-version
--no-snapshot-updates
--no-transfer-progress
@@ -61,7 +63,7 @@ env:
-pl -:nifi-nar-provider-assembly
-pl -:nifi-py4j-integration-tests
MAVEN_VERIFY_COMMAND: >-
-mvn verify
+verify
--show-version
--no-snapshot-updates
--no-transfer-progress
@@ -109,7 +111,8 @@ jobs:
java-version: '17'
- name: Maven Build
run: >
- mvn validate
+ ${{ env.MAVEN_COMMAND }}
+ validate
--no-snapshot-updates
--no-transfer-progress
--fail-fast
@@ -147,6 +150,7 @@ jobs:
MAVEN_OPTS: >-
${{ env.COMPILE_MAVEN_OPTS }}
run: >
+ ${{ env.MAVEN_COMMAND }}
${{ env.MAVEN_COMPILE_COMMAND }}
- name: Maven Verify
env:
@@ -162,6 +166,7 @@ jobs:
${{ env.DEFAULT_MAVEN_OPTS }}
-DargLine=${env.SUREFIRE_OPTS}
run: >
+ ${{ env.MAVEN_COMMAND }}
${{ env.MAVEN_VERIFY_COMMAND }}
${{ env.MAVEN_BUILD_PROFILES }}
${{ env.MAVEN_PROJECTS }}
@@ -208,6 +213,7 @@ jobs:
MAVEN_OPTS: >-
${{ env.COMPILE_MAVEN_OPTS }}
run: >
+ ${{ env.MAVEN_COMMAND }}
${{ env.MAVEN_COMPILE_COMMAND }}
- name: Maven Verify
env:
@@ -223,6 +229,7 @@ jobs:
${{ env.DEFAULT_MAVEN_OPTS }}
-DargLine=${env.SUREFIRE_OPTS}
run: >
+ ${{ env.MAVEN_COMMAND }}
${{ env.MAVEN_VERIFY_COMMAND }}
${{ env.MAVEN_BUILD_PROFILES }}
${{ env.MAVEN_PROJECTS }}
@@ -269,6 +276,7 @@ jobs:
MAVEN_OPTS: >-
${{ env.COMPILE_MAVEN_OPTS }}
run: >
+ ${{ env.MAVEN_COMMAND }}
${{ env.MAVEN_COMPILE_COMMAND }}
- name: Maven Verify
env:
@@ -284,6 +292,7 @@ jobs:
${{ env.DEFAULT_MAVEN_OPTS }}
-DargLine=${env.SUREFIRE_OPTS}
run: >-
+ ${{ env.MAVEN_COMMAND }}
${{ env.MAVEN_VERIFY_COMMAND }}
${{ env.MAVEN_BUILD_PROFILES }}
${{ env.MAVEN_PROJECTS }}
@@ -332,6 +341,7 @@ jobs:
MAVEN_OPTS: >-
${{ env.COMPILE_MAVEN_OPTS }}
run: >
+ ${{ env.MAVEN_COMMAND_WINDOWS }}
${{ env.MAVEN_COMPILE_COMMAND }}
- name: Maven Verify
env:
@@ -347,6 +357,7 @@ jobs:
${{ env.DEFAULT_MAVEN_OPTS }}
-DargLine=${env.SUREFIRE_OPTS}
run: >-
+ ${{ env.MAVEN_COMMAND_WINDOWS }}
${{ env.MAVEN_VERIFY_COMMAND }}
${{ env.MAVEN_BUILD_PROFILES }}
${{ env.MAVEN_PROJECTS }}
diff --git a/.github/workflows/system-tests.yml
b/.github/workflows/system-tests.yml
index cafd3845c2..4004bc9bf7 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -43,7 +43,7 @@ env:
-Duser.country=GB
-Duser.timezone=UTC
MAVEN_COMMAND: >-
-mvn
+./mvnw
-V
-nsu
-ntp
diff --git a/.mvn/wrapper/maven-wrapper.properties
b/.mvn/wrapper/maven-wrapper.properties
new file mode 100644
index 00..3c6fda8c6e
--- /dev/null
+++ b/.mvn/wrapper/maven-wrapper.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional inf
[nifi] branch support/nifi-1.x updated: NIFI-11679 Refactored EncryptedRepoContentAccessIT Configuration (#7369)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new e5474f7d22 NIFI-11679 Refactored EncryptedRepoContentAccessIT
Configuration (#7369)
e5474f7d22 is described below
commit e5474f7d221644db7f2cac4d794344adbf486470
Author: exceptionfactory
AuthorDate: Tue Jun 13 10:06:45 2023 -0500
NIFI-11679 Refactored EncryptedRepoContentAccessIT Configuration (#7369)
---
.../repositories/EncryptedRepoContentAccessIT.java | 56 --
1 file changed, 52 insertions(+), 4 deletions(-)
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
index 2cfdb44473..6fb10f6fd9 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
@@ -17,17 +17,65 @@
package org.apache.nifi.tests.system.repositories;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.SecureRandom;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
public class EncryptedRepoContentAccessIT extends ContentAccessIT {
+private static final String KEYSTORE_CREDENTIALS =
UUID.randomUUID().toString();
+
+private static final String KEYSTORE_NAME = "repository.p12";
+
+private static final String KEY_ID = "primary-key";
+
+private static final String KEYSTORE_TYPE = "PKCS12";
+
+private static final int KEY_LENGTH = 32;
+
+private static final String KEY_ALGORITHM = "AES";
+
+private static Path keyStorePath;
+
+@BeforeAll
+public static void setRepositoryKeystore(@TempDir final Path
temporaryDirectory) throws GeneralSecurityException, IOException {
+keyStorePath = temporaryDirectory.resolve(KEYSTORE_NAME);
+
+final SecureRandom secureRandom = new SecureRandom();
+final byte[] key = new byte[KEY_LENGTH];
+secureRandom.nextBytes(key);
+final SecretKeySpec secretKeySpec = new SecretKeySpec(key,
KEY_ALGORITHM);
+
+final KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE);
+keyStore.load(null);
+
+final KeyStore.SecretKeyEntry secretKeyEntry = new
KeyStore.SecretKeyEntry(secretKeySpec);
+final KeyStore.PasswordProtection passwordProtection = new
KeyStore.PasswordProtection(KEYSTORE_CREDENTIALS.toCharArray());
+keyStore.setEntry(KEY_ID, secretKeyEntry, passwordProtection);
+
+try (final OutputStream outputStream =
Files.newOutputStream(keyStorePath)) {
+keyStore.store(outputStream, KEYSTORE_CREDENTIALS.toCharArray());
+}
+}
+
@Override
protected Map getNifiPropertiesOverrides() {
final Map encryptedRepoProperties = new HashMap<>();
-encryptedRepoProperties.put("nifi.content.repository.implementation",
"org.apache.nifi.controller.repository.crypto.EncryptedFileSystemRepository");
-encryptedRepoProperties.put("nifi.content.repository.encryption.key",
"0123456789ABCDEFFEDCBA9876543210");
-
encryptedRepoProperties.put("nifi.content.repository.encryption.key.id", "k1");
-
encryptedRepoProperties.put("nifi.content.repository.encryption.key.provider.implementation",
"StaticKeyProvider");
+
encryptedRepoProperties.put("nifi.repository.encryption.protocol.version", "1");
+encryptedRepoProperties.put("nifi.repository.encryption.key.id",
KEY_ID);
+encryptedRepoProperties.put("nifi.repository.encryption.key.provider",
"KEYSTORE");
+
encryptedRepoProperties.put("nifi.repository.encryption.key.provider.keystore.location",
keyStorePath.toString());
+
encryptedRepoProperties.put("nifi.repository.encryption.key.provider.keystore.password",
KEYSTORE_CREDENTIALS);
return encryptedRepoProperties;
}
}
[nifi] branch main updated: NIFI-11679 Refactored EncryptedRepoContentAccessIT Configuration (#7369)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 962dc9bc38 NIFI-11679 Refactored EncryptedRepoContentAccessIT
Configuration (#7369)
962dc9bc38 is described below
commit 962dc9bc388182d5163aeea7a8c0c2ceaaafaac5
Author: exceptionfactory
AuthorDate: Tue Jun 13 10:06:45 2023 -0500
NIFI-11679 Refactored EncryptedRepoContentAccessIT Configuration (#7369)
---
.../repositories/EncryptedRepoContentAccessIT.java | 56 --
1 file changed, 52 insertions(+), 4 deletions(-)
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
index 2cfdb44473..6fb10f6fd9 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
@@ -17,17 +17,65 @@
package org.apache.nifi.tests.system.repositories;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.SecureRandom;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
public class EncryptedRepoContentAccessIT extends ContentAccessIT {
+private static final String KEYSTORE_CREDENTIALS =
UUID.randomUUID().toString();
+
+private static final String KEYSTORE_NAME = "repository.p12";
+
+private static final String KEY_ID = "primary-key";
+
+private static final String KEYSTORE_TYPE = "PKCS12";
+
+private static final int KEY_LENGTH = 32;
+
+private static final String KEY_ALGORITHM = "AES";
+
+private static Path keyStorePath;
+
+@BeforeAll
+public static void setRepositoryKeystore(@TempDir final Path
temporaryDirectory) throws GeneralSecurityException, IOException {
+keyStorePath = temporaryDirectory.resolve(KEYSTORE_NAME);
+
+final SecureRandom secureRandom = new SecureRandom();
+final byte[] key = new byte[KEY_LENGTH];
+secureRandom.nextBytes(key);
+final SecretKeySpec secretKeySpec = new SecretKeySpec(key,
KEY_ALGORITHM);
+
+final KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE);
+keyStore.load(null);
+
+final KeyStore.SecretKeyEntry secretKeyEntry = new
KeyStore.SecretKeyEntry(secretKeySpec);
+final KeyStore.PasswordProtection passwordProtection = new
KeyStore.PasswordProtection(KEYSTORE_CREDENTIALS.toCharArray());
+keyStore.setEntry(KEY_ID, secretKeyEntry, passwordProtection);
+
+try (final OutputStream outputStream =
Files.newOutputStream(keyStorePath)) {
+keyStore.store(outputStream, KEYSTORE_CREDENTIALS.toCharArray());
+}
+}
+
@Override
protected Map getNifiPropertiesOverrides() {
final Map encryptedRepoProperties = new HashMap<>();
-encryptedRepoProperties.put("nifi.content.repository.implementation",
"org.apache.nifi.controller.repository.crypto.EncryptedFileSystemRepository");
-encryptedRepoProperties.put("nifi.content.repository.encryption.key",
"0123456789ABCDEFFEDCBA9876543210");
-
encryptedRepoProperties.put("nifi.content.repository.encryption.key.id", "k1");
-
encryptedRepoProperties.put("nifi.content.repository.encryption.key.provider.implementation",
"StaticKeyProvider");
+
encryptedRepoProperties.put("nifi.repository.encryption.protocol.version", "1");
+encryptedRepoProperties.put("nifi.repository.encryption.key.id",
KEY_ID);
+encryptedRepoProperties.put("nifi.repository.encryption.key.provider",
"KEYSTORE");
+
encryptedRepoProperties.put("nifi.repository.encryption.key.provider.keystore.location",
keyStorePath.toString());
+
encryptedRepoProperties.put("nifi.repository.encryption.key.provider.keystore.password",
KEYSTORE_CREDENTIALS);
return encryptedRepoProperties;
}
}
[nifi] branch support/nifi-1.x updated: NIFI-11680 Corrected Buffer Size Calculation for Connection Balancing (#7370)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 65436cb829 NIFI-11680 Corrected Buffer Size Calculation for Connection
Balancing (#7370)
65436cb829 is described below
commit 65436cb829472f1d2e9f4cb84ab87cf807c33825
Author: exceptionfactory
AuthorDate: Tue Jun 13 10:01:27 2023 -0500
NIFI-11680 Corrected Buffer Size Calculation for Connection Balancing
(#7370)
- Resolved BufferOverflowException in PeerChannel with Bouncy Castle
Provider
- Changed prepareForWrite() to use Destination Buffer remaining instead of
Application Buffer Size
- Changed encrypt() to Packet Buffer Size instead of Application Buffer Size
---
.../clustered/client/async/nio/PeerChannel.java | 20 +++-
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
index 5bee319089..abb69a3e87 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
@@ -162,12 +162,13 @@ public class PeerChannel implements Closeable {
while (plaintext.hasRemaining()) {
encrypt(plaintext);
-final int bytesRemaining = prepared.capacity() -
prepared.position();
-if (bytesRemaining < destinationBuffer.remaining()) {
-final ByteBuffer temp =
ByteBuffer.allocate(prepared.capacity() +
sslEngine.getSession().getApplicationBufferSize());
+final int destinationBufferRemaining =
destinationBuffer.remaining();
+if (prepared.remaining() < destinationBufferRemaining) {
+// Expand Prepared Buffer to hold current bytes plus remaining
size of Destination Buffer
+final ByteBuffer expanded =
ByteBuffer.allocate(prepared.capacity() + destinationBufferRemaining);
prepared.flip();
-temp.put(prepared);
-prepared = temp;
+expanded.put(prepared);
+prepared = expanded;
}
prepared.put(destinationBuffer);
@@ -289,11 +290,12 @@ public class PeerChannel implements Closeable {
case CLOSED:
throw new IOException("Failed to encrypt data to write to
Peer " + peerDescription + " because Peer unexpectedly closed connection");
case BUFFER_OVERFLOW:
-// destinationBuffer is not large enough. Need to increase
the size.
-final ByteBuffer tempBuffer =
ByteBuffer.allocate(destinationBuffer.capacity() +
sslEngine.getSession().getApplicationBufferSize());
+// Expand Destination Buffer using current capacity plus
encrypted Packet Buffer Size
+final int packetBufferSize =
sslEngine.getSession().getPacketBufferSize();
+final ByteBuffer expanded =
ByteBuffer.allocate(destinationBuffer.capacity() + packetBufferSize);
destinationBuffer.flip();
-tempBuffer.put(destinationBuffer);
-destinationBuffer = tempBuffer;
+expanded.put(destinationBuffer);
+destinationBuffer = expanded;
break;
case BUFFER_UNDERFLOW:
// We should never get this result on a call to
SSLEngine.wrap(), only on a call to unwrap().
[nifi] branch main updated: NIFI-11680 Corrected Buffer Size Calculation for Connection Balancing (#7370)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 9c2f15cc18 NIFI-11680 Corrected Buffer Size Calculation for Connection
Balancing (#7370)
9c2f15cc18 is described below
commit 9c2f15cc18a9fbb6e7035d3b70d5cc2036676fbc
Author: exceptionfactory
AuthorDate: Tue Jun 13 10:01:27 2023 -0500
NIFI-11680 Corrected Buffer Size Calculation for Connection Balancing
(#7370)
- Resolved BufferOverflowException in PeerChannel with Bouncy Castle
Provider
- Changed prepareForWrite() to use Destination Buffer remaining instead of
Application Buffer Size
- Changed encrypt() to Packet Buffer Size instead of Application Buffer Size
---
.../clustered/client/async/nio/PeerChannel.java | 20 +++-
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
index 5bee319089..abb69a3e87 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
@@ -162,12 +162,13 @@ public class PeerChannel implements Closeable {
while (plaintext.hasRemaining()) {
encrypt(plaintext);
-final int bytesRemaining = prepared.capacity() -
prepared.position();
-if (bytesRemaining < destinationBuffer.remaining()) {
-final ByteBuffer temp =
ByteBuffer.allocate(prepared.capacity() +
sslEngine.getSession().getApplicationBufferSize());
+final int destinationBufferRemaining =
destinationBuffer.remaining();
+if (prepared.remaining() < destinationBufferRemaining) {
+// Expand Prepared Buffer to hold current bytes plus remaining
size of Destination Buffer
+final ByteBuffer expanded =
ByteBuffer.allocate(prepared.capacity() + destinationBufferRemaining);
prepared.flip();
-temp.put(prepared);
-prepared = temp;
+expanded.put(prepared);
+prepared = expanded;
}
prepared.put(destinationBuffer);
@@ -289,11 +290,12 @@ public class PeerChannel implements Closeable {
case CLOSED:
throw new IOException("Failed to encrypt data to write to
Peer " + peerDescription + " because Peer unexpectedly closed connection");
case BUFFER_OVERFLOW:
-// destinationBuffer is not large enough. Need to increase
the size.
-final ByteBuffer tempBuffer =
ByteBuffer.allocate(destinationBuffer.capacity() +
sslEngine.getSession().getApplicationBufferSize());
+// Expand Destination Buffer using current capacity plus
encrypted Packet Buffer Size
+final int packetBufferSize =
sslEngine.getSession().getPacketBufferSize();
+final ByteBuffer expanded =
ByteBuffer.allocate(destinationBuffer.capacity() + packetBufferSize);
destinationBuffer.flip();
-tempBuffer.put(destinationBuffer);
-destinationBuffer = tempBuffer;
+expanded.put(destinationBuffer);
+destinationBuffer = expanded;
break;
case BUFFER_UNDERFLOW:
// We should never get this result on a call to
SSLEngine.wrap(), only on a call to unwrap().
[nifi] 02/02: NIFI-11471: Added a system test to verify controller service enable timeout. This closes #7242.
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 125e456a6439ed269edc74b3339070ffd95095a1
Author: Mark Payne
AuthorDate: Mon Jun 12 14:47:01 2023 -0400
NIFI-11471: Added a system test to verify controller service enable timeout.
This closes #7242.
---
.../apache/nifi/stateless/StatelessSystemIT.java | 9
.../basics/ControllerServiceLifecycleIT.java | 53 ++
2 files changed, 62 insertions(+)
diff --git
a/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/StatelessSystemIT.java
b/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/StatelessSystemIT.java
index aa4e2845b7..411704ffc8 100644
---
a/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/StatelessSystemIT.java
+++
b/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/StatelessSystemIT.java
@@ -120,9 +120,18 @@ public class StatelessSystemIT {
public String getStatusTaskInterval() {
return null;
}
+
+@Override
+public String getComponentEnableTimeout() {
+return StatelessSystemIT.this.getComponentEnableTimeout();
+}
};
}
+protected String getComponentEnableTimeout() {
+return "10 sec";
+}
+
protected Optional getContentRepoDirectory() {
return Optional.empty();
}
diff --git
a/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/basics/ControllerServiceLifecycleIT.java
b/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/basics/ControllerServiceLifecycleIT.java
new file mode 100644
index 00..bac8dddadf
--- /dev/null
+++
b/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/basics/ControllerServiceLifecycleIT.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.stateless.basics;
+
+import org.apache.nifi.flow.VersionedControllerService;
+import org.apache.nifi.stateless.StatelessSystemIT;
+import org.apache.nifi.stateless.VersionedFlowBuilder;
+import org.apache.nifi.stateless.config.StatelessConfigurationException;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+public class ControllerServiceLifecycleIT extends StatelessSystemIT {
+
+@Override
+protected String getComponentEnableTimeout() {
+return "1 sec";
+}
+
+@Test
+public void testControllerServices() throws IOException,
StatelessConfigurationException, InterruptedException {
+final VersionedFlowBuilder builder = new VersionedFlowBuilder();
+
+final VersionedControllerService sleepService =
builder.createSimpleControllerService("StandardSleepService", "SleepService");
+sleepService.setProperties(Collections.singletonMap("@OnEnabled Sleep
Time", "3 sec"));
+
+try {
+loadDataflow(builder.getFlowSnapshot());
+} catch (final IllegalStateException expected) {
+assertInstanceOf(TimeoutException.class, expected.getCause());
+}
+}
+
+}
[nifi] branch support/nifi-1.x updated (f6e1b526c9 -> 125e456a64)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch support/nifi-1.x in repository https://gitbox.apache.org/repos/asf/nifi.git from f6e1b526c9 NIFI-11672 Adjusted ListSFTP to handle incomplete File Types new 7f1dd9c960 NIFI-11471: Define new stateless configuration points Add two new properties: new 125e456a64 NIFI-11471: Added a system test to verify controller service enable timeout. This closes #7242. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../PropertiesFileEngineConfigurationParser.java | 16 +++ .../engine/StatelessEngineConfiguration.java | 16 +++ .../scheduling/StatelessProcessScheduler.java | 9 ++-- .../stateless/engine/StandardStatelessEngine.java | 10 +++- .../flow/StandardStatelessDataflowFactory.java | 7 ++- .../nifi/stateless/flow/StandardStatelessFlow.java | 13 -- .../apache/nifi/stateless/StatelessSystemIT.java | 9 .../basics/ControllerServiceLifecycleIT.java | 53 ++ 8 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/basics/ControllerServiceLifecycleIT.java
[nifi] 01/02: NIFI-11471: Define new stateless configuration points Add two new properties:
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 7f1dd9c9600f41f94b0aabf9f789bd610d93f553
Author: David Young
AuthorDate: Fri May 12 14:38:35 2023 +
NIFI-11471: Define new stateless configuration points
Add two new properties:
nifi.stateless.component.enableTimeout
nifi.stateless.processor.startTimeout
to allow configuring the StatelessEngine and ProcessScheduler.
This allows an operator to configure what kind of startup time the flow can
tolerate.
Previously these values were hard coded.
---
.../config/PropertiesFileEngineConfigurationParser.java | 16
.../stateless/engine/StatelessEngineConfiguration.java | 16
.../controller/scheduling/StatelessProcessScheduler.java | 9 ++---
.../nifi/stateless/engine/StandardStatelessEngine.java | 10 +-
.../stateless/flow/StandardStatelessDataflowFactory.java | 7 ++-
.../nifi/stateless/flow/StandardStatelessFlow.java | 13 -
6 files changed, 61 insertions(+), 10 deletions(-)
diff --git
a/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/config/PropertiesFileEngineConfigurationParser.java
b/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/config/PropertiesFileEngineConfigurationParser.java
index 499e6d8a53..65ccd00384 100644
---
a/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/config/PropertiesFileEngineConfigurationParser.java
+++
b/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/config/PropertiesFileEngineConfigurationParser.java
@@ -53,6 +53,9 @@ public class PropertiesFileEngineConfigurationParser {
private static final String CONTENT_REPO_DIRECTORY = PREFIX +
"content.repository.directory";
private static final String STATUS_TASK_INTERVAL = PREFIX +
"status.task.interval";
+private static final String COMPONENT_ENABLE_TIMEOUT = PREFIX +
"component.enableTimeout";
+private static final String PROCESSOR_START_TIMEOUT = PREFIX +
"processor.startTimeout";
+
private static final String TRUSTSTORE_FILE = PREFIX +
"security.truststore";
private static final String TRUSTSTORE_TYPE = PREFIX +
"security.truststoreType";
private static final String TRUSTSTORE_PASSWORD = PREFIX +
"security.truststorePasswd";
@@ -111,6 +114,9 @@ public class PropertiesFileEngineConfigurationParser {
final String statusTaskInterval =
properties.getProperty(STATUS_TASK_INTERVAL, "1 min");
+final String processorStartTimeout =
properties.getProperty(PROCESSOR_START_TIMEOUT, "10 secs");
+final String componentEnableTimeout =
properties.getProperty(COMPONENT_ENABLE_TIMEOUT, "10 secs");
+
return new StatelessEngineConfiguration() {
@Override
public File getWorkingDirectory() {
@@ -161,6 +167,16 @@ public class PropertiesFileEngineConfigurationParser {
public String getStatusTaskInterval() {
return statusTaskInterval;
}
+
+@Override
+public String getProcessorStartTimeout() {
+return processorStartTimeout;
+}
+
+@Override
+public String getComponentEnableTimeout() {
+return componentEnableTimeout;
+}
};
}
diff --git
a/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/engine/StatelessEngineConfiguration.java
b/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/engine/StatelessEngineConfiguration.java
index 1a0bd4dd71..e15cfc18ef 100644
---
a/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/engine/StatelessEngineConfiguration.java
+++
b/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/engine/StatelessEngineConfiguration.java
@@ -89,4 +89,20 @@ public interface StatelessEngineConfiguration {
* A null value indicates that no status tasks are scheduled.
*/
String getStatusTaskInterval();
+
+/**
+ * @return a String representing the length of time that the process
scheduler should wait for a process to start
+ * Defaults to "10 secs"
+ */
+default String getProcessorStartTimeout() {
+ return "10 secs";
+}
+
+/**
+ * @return a String representing the length of time that the
StatelessEngine should wait for a component to enable
+ * Defaults to "10 secs"
+ */
+default String getComponentEnableTimeout() {
+return "10 sec";
+}
}
diff --git
a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/controller/sched
[nifi] 02/02: NIFI-11471: Added a system test to verify controller service enable timeout. This closes #7242.
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 15f8f872f64a0f30170218fa44c24dbfa63083ae
Author: Mark Payne
AuthorDate: Mon Jun 12 14:47:01 2023 -0400
NIFI-11471: Added a system test to verify controller service enable timeout.
This closes #7242.
---
.../apache/nifi/stateless/StatelessSystemIT.java | 9
.../basics/ControllerServiceLifecycleIT.java | 53 ++
2 files changed, 62 insertions(+)
diff --git
a/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/StatelessSystemIT.java
b/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/StatelessSystemIT.java
index aa4e2845b7..411704ffc8 100644
---
a/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/StatelessSystemIT.java
+++
b/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/StatelessSystemIT.java
@@ -120,9 +120,18 @@ public class StatelessSystemIT {
public String getStatusTaskInterval() {
return null;
}
+
+@Override
+public String getComponentEnableTimeout() {
+return StatelessSystemIT.this.getComponentEnableTimeout();
+}
};
}
+protected String getComponentEnableTimeout() {
+return "10 sec";
+}
+
protected Optional getContentRepoDirectory() {
return Optional.empty();
}
diff --git
a/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/basics/ControllerServiceLifecycleIT.java
b/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/basics/ControllerServiceLifecycleIT.java
new file mode 100644
index 00..bac8dddadf
--- /dev/null
+++
b/nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/basics/ControllerServiceLifecycleIT.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.stateless.basics;
+
+import org.apache.nifi.flow.VersionedControllerService;
+import org.apache.nifi.stateless.StatelessSystemIT;
+import org.apache.nifi.stateless.VersionedFlowBuilder;
+import org.apache.nifi.stateless.config.StatelessConfigurationException;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+public class ControllerServiceLifecycleIT extends StatelessSystemIT {
+
+@Override
+protected String getComponentEnableTimeout() {
+return "1 sec";
+}
+
+@Test
+public void testControllerServices() throws IOException,
StatelessConfigurationException, InterruptedException {
+final VersionedFlowBuilder builder = new VersionedFlowBuilder();
+
+final VersionedControllerService sleepService =
builder.createSimpleControllerService("StandardSleepService", "SleepService");
+sleepService.setProperties(Collections.singletonMap("@OnEnabled Sleep
Time", "3 sec"));
+
+try {
+loadDataflow(builder.getFlowSnapshot());
+} catch (final IllegalStateException expected) {
+assertInstanceOf(TimeoutException.class, expected.getCause());
+}
+}
+
+}
[nifi] branch main updated (45e8226f89 -> 15f8f872f6)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 45e8226f89 NIFI-11672 Adjusted ListSFTP to handle incomplete File Types new ea5b303917 NIFI-11471: Define new stateless configuration points Add two new properties: new 15f8f872f6 NIFI-11471: Added a system test to verify controller service enable timeout. This closes #7242. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../PropertiesFileEngineConfigurationParser.java | 16 +++ .../engine/StatelessEngineConfiguration.java | 16 +++ .../scheduling/StatelessProcessScheduler.java | 9 ++-- .../stateless/engine/StandardStatelessEngine.java | 10 +++- .../flow/StandardStatelessDataflowFactory.java | 7 ++- .../nifi/stateless/flow/StandardStatelessFlow.java | 13 -- .../apache/nifi/stateless/StatelessSystemIT.java | 9 .../basics/ControllerServiceLifecycleIT.java | 53 ++ 8 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 nifi-system-tests/nifi-stateless-system-test-suite/src/test/java/org/apache/nifi/stateless/basics/ControllerServiceLifecycleIT.java
[nifi] 01/02: NIFI-11471: Define new stateless configuration points Add two new properties:
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit ea5b30391799050ac50df914584dd3d390bc58f5
Author: David Young
AuthorDate: Fri May 12 14:38:35 2023 +
NIFI-11471: Define new stateless configuration points
Add two new properties:
nifi.stateless.component.enableTimeout
nifi.stateless.processor.startTimeout
to allow configuring the StatelessEngine and ProcessScheduler.
This allows an operator to configure what kind of startup time the flow can
tolerate.
Previously these values were hard coded.
---
.../config/PropertiesFileEngineConfigurationParser.java | 16
.../stateless/engine/StatelessEngineConfiguration.java | 16
.../controller/scheduling/StatelessProcessScheduler.java | 9 ++---
.../nifi/stateless/engine/StandardStatelessEngine.java | 10 +-
.../stateless/flow/StandardStatelessDataflowFactory.java | 7 ++-
.../nifi/stateless/flow/StandardStatelessFlow.java | 13 -
6 files changed, 61 insertions(+), 10 deletions(-)
diff --git
a/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/config/PropertiesFileEngineConfigurationParser.java
b/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/config/PropertiesFileEngineConfigurationParser.java
index 499e6d8a53..65ccd00384 100644
---
a/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/config/PropertiesFileEngineConfigurationParser.java
+++
b/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/config/PropertiesFileEngineConfigurationParser.java
@@ -53,6 +53,9 @@ public class PropertiesFileEngineConfigurationParser {
private static final String CONTENT_REPO_DIRECTORY = PREFIX +
"content.repository.directory";
private static final String STATUS_TASK_INTERVAL = PREFIX +
"status.task.interval";
+private static final String COMPONENT_ENABLE_TIMEOUT = PREFIX +
"component.enableTimeout";
+private static final String PROCESSOR_START_TIMEOUT = PREFIX +
"processor.startTimeout";
+
private static final String TRUSTSTORE_FILE = PREFIX +
"security.truststore";
private static final String TRUSTSTORE_TYPE = PREFIX +
"security.truststoreType";
private static final String TRUSTSTORE_PASSWORD = PREFIX +
"security.truststorePasswd";
@@ -111,6 +114,9 @@ public class PropertiesFileEngineConfigurationParser {
final String statusTaskInterval =
properties.getProperty(STATUS_TASK_INTERVAL, "1 min");
+final String processorStartTimeout =
properties.getProperty(PROCESSOR_START_TIMEOUT, "10 secs");
+final String componentEnableTimeout =
properties.getProperty(COMPONENT_ENABLE_TIMEOUT, "10 secs");
+
return new StatelessEngineConfiguration() {
@Override
public File getWorkingDirectory() {
@@ -161,6 +167,16 @@ public class PropertiesFileEngineConfigurationParser {
public String getStatusTaskInterval() {
return statusTaskInterval;
}
+
+@Override
+public String getProcessorStartTimeout() {
+return processorStartTimeout;
+}
+
+@Override
+public String getComponentEnableTimeout() {
+return componentEnableTimeout;
+}
};
}
diff --git
a/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/engine/StatelessEngineConfiguration.java
b/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/engine/StatelessEngineConfiguration.java
index 1a0bd4dd71..e15cfc18ef 100644
---
a/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/engine/StatelessEngineConfiguration.java
+++
b/nifi-stateless/nifi-stateless-api/src/main/java/org/apache/nifi/stateless/engine/StatelessEngineConfiguration.java
@@ -89,4 +89,20 @@ public interface StatelessEngineConfiguration {
* A null value indicates that no status tasks are scheduled.
*/
String getStatusTaskInterval();
+
+/**
+ * @return a String representing the length of time that the process
scheduler should wait for a process to start
+ * Defaults to "10 secs"
+ */
+default String getProcessorStartTimeout() {
+ return "10 secs";
+}
+
+/**
+ * @return a String representing the length of time that the
StatelessEngine should wait for a component to enable
+ * Defaults to "10 secs"
+ */
+default String getComponentEnableTimeout() {
+return "10 sec";
+}
}
diff --git
a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/controller/scheduling/State
[nifi] branch support/nifi-1.x updated: NIFI-11567: Auto-reload database file in GeoEnrichIP processors (#7266)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 9949f079e1 NIFI-11567: Auto-reload database file in GeoEnrichIP
processors (#7266)
9949f079e1 is described below
commit 9949f079e1cb2acdc82ccdb80f03eba42d99253b
Author: Matt Burgess
AuthorDate: Mon Jun 5 13:54:17 2023 -0400
NIFI-11567: Auto-reload database file in GeoEnrichIP processors (#7266)
NIFI-11567: Auto-reload database file in GeoEnrichIP processors
---
.../apache/nifi/processors/AbstractEnrichIP.java | 39 -
.../org/apache/nifi/processors/GeoEnrichIP.java| 68 --
.../apache/nifi/processors/GeoEnrichIPRecord.java | 35 ++-
.../apache/nifi/processors/TestGeoEnrichIP.java| 4 ++
.../nifi/processors/TestGeoEnrichIPRecord.java | 3 +
5 files changed, 128 insertions(+), 21 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/AbstractEnrichIP.java
b/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/AbstractEnrichIP.java
index ab0fb9faaf..c29553121b 100644
---
a/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/AbstractEnrichIP.java
+++
b/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/AbstractEnrichIP.java
@@ -30,9 +30,12 @@ import
org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.util.StopWatch;
+import org.apache.nifi.util.file.monitor.LastModifiedMonitor;
+import org.apache.nifi.util.file.monitor.SynchronousFileWatcher;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -40,6 +43,9 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
public abstract class AbstractEnrichIP extends AbstractProcessor {
@@ -77,6 +83,13 @@ public abstract class AbstractEnrichIP extends
AbstractProcessor {
private Set relationships;
private List propertyDescriptors;
final AtomicReference databaseReaderRef = new
AtomicReference<>(null);
+private volatile SynchronousFileWatcher watcher;
+
+private final ReadWriteLock dbReadWriteLock = new ReentrantReadWriteLock();
+
+private volatile File dbFile;
+
+private volatile boolean needsReload = true;
@Override
public Set getRelationships() {
@@ -90,7 +103,12 @@ public abstract class AbstractEnrichIP extends
AbstractProcessor {
@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
-final File dbFile =
context.getProperty(GEO_DATABASE_FILE).evaluateAttributeExpressions().asResource().asFile();
+dbFile =
context.getProperty(GEO_DATABASE_FILE).evaluateAttributeExpressions().asResource().asFile();
+this.watcher = new SynchronousFileWatcher(Paths.get(dbFile.toURI()),
new LastModifiedMonitor(), 3L);
+loadDatabaseFile();
+}
+
+protected void loadDatabaseFile() throws IOException {
final StopWatch stopWatch = new StopWatch(true);
final DatabaseReader reader = new
DatabaseReader.Builder(dbFile).build();
stopWatch.stop();
@@ -119,4 +137,23 @@ public abstract class AbstractEnrichIP extends
AbstractProcessor {
this.propertyDescriptors = Collections.unmodifiableList(props);
}
+protected SynchronousFileWatcher getWatcher() {
+return watcher;
+}
+
+protected Lock getDbWriteLock() {
+return dbReadWriteLock.writeLock();
+}
+
+protected Lock getDbReadLock() {
+return dbReadWriteLock.readLock();
+}
+
+protected boolean isNeedsReload() {
+return needsReload;
+}
+
+protected void setNeedsReload(final boolean needsReload) {
+this.needsReload = needsReload;
+}
}
diff --git
a/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/GeoEnrichIP.java
b/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/GeoEnrichIP.java
index 763bd5b7ff..f30d414218 100644
---
a/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/GeoEnrichIP.java
+++
b/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/proc
[nifi] branch main updated: NIFI-11567: Auto-reload database file in GeoEnrichIP processors (#7266)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new d200186096 NIFI-11567: Auto-reload database file in GeoEnrichIP
processors (#7266)
d200186096 is described below
commit d200186096710096d566c871bf6dbc6e0de7484e
Author: Matt Burgess
AuthorDate: Mon Jun 5 13:54:17 2023 -0400
NIFI-11567: Auto-reload database file in GeoEnrichIP processors (#7266)
NIFI-11567: Auto-reload database file in GeoEnrichIP processors
---
.../apache/nifi/processors/AbstractEnrichIP.java | 39 -
.../org/apache/nifi/processors/GeoEnrichIP.java| 68 --
.../apache/nifi/processors/GeoEnrichIPRecord.java | 35 ++-
.../apache/nifi/processors/TestGeoEnrichIP.java| 4 ++
.../nifi/processors/TestGeoEnrichIPRecord.java | 3 +
5 files changed, 128 insertions(+), 21 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/AbstractEnrichIP.java
b/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/AbstractEnrichIP.java
index ab0fb9faaf..c29553121b 100644
---
a/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/AbstractEnrichIP.java
+++
b/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/AbstractEnrichIP.java
@@ -30,9 +30,12 @@ import
org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.util.StopWatch;
+import org.apache.nifi.util.file.monitor.LastModifiedMonitor;
+import org.apache.nifi.util.file.monitor.SynchronousFileWatcher;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -40,6 +43,9 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
public abstract class AbstractEnrichIP extends AbstractProcessor {
@@ -77,6 +83,13 @@ public abstract class AbstractEnrichIP extends
AbstractProcessor {
private Set relationships;
private List propertyDescriptors;
final AtomicReference databaseReaderRef = new
AtomicReference<>(null);
+private volatile SynchronousFileWatcher watcher;
+
+private final ReadWriteLock dbReadWriteLock = new ReentrantReadWriteLock();
+
+private volatile File dbFile;
+
+private volatile boolean needsReload = true;
@Override
public Set getRelationships() {
@@ -90,7 +103,12 @@ public abstract class AbstractEnrichIP extends
AbstractProcessor {
@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
-final File dbFile =
context.getProperty(GEO_DATABASE_FILE).evaluateAttributeExpressions().asResource().asFile();
+dbFile =
context.getProperty(GEO_DATABASE_FILE).evaluateAttributeExpressions().asResource().asFile();
+this.watcher = new SynchronousFileWatcher(Paths.get(dbFile.toURI()),
new LastModifiedMonitor(), 3L);
+loadDatabaseFile();
+}
+
+protected void loadDatabaseFile() throws IOException {
final StopWatch stopWatch = new StopWatch(true);
final DatabaseReader reader = new
DatabaseReader.Builder(dbFile).build();
stopWatch.stop();
@@ -119,4 +137,23 @@ public abstract class AbstractEnrichIP extends
AbstractProcessor {
this.propertyDescriptors = Collections.unmodifiableList(props);
}
+protected SynchronousFileWatcher getWatcher() {
+return watcher;
+}
+
+protected Lock getDbWriteLock() {
+return dbReadWriteLock.writeLock();
+}
+
+protected Lock getDbReadLock() {
+return dbReadWriteLock.readLock();
+}
+
+protected boolean isNeedsReload() {
+return needsReload;
+}
+
+protected void setNeedsReload(final boolean needsReload) {
+this.needsReload = needsReload;
+}
}
diff --git
a/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/GeoEnrichIP.java
b/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/GeoEnrichIP.java
index 763bd5b7ff..f30d414218 100644
---
a/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/GeoEnrichIP.java
+++
b/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/GeoEnrichIP.java
@@ -16,6
[nifi] branch support/nifi-1.x updated: NIFI-11392 for CRON scheduled components, improved cancellation of … (#7232)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 8143a12715 NIFI-11392 for CRON scheduled components, improved
cancellation of … (#7232)
8143a12715 is described below
commit 8143a127151a8b0ae4a12b3f300babe48a36c384
Author: Michael Moser
AuthorDate: Tue May 30 11:09:06 2023 -0400
NIFI-11392 for CRON scheduled components, improved cancellation of … (#7232)
NIFI-11392 for CRON scheduled components, improved cancellation of futures
for thread cleanup
---
.../scheduling/QuartzSchedulingAgent.java | 75 ++
1 file changed, 33 insertions(+), 42 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
index 3b15ee45fb..beabd265e0 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
@@ -25,16 +25,15 @@ import org.apache.nifi.engine.FlowEngine;
import org.apache.nifi.processor.exception.ProcessException;
import org.quartz.CronExpression;
-import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
public class QuartzSchedulingAgent extends AbstractTimeBasedSchedulingAgent {
-private final Map> canceledTriggers = new
HashMap<>();
+private final Map>> quartzFutures
= new HashMap<>();
public QuartzSchedulingAgent(final FlowController flowController, final
FlowEngine flowEngine, final RepositoryContextFactory contextFactory) {
super(flowEngine, flowController, contextFactory);
@@ -42,12 +41,19 @@ public class QuartzSchedulingAgent extends
AbstractTimeBasedSchedulingAgent {
@Override
public void shutdown() {
+quartzFutures.values().forEach(map -> map.values().forEach(future -> {
+if (!future.isCancelled()) {
+// stop scheduling to run and interrupt currently running
tasks.
+future.cancel(true);
+}
+}));
+flowEngine.shutdown();
}
@Override
public void doSchedule(final ReportingTaskNode taskNode, final
LifecycleState scheduleState) {
-final List existingTriggers =
canceledTriggers.get(taskNode);
-if (existingTriggers != null) {
+final Map> componentFuturesMap =
quartzFutures.computeIfAbsent(taskNode, k -> new HashMap<>());
+if (!componentFuturesMap.values().isEmpty()) {
throw new IllegalStateException("Cannot schedule " +
taskNode.getReportingTask().getIdentifier() + " because it is already scheduled
to run");
}
@@ -61,7 +67,6 @@ public class QuartzSchedulingAgent extends
AbstractTimeBasedSchedulingAgent {
final ReportingTaskWrapper taskWrapper = new
ReportingTaskWrapper(taskNode, scheduleState,
flowController.getExtensionManager());
-final AtomicBoolean canceled = new AtomicBoolean(false);
final Date initialDate = cronExpression.getTimeAfter(new Date());
final long initialDelay = initialDate.getTime() -
System.currentTimeMillis();
@@ -71,37 +76,30 @@ public class QuartzSchedulingAgent extends
AbstractTimeBasedSchedulingAgent {
@Override
public void run() {
-if (canceled.get()) {
-return;
-}
-
taskWrapper.run();
-if (canceled.get()) {
-return;
-}
-
nextSchedule = getNextSchedule(nextSchedule, cronExpression);
final long delay = getDelay(nextSchedule);
logger.debug("Finished running Reporting Task {}; next
scheduled time is at {} after a delay of {} milliseconds", taskNode,
nextSchedule, delay);
-flowEngine.schedule(this, delay, TimeUnit.MILLISECONDS);
+final ScheduledFuture newFuture = flowEngine.schedule(this,
delay, TimeUnit.MILLISECONDS);
+final ScheduledFuture oldFuture =
componentFuturesMap.put(0, newFuture);
+scheduleState.replaceFuture(oldFuture, newFuture);
[nifi] branch main updated: NIFI-11392 for CRON scheduled components, improved cancellation of … (#7232)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new eca4f5d68f NIFI-11392 for CRON scheduled components, improved
cancellation of … (#7232)
eca4f5d68f is described below
commit eca4f5d68ffe62e4f4671847b8e1def4c53b0e2d
Author: Michael Moser
AuthorDate: Tue May 30 11:09:06 2023 -0400
NIFI-11392 for CRON scheduled components, improved cancellation of … (#7232)
NIFI-11392 for CRON scheduled components, improved cancellation of futures
for thread cleanup
---
.../scheduling/QuartzSchedulingAgent.java | 75 ++
1 file changed, 33 insertions(+), 42 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
index 3b15ee45fb..beabd265e0 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
@@ -25,16 +25,15 @@ import org.apache.nifi.engine.FlowEngine;
import org.apache.nifi.processor.exception.ProcessException;
import org.quartz.CronExpression;
-import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
public class QuartzSchedulingAgent extends AbstractTimeBasedSchedulingAgent {
-private final Map> canceledTriggers = new
HashMap<>();
+private final Map>> quartzFutures
= new HashMap<>();
public QuartzSchedulingAgent(final FlowController flowController, final
FlowEngine flowEngine, final RepositoryContextFactory contextFactory) {
super(flowEngine, flowController, contextFactory);
@@ -42,12 +41,19 @@ public class QuartzSchedulingAgent extends
AbstractTimeBasedSchedulingAgent {
@Override
public void shutdown() {
+quartzFutures.values().forEach(map -> map.values().forEach(future -> {
+if (!future.isCancelled()) {
+// stop scheduling to run and interrupt currently running
tasks.
+future.cancel(true);
+}
+}));
+flowEngine.shutdown();
}
@Override
public void doSchedule(final ReportingTaskNode taskNode, final
LifecycleState scheduleState) {
-final List existingTriggers =
canceledTriggers.get(taskNode);
-if (existingTriggers != null) {
+final Map> componentFuturesMap =
quartzFutures.computeIfAbsent(taskNode, k -> new HashMap<>());
+if (!componentFuturesMap.values().isEmpty()) {
throw new IllegalStateException("Cannot schedule " +
taskNode.getReportingTask().getIdentifier() + " because it is already scheduled
to run");
}
@@ -61,7 +67,6 @@ public class QuartzSchedulingAgent extends
AbstractTimeBasedSchedulingAgent {
final ReportingTaskWrapper taskWrapper = new
ReportingTaskWrapper(taskNode, scheduleState,
flowController.getExtensionManager());
-final AtomicBoolean canceled = new AtomicBoolean(false);
final Date initialDate = cronExpression.getTimeAfter(new Date());
final long initialDelay = initialDate.getTime() -
System.currentTimeMillis();
@@ -71,37 +76,30 @@ public class QuartzSchedulingAgent extends
AbstractTimeBasedSchedulingAgent {
@Override
public void run() {
-if (canceled.get()) {
-return;
-}
-
taskWrapper.run();
-if (canceled.get()) {
-return;
-}
-
nextSchedule = getNextSchedule(nextSchedule, cronExpression);
final long delay = getDelay(nextSchedule);
logger.debug("Finished running Reporting Task {}; next
scheduled time is at {} after a delay of {} milliseconds", taskNode,
nextSchedule, delay);
-flowEngine.schedule(this, delay, TimeUnit.MILLISECONDS);
+final ScheduledFuture newFuture = flowEngine.schedule(this,
delay, TimeUnit.MILLISECONDS);
+final ScheduledFuture oldFuture =
componentFuturesMap.put(0, newFuture);
+scheduleState.replaceFuture(oldFuture, newFuture);
}
};
-fin
[nifi] branch support/nifi-1.x updated: NIFI-11109 Allow registry client's class in flow.xml/json to not change (#7273)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new e18886fcb3 NIFI-11109 Allow registry client's class in flow.xml/json
to not change (#7273)
e18886fcb3 is described below
commit e18886fcb361ee09ea5af7c68b80c87bd11181a9
Author: NissimShiman <57680836+nissimshi...@users.noreply.github.com>
AuthorDate: Wed May 24 13:32:31 2023 -0400
NIFI-11109 Allow registry client's class in flow.xml/json to not change
(#7273)
when underlying nar for nifi-flow-registry-client-nar is missing
---
.../src/main/java/org/apache/nifi/controller/ExtensionBuilder.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/ExtensionBuilder.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/ExtensionBuilder.java
index d978c29055..ed4e25651a 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/ExtensionBuilder.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/ExtensionBuilder.java
@@ -513,7 +513,7 @@ public class ExtensionBuilder {
validationContextFactory,
serviceProvider,
componentType,
-simpleClassName,
+type,
componentVarRegistry,
reloadComponent,
extensionManager,
[nifi] branch main updated (ca2a829d47 -> a59ea4a135)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from ca2a829d47 NIFI-11584 Improved MergeContent stream handling add a59ea4a135 NIFI-11109 Allow registry client's class in flow.xml/json to not change (#7273) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/nifi/controller/ExtensionBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[nifi] branch support/nifi-1.x updated: NIFI-11436 Fix NPE during updateFlow when called from a replace request for a PG that is not under version control (#7167)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 7cb9de69c4 NIFI-11436 Fix NPE during updateFlow when called from a
replace request for a PG that is not under version control (#7167)
7cb9de69c4 is described below
commit 7cb9de69c4c51e233088e0f423d0ceeba03e55b7
Author: Bryan Bende
AuthorDate: Wed Apr 12 19:24:22 2023 -0400
NIFI-11436 Fix NPE during updateFlow when called from a replace request for
a PG that is not under version control (#7167)
---
.../apache/nifi/web/api/FlowUpdateResource.java| 3 +-
.../tests/system/pg/ReplaceProcessGroupIT.java | 102 +
2 files changed, 104 insertions(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowUpdateResource.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowUpdateResource.java
index 6b3311c572..4eb86091e4 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowUpdateResource.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowUpdateResource.java
@@ -377,8 +377,9 @@ public abstract class FlowUpdateResourcehttp://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.tests.system.pg;
+
+import org.apache.nifi.flow.Position;
+import org.apache.nifi.flow.VersionedLabel;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.registry.flow.RegisteredFlowSnapshot;
+import org.apache.nifi.tests.system.NiFiSystemIT;
+import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException;
+import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO;
+import org.apache.nifi.web.api.entity.LabelEntity;
+import org.apache.nifi.web.api.entity.ProcessGroupEntity;
+import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity;
+import org.apache.nifi.web.api.entity.ProcessGroupImportEntity;
+import org.apache.nifi.web.api.entity.ProcessGroupReplaceRequestEntity;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+public class ReplaceProcessGroupIT extends NiFiSystemIT {
+
+@Test
+public void testReplaceProcessGroup() throws NiFiClientException,
IOException, InterruptedException {
+final ProcessGroupEntity emptyProcessGroup =
getClientUtil().createProcessGroup("My Group", "root");
+
+final VersionedLabel versionedLabel = new VersionedLabel();
+versionedLabel.setIdentifier(UUID.randomUUID().toString());
+versionedLabel.setLabel("New Label");
+versionedLabel.setPosition(new Position(0, 0));
+versionedLabel.setWidth(100.00);
+versionedLabel.setHeight(100.00);
+
+final VersionedProcessGroup versionedProcessGroup = new
VersionedProcessGroup();
+versionedProcessGroup.getLabels().add(versionedLabel);
+
+final RegisteredFlowSnapshot snapshot = new RegisteredFlowSnapshot();
+snapshot.setFlowContents(versionedProcessGroup);
+
+final ProcessGroupImportEntity importEntity = new
ProcessGroupImportEntity();
+importEntity.setVersionedFlowSnapshot(snapshot);
+importEntity.setProcessGroupRevision(emptyProcessGroup.getRevision());
+
+final String pgId = emptyProcessGroup.getId();
+final ProcessGroupReplaceRequestEntity createdReplaceRequestEntity =
+
getNifiClient().getProcessGroupClient().replaceProcessGroup(pgId, importEntity);
+final String requestId =
createdReplaceRequestEntity.getRequest().getRequestId();
+
+waitForReplaceRequest(pgId, requestId);
+
+final ProcessGroupReplaceRequestEntity replaceRequest =
getNifiClient().getProcessGroupClient().getProcessGroupReplaceRequest(pgId,
requestId);
+assertNull(replaceRequest.getRequest().getFailureReason());
+
+final ProcessGroupFlowEntity replacedPgFlowEntity =
getNifiClient().getFlowClient().getProcessGroup(pgId);
+assertNotNull(replacedPgFlowEntity);
+
+final ProcessGroupFlowDTO replacedPgFlowDTO =
replacedPgFlowEntity.getPr
[nifi] branch main updated: NIFI-11436 Fix NPE during updateFlow when called from a replace request for a PG that is not under version control (#7167)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new ba93f60731 NIFI-11436 Fix NPE during updateFlow when called from a
replace request for a PG that is not under version control (#7167)
ba93f60731 is described below
commit ba93f607315b8a3a43c9fcd0c5fcb1b9668fb701
Author: Bryan Bende
AuthorDate: Wed Apr 12 19:24:22 2023 -0400
NIFI-11436 Fix NPE during updateFlow when called from a replace request for
a PG that is not under version control (#7167)
---
.../apache/nifi/web/api/FlowUpdateResource.java| 3 +-
.../tests/system/pg/ReplaceProcessGroupIT.java | 102 +
2 files changed, 104 insertions(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowUpdateResource.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowUpdateResource.java
index 6b3311c572..4eb86091e4 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowUpdateResource.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowUpdateResource.java
@@ -377,8 +377,9 @@ public abstract class FlowUpdateResourcehttp://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.tests.system.pg;
+
+import org.apache.nifi.flow.Position;
+import org.apache.nifi.flow.VersionedLabel;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.registry.flow.RegisteredFlowSnapshot;
+import org.apache.nifi.tests.system.NiFiSystemIT;
+import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException;
+import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO;
+import org.apache.nifi.web.api.entity.LabelEntity;
+import org.apache.nifi.web.api.entity.ProcessGroupEntity;
+import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity;
+import org.apache.nifi.web.api.entity.ProcessGroupImportEntity;
+import org.apache.nifi.web.api.entity.ProcessGroupReplaceRequestEntity;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+public class ReplaceProcessGroupIT extends NiFiSystemIT {
+
+@Test
+public void testReplaceProcessGroup() throws NiFiClientException,
IOException, InterruptedException {
+final ProcessGroupEntity emptyProcessGroup =
getClientUtil().createProcessGroup("My Group", "root");
+
+final VersionedLabel versionedLabel = new VersionedLabel();
+versionedLabel.setIdentifier(UUID.randomUUID().toString());
+versionedLabel.setLabel("New Label");
+versionedLabel.setPosition(new Position(0, 0));
+versionedLabel.setWidth(100.00);
+versionedLabel.setHeight(100.00);
+
+final VersionedProcessGroup versionedProcessGroup = new
VersionedProcessGroup();
+versionedProcessGroup.getLabels().add(versionedLabel);
+
+final RegisteredFlowSnapshot snapshot = new RegisteredFlowSnapshot();
+snapshot.setFlowContents(versionedProcessGroup);
+
+final ProcessGroupImportEntity importEntity = new
ProcessGroupImportEntity();
+importEntity.setVersionedFlowSnapshot(snapshot);
+importEntity.setProcessGroupRevision(emptyProcessGroup.getRevision());
+
+final String pgId = emptyProcessGroup.getId();
+final ProcessGroupReplaceRequestEntity createdReplaceRequestEntity =
+
getNifiClient().getProcessGroupClient().replaceProcessGroup(pgId, importEntity);
+final String requestId =
createdReplaceRequestEntity.getRequest().getRequestId();
+
+waitForReplaceRequest(pgId, requestId);
+
+final ProcessGroupReplaceRequestEntity replaceRequest =
getNifiClient().getProcessGroupClient().getProcessGroupReplaceRequest(pgId,
requestId);
+assertNull(replaceRequest.getRequest().getFailureReason());
+
+final ProcessGroupFlowEntity replacedPgFlowEntity =
getNifiClient().getFlowClient().getProcessGroup(pgId);
+assertNotNull(replacedPgFlowEntity);
+
+final ProcessGroupFlowDTO replacedPgFlowDTO =
replacedPgFlowEntity.getProcessGroupFlow();
[nifi] branch support/nifi-1.x updated: NIFI-11337: Update PutDatabaseRecord's Max Batch Size property documentation to reflect actual behavior (#7081)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new e7be544989 NIFI-11337: Update PutDatabaseRecord's Max Batch Size
property documentation to reflect actual behavior (#7081)
e7be544989 is described below
commit e7be5449893ee0e19d0348a902acc4438713a6f1
Author: Matt Burgess
AuthorDate: Fri Mar 24 10:22:56 2023 -0400
NIFI-11337: Update PutDatabaseRecord's Max Batch Size property
documentation to reflect actual behavior (#7081)
---
.../org/apache/nifi/processors/standard/PutDatabaseRecord.java | 7 +++
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutDatabaseRecord.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutDatabaseRecord.java
index 3a1c8ca56e..f66b75e377 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutDatabaseRecord.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutDatabaseRecord.java
@@ -337,13 +337,12 @@ public class PutDatabaseRecord extends AbstractProcessor {
static final PropertyDescriptor MAX_BATCH_SIZE = new Builder()
.name("put-db-record-max-batch-size")
.displayName("Maximum Batch Size")
-.description("Specifies maximum batch size for INSERT and UPDATE
statements. This parameter has no effect for other statements specified in
'Statement Type'."
-+ " Zero means the batch size is not limited.")
-.defaultValue("0")
+.description("Specifies maximum number of statements to be
included in each batch. Zero means the batch size is not limited, "
++ "which can cause memory usage issues for a large number
of statements.")
+.defaultValue("1000")
.required(false)
.addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
.expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
-.dependsOn(STATEMENT_TYPE, INSERT_TYPE, UPDATE_TYPE,
USE_ATTR_TYPE, USE_RECORD_PATH)
.build();
static final PropertyDescriptor DB_TYPE;
[nifi] branch main updated (2698000a85 -> 1a9acb1f4b)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 2698000a85 NIFI-11254 Upgraded SnakeYAML from 1.33 to 2.0 add 1a9acb1f4b NIFI-11337: Update PutDatabaseRecord's Max Batch Size property documentation to reflect actual behavior (#7081) No new revisions were added by this update. Summary of changes: .../org/apache/nifi/processors/standard/PutDatabaseRecord.java | 7 +++ 1 file changed, 3 insertions(+), 4 deletions(-)
[nifi] branch support/nifi-1.x updated: NIFI-11319 Fixing TestStatelessBootstrap::testClassNotAllowed (#7070)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new cef61c866d NIFI-11319 Fixing
TestStatelessBootstrap::testClassNotAllowed (#7070)
cef61c866d is described below
commit cef61c866d80816a93b1836e700acc5d3f249c5e
Author: simonbence <61191107+simonbe...@users.noreply.github.com>
AuthorDate: Wed Mar 22 21:23:44 2023 +0100
NIFI-11319 Fixing TestStatelessBootstrap::testClassNotAllowed (#7070)
---
.../org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/nifi-stateless/nifi-stateless-bootstrap/src/test/java/org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java
b/nifi-stateless/nifi-stateless-bootstrap/src/test/java/org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java
index 7804393f5d..2d30062adc 100644
---
a/nifi-stateless/nifi-stateless-bootstrap/src/test/java/org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java
+++
b/nifi-stateless/nifi-stateless-bootstrap/src/test/java/org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java
@@ -66,7 +66,7 @@ public class TestStatelessBootstrap {
final String classToLoad =
"org.apache.nifi.stateless.bootstrap.RunStatelessFlow";
// A directory for NARs, jars, etc. that are allowed by the
AllowListClassLoader
-final File narDirectory = new File("target");
+final File narDirectory = new File("target/generated-sources");
// Create a URLClassLoader to use for the System ClassLoader. This
will load the classes from the target/ directory.
// Then create an AllowListClassLoader that will not allow these
classes through.
[nifi] branch main updated: NIFI-11319 Fixing TestStatelessBootstrap::testClassNotAllowed (#7070)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new f062b0d1f1 NIFI-11319 Fixing
TestStatelessBootstrap::testClassNotAllowed (#7070)
f062b0d1f1 is described below
commit f062b0d1f1e39c869fe2632d5412012a56c3e4f8
Author: simonbence <61191107+simonbe...@users.noreply.github.com>
AuthorDate: Wed Mar 22 21:23:44 2023 +0100
NIFI-11319 Fixing TestStatelessBootstrap::testClassNotAllowed (#7070)
---
.../org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/nifi-stateless/nifi-stateless-bootstrap/src/test/java/org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java
b/nifi-stateless/nifi-stateless-bootstrap/src/test/java/org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java
index 7804393f5d..2d30062adc 100644
---
a/nifi-stateless/nifi-stateless-bootstrap/src/test/java/org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java
+++
b/nifi-stateless/nifi-stateless-bootstrap/src/test/java/org/apache/nifi/stateless/bootstrap/TestStatelessBootstrap.java
@@ -66,7 +66,7 @@ public class TestStatelessBootstrap {
final String classToLoad =
"org.apache.nifi.stateless.bootstrap.RunStatelessFlow";
// A directory for NARs, jars, etc. that are allowed by the
AllowListClassLoader
-final File narDirectory = new File("target");
+final File narDirectory = new File("target/generated-sources");
// Create a URLClassLoader to use for the System ClassLoader. This
will load the classes from the target/ directory.
// Then create an AllowListClassLoader that will not allow these
classes through.
[nifi] branch support/nifi-1.x updated: NIFI-11305: Fixed out-of-order handling of binlog client shutdown in CaptureChangeMySQL (#7068)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 3321de01f2 NIFI-11305: Fixed out-of-order handling of binlog client
shutdown in CaptureChangeMySQL (#7068)
3321de01f2 is described below
commit 3321de01f210aaadf9f6a3c9628b8e99a6fa72cf
Author: Matt Burgess
AuthorDate: Wed Mar 22 09:47:27 2023 -0400
NIFI-11305: Fixed out-of-order handling of binlog client shutdown in
CaptureChangeMySQL (#7068)
NIFI-11305: Fixed out-of-order handling of binlog client shutdown in
CaptureChangeMySQL
- Changed to allow 'unexpected' events to still be processed
---
.../nifi/cdc/mysql/event/BinlogEventListener.java | 8 +++--
.../cdc/mysql/processors/CaptureChangeMySQL.java | 36 +-
2 files changed, 34 insertions(+), 10 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/event/BinlogEventListener.java
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/event/BinlogEventListener.java
index c0f5800f67..48f35d3e65 100644
---
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/event/BinlogEventListener.java
+++
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/event/BinlogEventListener.java
@@ -18,6 +18,8 @@ package org.apache.nifi.cdc.mysql.event;
import com.github.shyiko.mysql.binlog.BinaryLogClient;
import com.github.shyiko.mysql.binlog.event.Event;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -28,6 +30,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
*/
public class BinlogEventListener implements BinaryLogClient.EventListener {
+private static final Logger logger =
LoggerFactory.getLogger(BinlogEventListener.class);
+
private final AtomicBoolean stopNow = new AtomicBoolean(false);
private static final int QUEUE_OFFER_TIMEOUT_MSEC = 100;
@@ -57,9 +61,9 @@ public class BinlogEventListener implements
BinaryLogClient.EventListener {
}
}
-throw new RuntimeException("Stopped while waiting to enqueue
event");
+logger.info("Stopped while waiting to enqueue event");
} catch (InterruptedException e) {
-throw new RuntimeException("Interrupted while adding event to the
queue");
+logger.warn("Interrupted while adding event to the queue", e);
}
}
}
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
index 4298af4379..76ada9b306 100644
---
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
+++
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
@@ -719,6 +719,12 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
connect(hosts, username, password, serverId,
createEnrichmentConnection, driverLocation, driverName, connectTimeout,
sslContextService, sslMode);
} catch (IOException | IllegalStateException e) {
+if (eventListener != null) {
+eventListener.stop();
+if (binlogClient != null) {
+binlogClient.unregisterEventListener(eventListener);
+}
+}
context.yield();
binlogClient = null;
throw new ProcessException(e.getMessage(), e);
@@ -901,6 +907,12 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
}
}
if (!binlogClient.isConnected()) {
+if (eventListener != null) {
+eventListener.stop();
+if (binlogClient != null) {
+binlogClient.unregisterEventListener(eventListener);
+}
+}
binlogClient.disconnect();
binlogClient = null;
throw new IOException("Could not connect binlog client to any of
the specified hosts due to: " + lastConnectException.getMessage(),
lastConnectException);
@@ -915,9 +927,18 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryPro
[nifi] branch main updated (00707f684f -> 3bf1195f4b)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 00707f684f NIFI-11323: Fixed last modified change detection of dynamicallyModifiesClasspath resources (#7069) add 3bf1195f4b NIFI-11305: Fixed out-of-order handling of binlog client shutdown in CaptureChangeMySQL (#7068) No new revisions were added by this update. Summary of changes: .../nifi/cdc/mysql/event/BinlogEventListener.java | 8 +++-- .../cdc/mysql/processors/CaptureChangeMySQL.java | 36 +- 2 files changed, 34 insertions(+), 10 deletions(-)
[nifi] branch main updated: NIFI-11323: Fixed last modified change detection of dynamicallyModifiesClasspath resources (#7069)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 00707f684f NIFI-11323: Fixed last modified change detection of
dynamicallyModifiesClasspath resources (#7069)
00707f684f is described below
commit 00707f684fd2f5a6989e520d518811dbfafc26fc
Author: Peter Turcsanyi <35004384+turcsan...@users.noreply.github.com>
AuthorDate: Wed Mar 22 14:15:48 2023 +0100
NIFI-11323: Fixed last modified change detection of
dynamicallyModifiesClasspath resources (#7069)
---
.../apache/nifi/util/file/classloader/ClassLoaderUtils.java | 10 +++---
.../nifi/util/file/classloader/TestClassLoaderUtils.java | 12 ++--
2 files changed, 17 insertions(+), 5 deletions(-)
diff --git
a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/classloader/ClassLoaderUtils.java
b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/classloader/ClassLoaderUtils.java
index 61ef0173e0..487c9d39b6 100644
---
a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/classloader/ClassLoaderUtils.java
+++
b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/classloader/ClassLoaderUtils.java
@@ -155,13 +155,17 @@ public class ClassLoaderUtils {
}
private static long getLastModified(String url) {
-File file = null;
+long lastModified = 0;
try {
-file = new File(new URI(url));
+final URI uri = new URI(url);
+if (uri.getScheme().equals("file")) {
+final File file = new File(uri);
+lastModified = file.lastModified();
+}
} catch (URISyntaxException e) {
LOGGER.error("Error getting last modified date for " + url);
}
-return file != null ? file.lastModified() : 0;
+return lastModified;
}
protected static ClassLoader createModuleClassLoader(URL[] modules,
ClassLoader parentClassLoader) {
diff --git
a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/classloader/TestClassLoaderUtils.java
b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/classloader/TestClassLoaderUtils.java
index e2bec02eba..2ba4796bdc 100644
---
a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/classloader/TestClassLoaderUtils.java
+++
b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/classloader/TestClassLoaderUtils.java
@@ -20,7 +20,6 @@ import org.junit.jupiter.api.Test;
import java.io.FilenameFilter;
import java.net.MalformedURLException;
-import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Paths;
import java.util.HashSet;
@@ -114,7 +113,7 @@ public class TestClassLoaderUtils {
}
@Test
-public void testGenerateAdditionalUrlsFingerprint() throws
MalformedURLException, URISyntaxException {
+public void testGenerateAdditionalUrlsFingerprintForFileUrl() throws
MalformedURLException {
final Set urls = new HashSet<>();
URL testUrl =
Paths.get("src/test/resources/TestClassLoaderUtils/TestSuccess.jar").toUri().toURL();
urls.add(testUrl);
@@ -122,6 +121,15 @@ public class TestClassLoaderUtils {
assertNotNull(testFingerprint);
}
+@Test
+public void testGenerateAdditionalUrlsFingerprintForHttpUrl() throws
MalformedURLException {
+final Set urls = new HashSet<>();
+URL testUrl = new URL("http://myhost/TestSuccess.jar;);
+urls.add(testUrl);
+String testFingerprint =
ClassLoaderUtils.generateAdditionalUrlsFingerprint(urls, null);
+assertNotNull(testFingerprint);
+}
+
protected FilenameFilter getJarFilenameFilter(){
return (dir, name) -> name != null && name.endsWith(".jar");
}
[nifi] branch support/nifi-1.x updated: NIFI-11323: Fixed last modified change detection of dynamicallyModifiesClasspath resources (#7069)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 22c072e619 NIFI-11323: Fixed last modified change detection of
dynamicallyModifiesClasspath resources (#7069)
22c072e619 is described below
commit 22c072e619e12725fae74c86570392035d5e418f
Author: Peter Turcsanyi <35004384+turcsan...@users.noreply.github.com>
AuthorDate: Wed Mar 22 14:15:48 2023 +0100
NIFI-11323: Fixed last modified change detection of
dynamicallyModifiesClasspath resources (#7069)
---
.../apache/nifi/util/file/classloader/ClassLoaderUtils.java | 10 +++---
.../nifi/util/file/classloader/TestClassLoaderUtils.java | 12 ++--
2 files changed, 17 insertions(+), 5 deletions(-)
diff --git
a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/classloader/ClassLoaderUtils.java
b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/classloader/ClassLoaderUtils.java
index 61ef0173e0..487c9d39b6 100644
---
a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/classloader/ClassLoaderUtils.java
+++
b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/classloader/ClassLoaderUtils.java
@@ -155,13 +155,17 @@ public class ClassLoaderUtils {
}
private static long getLastModified(String url) {
-File file = null;
+long lastModified = 0;
try {
-file = new File(new URI(url));
+final URI uri = new URI(url);
+if (uri.getScheme().equals("file")) {
+final File file = new File(uri);
+lastModified = file.lastModified();
+}
} catch (URISyntaxException e) {
LOGGER.error("Error getting last modified date for " + url);
}
-return file != null ? file.lastModified() : 0;
+return lastModified;
}
protected static ClassLoader createModuleClassLoader(URL[] modules,
ClassLoader parentClassLoader) {
diff --git
a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/classloader/TestClassLoaderUtils.java
b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/classloader/TestClassLoaderUtils.java
index e2bec02eba..2ba4796bdc 100644
---
a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/classloader/TestClassLoaderUtils.java
+++
b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/classloader/TestClassLoaderUtils.java
@@ -20,7 +20,6 @@ import org.junit.jupiter.api.Test;
import java.io.FilenameFilter;
import java.net.MalformedURLException;
-import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Paths;
import java.util.HashSet;
@@ -114,7 +113,7 @@ public class TestClassLoaderUtils {
}
@Test
-public void testGenerateAdditionalUrlsFingerprint() throws
MalformedURLException, URISyntaxException {
+public void testGenerateAdditionalUrlsFingerprintForFileUrl() throws
MalformedURLException {
final Set urls = new HashSet<>();
URL testUrl =
Paths.get("src/test/resources/TestClassLoaderUtils/TestSuccess.jar").toUri().toURL();
urls.add(testUrl);
@@ -122,6 +121,15 @@ public class TestClassLoaderUtils {
assertNotNull(testFingerprint);
}
+@Test
+public void testGenerateAdditionalUrlsFingerprintForHttpUrl() throws
MalformedURLException {
+final Set urls = new HashSet<>();
+URL testUrl = new URL("http://myhost/TestSuccess.jar;);
+urls.add(testUrl);
+String testFingerprint =
ClassLoaderUtils.generateAdditionalUrlsFingerprint(urls, null);
+assertNotNull(testFingerprint);
+}
+
protected FilenameFilter getJarFilenameFilter(){
return (dir, name) -> name != null && name.endsWith(".jar");
}
[nifi] branch support/nifi-1.x updated: NIFI-11305: Fix logic to correctly stop CaptureChangeMySQL (#7059)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new b554102f4a NIFI-11305: Fix logic to correctly stop CaptureChangeMySQL
(#7059)
b554102f4a is described below
commit b554102f4aa5f3cd40ba505ad233d4032b4accff
Author: Matt Burgess
AuthorDate: Mon Mar 20 16:04:19 2023 -0400
NIFI-11305: Fix logic to correctly stop CaptureChangeMySQL (#7059)
---
.../org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java | 5 +
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
index 37a6820b8a..4298af4379 100644
---
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
+++
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
@@ -492,7 +492,6 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
private volatile boolean inTransaction = false;
private volatile boolean skipTable = false;
-private final AtomicBoolean doStop = new AtomicBoolean(false);
private final AtomicBoolean hasRun = new AtomicBoolean(false);
private int currentHost = 0;
@@ -923,7 +922,6 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
}
gtidSet = new GtidSet(binlogClient.getGtidSet());
-doStop.set(false);
}
@@ -931,7 +929,7 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
RawBinlogEvent rawBinlogEvent;
// Drain the queue
-while ((rawBinlogEvent = queue.poll()) != null && !doStop.get()) {
+while (isScheduled() && (rawBinlogEvent = queue.poll()) != null) {
Event event = rawBinlogEvent.getEvent();
EventHeaderV4 header = event.getHeader();
long timestamp = header.getTimestamp();
@@ -1266,7 +1264,6 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
currentSession.commitAsync();
}
-doStop.set(true);
currentBinlogPosition = -1;
} catch (IOException e) {
throw new CDCException("Error closing CDC connection", e);
[nifi] branch main updated: NIFI-11305: Fix logic to correctly stop CaptureChangeMySQL (#7059)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 0c7de2c748 NIFI-11305: Fix logic to correctly stop CaptureChangeMySQL
(#7059)
0c7de2c748 is described below
commit 0c7de2c7482f18507877750ea00f9e9b9134fac7
Author: Matt Burgess
AuthorDate: Mon Mar 20 16:04:19 2023 -0400
NIFI-11305: Fix logic to correctly stop CaptureChangeMySQL (#7059)
---
.../org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java | 5 +
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
index 37a6820b8a..4298af4379 100644
---
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
+++
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
@@ -492,7 +492,6 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
private volatile boolean inTransaction = false;
private volatile boolean skipTable = false;
-private final AtomicBoolean doStop = new AtomicBoolean(false);
private final AtomicBoolean hasRun = new AtomicBoolean(false);
private int currentHost = 0;
@@ -923,7 +922,6 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
}
gtidSet = new GtidSet(binlogClient.getGtidSet());
-doStop.set(false);
}
@@ -931,7 +929,7 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
RawBinlogEvent rawBinlogEvent;
// Drain the queue
-while ((rawBinlogEvent = queue.poll()) != null && !doStop.get()) {
+while (isScheduled() && (rawBinlogEvent = queue.poll()) != null) {
Event event = rawBinlogEvent.getEvent();
EventHeaderV4 header = event.getHeader();
long timestamp = header.getTimestamp();
@@ -1266,7 +1264,6 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
currentSession.commitAsync();
}
-doStop.set(true);
currentBinlogPosition = -1;
} catch (IOException e) {
throw new CDCException("Error closing CDC connection", e);
[nifi] branch main updated: NIFI-11213 Showing version change in older (pre 1.18.0) contained version flows properly (#7017)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 7954ff355c NIFI-11213 Showing version change in older (pre 1.18.0)
contained version flows properly (#7017)
7954ff355c is described below
commit 7954ff355cbadefd070a3601a91128e380f5aa87
Author: simonbence <61191107+simonbe...@users.noreply.github.com>
AuthorDate: Mon Mar 20 16:52:07 2023 +0100
NIFI-11213 Showing version change in older (pre 1.18.0) contained version
flows properly (#7017)
---
.../apache/nifi/util/FlowDifferenceFilters.java| 22 +++---
.../diff/ConciseEvolvingDifferenceDescriptor.java | 2 +-
.../registry/flow/diff/FlowDifferenceUtil.java | 50 ++
3 files changed, 58 insertions(+), 16 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
index db714946db..ad2fa76122 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
@@ -34,6 +34,7 @@ import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.registry.flow.diff.DifferenceType;
import org.apache.nifi.registry.flow.diff.FlowDifference;
+import org.apache.nifi.registry.flow.diff.FlowDifferenceUtil;
import org.apache.nifi.registry.flow.mapping.InstantiatedVersionedComponent;
import
org.apache.nifi.registry.flow.mapping.InstantiatedVersionedControllerService;
import org.apache.nifi.registry.flow.mapping.InstantiatedVersionedProcessor;
@@ -192,22 +193,13 @@ public class FlowDifferenceFilters {
final VersionedFlowCoordinates coordinatesB =
versionedProcessGroupB.getVersionedFlowCoordinates();
if (coordinatesA != null && coordinatesB != null) {
-String registryUrlA = coordinatesA.getRegistryUrl();
-String registryUrlB = coordinatesB.getRegistryUrl();
-
-if (registryUrlA != null && registryUrlB != null &&
!registryUrlA.equals(registryUrlB)) {
-if (registryUrlA.endsWith("/")) {
-registryUrlA = registryUrlA.substring(0,
registryUrlA.length() - 1);
-}
-
-if (registryUrlB.endsWith("/")) {
-registryUrlB = registryUrlB.substring(0,
registryUrlB.length() - 1);
-}
-
-if (registryUrlA.equals(registryUrlB)) {
-return true;
-}
+if (coordinatesA.getStorageLocation() != null ||
coordinatesB.getStorageLocation() != null) {
+return false;
}
+
+return
!FlowDifferenceUtil.areRegistryStrictlyEqual(coordinatesA, coordinatesB)
+&&
FlowDifferenceUtil.areRegistryUrlsEqual(coordinatesA, coordinatesB)
+&& coordinatesA.getVersion() ==
coordinatesB.getVersion();
}
}
}
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
index 144f865af4..3fc4e4b1bf 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ConciseEvolvingDifferenceDescriptor.java
@@ -80,7 +80,7 @@ public class ConciseEvolvingDifferenceDescriptor implements
DifferenceDescriptor
final VersionedFlowCoordinates coordinatesB =
(VersionedFlowCoordinates) valueB;
// If the two vary only by version, then use a more
concise message. If anything else is different, then use a fully explanation.
-if (Objects.equals(coordinatesA.getRegistryUrl(),
coordinatesB.getRegistryUrl()) && Objects.equals(coordinatesA.getBucketId(),
coordinatesB.getBucketId())
+if (FlowDifferenceUtil.areRegistryUrlsEqual(coordinatesA,
coordinates
[nifi] branch support/nifi-1.x updated: NIFI-11279: Allow event stream processing to continue in CaptureChangeMySQL after sync issue (#7039)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 9e804fec4e NIFI-11279: Allow event stream processing to continue in
CaptureChangeMySQL after sync issue (#7039)
9e804fec4e is described below
commit 9e804fec4ef1ddf0b813b7b25147038b47429001
Author: Matt Burgess
AuthorDate: Tue Mar 14 16:23:43 2023 -0400
NIFI-11279: Allow event stream processing to continue in CaptureChangeMySQL
after sync issue (#7039)
---
.../cdc/mysql/processors/CaptureChangeMySQL.java | 33 ---
.../mysql/processors/CaptureChangeMySQLTest.groovy | 15 -
.../repository/StandardProcessSession.java | 38 ++
3 files changed, 46 insertions(+), 40 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
index 0badd08aa8..37a6820b8a 100644
---
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
+++
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
@@ -739,6 +739,11 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
setup(context);
}
+// If no client could be created, try again
+if (binlogClient == null) {
+return;
+}
+
// If the client has been disconnected, try to reconnect
if (!binlogClient.isConnected()) {
Exception e = lifecycleListener.getException();
@@ -764,7 +769,8 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
try {
outputEvents(currentSession, log);
-} catch (IOException ioe) {
+} catch (Exception eventException) {
+getLogger().error("Exception during event processing at file={}
pos={}", currentBinlogFile, currentBinlogPosition, eventException);
try {
// Perform some processor-level "rollback", then rollback the
session
currentBinlogFile = xactBinlogFile == null ? "" :
xactBinlogFile;
@@ -773,13 +779,14 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
currentGtidSet = xactGtidSet;
inTransaction = false;
stop();
-queue.clear();
-currentSession.rollback();
} catch (Exception e) {
// Not much we can recover from here
-log.warn("Error occurred during rollback", e);
+log.error("Error stopping CDC client", e);
+} finally {
+queue.clear();
+currentSession.rollback();
}
-throw new ProcessException(ioe);
+context.yield();
}
}
@@ -936,7 +943,7 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
if (eventType != ROTATE && eventType != FORMAT_DESCRIPTION &&
!useGtid) {
currentBinlogPosition = header.getPosition();
}
-log.debug("Got message event type: {} ",
header.getEventType().toString());
+log.debug("Message event, type={} pos={} file={}", eventType,
currentBinlogPosition, currentBinlogFile);
switch (eventType) {
case TABLE_MAP:
// This is sent to inform which table is about to be
changed by subsequent events
@@ -988,7 +995,8 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
if ("BEGIN".equals(sql)) {
// If we're already in a transaction, something bad
happened, alert the user
if (inTransaction) {
-throw new IOException("BEGIN event received while
already processing a transaction. This could indicate that your binlog position
is invalid.");
+getLogger().debug("BEGIN event received at pos={}
file={} while already processing a transaction. This could indicate that your
binlog position is invalid "
++ "or the event stream is out of sync or
there was an issue with the processor state.", currentBinlogPosition,
currentBinlogFile);
}
// Mark the current binlog position and
[nifi] branch main updated (1c883adca8 -> 668d309c7e)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 1c883adca8 NIFI-11275 Removed HashContent from See Also in FuzzyHashContent add 668d309c7e NIFI-11279: Allow event stream processing to continue in CaptureChangeMySQL after sync issue (#7039) No new revisions were added by this update. Summary of changes: .../cdc/mysql/processors/CaptureChangeMySQL.java | 33 ++--- .../mysql/processors/CaptureChangeMySQLTest.groovy | 15 .../repository/StandardProcessSession.java | 42 +++--- 3 files changed, 48 insertions(+), 42 deletions(-)
[nifi] branch main updated: NIFI-5501: Fixed classloader issue leading to multiple abandoned threads (#7031)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new c1890a5bb8 NIFI-5501: Fixed classloader issue leading to multiple
abandoned threads (#7031)
c1890a5bb8 is described below
commit c1890a5bb813c64aca6c3c9684ceef01f43f9e86
Author: Matt Burgess
AuthorDate: Fri Mar 10 15:37:32 2023 -0500
NIFI-5501: Fixed classloader issue leading to multiple abandoned threads
(#7031)
---
.../nifi/cdc/mysql/processors/CaptureChangeMySQL.java | 18 --
.../cdc/mysql/processors/CaptureChangeMySQLTest.groovy | 3 ++-
2 files changed, 6 insertions(+), 15 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
index d2665fa231..0badd08aa8 100644
---
a/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
+++
b/nifi-nar-bundles/nifi-cdc/nifi-cdc-mysql-bundle/nifi-cdc-mysql-processors/src/main/java/org/apache/nifi/cdc/mysql/processors/CaptureChangeMySQL.java
@@ -29,6 +29,7 @@ import com.github.shyiko.mysql.binlog.network.SSLMode;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.PrimaryNodeOnly;
+import org.apache.nifi.annotation.behavior.RequiresInstanceClassLoading;
import org.apache.nifi.annotation.behavior.Stateful;
import org.apache.nifi.annotation.behavior.TriggerSerially;
import org.apache.nifi.annotation.behavior.WritesAttribute;
@@ -92,13 +93,11 @@ import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.security.util.TlsConfiguration;
import org.apache.nifi.ssl.SSLContextService;
-import org.apache.nifi.util.file.classloader.ClassLoaderUtils;
import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.net.ConnectException;
import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
@@ -161,6 +160,7 @@ import static
org.apache.nifi.cdc.event.io.FlowFileEventWriteStrategy.MAX_EVENTS
@WritesAttribute(attribute = "mime.type", description = "The processor
outputs flow file content in JSON format, and sets the mime.type attribute to "
+ "application/json")
})
+@RequiresInstanceClassLoading
public class CaptureChangeMySQL extends AbstractSessionFactoryProcessor {
// Random invalid constant used as an indicator to not set the binlog
position on the client (thereby using the latest available)
@@ -257,6 +257,7 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
.required(false)
.identifiesExternalResource(ResourceCardinality.MULTIPLE,
ResourceType.FILE, ResourceType.DIRECTORY, ResourceType.URL)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+.dynamicallyModifiesClasspath(true)
.build();
public static final PropertyDescriptor USERNAME = new
PropertyDescriptor.Builder()
@@ -1403,16 +1404,7 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
protected void registerDriver(String locationString, String drvName)
throws InitializationException {
if (locationString != null && locationString.length() > 0) {
try {
-// Split and trim the entries
-final ClassLoader classLoader =
ClassLoaderUtils.getCustomClassLoader(
-locationString,
-this.getClass().getClassLoader(),
-(dir, name) -> name != null && name.endsWith(".jar")
-);
-
-// Workaround which allows to use URLClassLoader for JDBC
driver loading.
-// (Because the DriverManager will refuse to use a driver not
loaded by the system ClassLoader.)
-final Class clazz = Class.forName(drvName, true,
classLoader);
+final Class clazz = Class.forName(drvName);
if (clazz == null) {
throw new InitializationException("Can't load Database
Driver " + drvName);
}
@@ -1421,8 +1413,6 @@ public class CaptureChangeMySQL extends
AbstractSessionFactoryProcessor {
} catch (final InitializationException e) {
throw e;
-} catch (final MalformedURLException e) {
-
[nifi] branch main updated: NIFI-11257 Enable Maven HTTP pooling and additional retries (#7020)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/main by this push: new a13c2cf010 NIFI-11257 Enable Maven HTTP pooling and additional retries (#7020) a13c2cf010 is described below commit a13c2cf010c4e12ee266ca5fbc45b485813e9c09 Author: exceptionfactory AuthorDate: Wed Mar 8 10:36:40 2023 -0600 NIFI-11257 Enable Maven HTTP pooling and additional retries (#7020) * NIFI-11257 Enabled Maven HTTP pooling and additional retries - Enabled HTTP connection pooling for Maven Wagon - Configured 30 second timeout for HTTP pooled connection lifespan - Enabled 5 retries for HTTP connections - Set maximum connection per route to 5 instead of 20 - Enabled retry for sent HTTP requests --- .github/workflows/ci-workflow.yml | 14 ++ 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 8ca441c758..d6d692a927 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -23,12 +23,18 @@ env: -XX:ReservedCodeCacheSize=1g -XX:+UseG1GC -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --Dhttp.keepAlive=false --Dmaven.wagon.http.pool=false +-Dmaven.wagon.http.retryHandler.class=standard +-Dmaven.wagon.http.retryHandler.count=5 +-Dmaven.wagon.http.retryHandler.requestSentEnabled=true +-Dmaven.wagon.httpconnectionManager.maxPerRoute=5 +-Dmaven.wagon.httpconnectionManager.ttlSeconds=30 COMPILE_MAVEN_OPTS: >- -Xmx3g --Dhttp.keepAlive=false --Dmaven.wagon.http.pool=false +-Dmaven.wagon.http.retryHandler.class=standard +-Dmaven.wagon.http.retryHandler.count=5 +-Dmaven.wagon.http.retryHandler.requestSentEnabled=true +-Dmaven.wagon.httpconnectionManager.maxPerRoute=5 +-Dmaven.wagon.httpconnectionManager.ttlSeconds=30 MAVEN_COMPILE_COMMAND: >- mvn test-compile --show-version
[nifi] branch support/nifi-1.x updated: NIFI-11245 Corrected reducing Maximum Thread Count while running (#7005)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 88d2c03b37 NIFI-11245 Corrected reducing Maximum Thread Count while
running (#7005)
88d2c03b37 is described below
commit 88d2c03b37d6cdabd260658c84e4305b4fa14c99
Author: exceptionfactory
AuthorDate: Fri Mar 3 10:35:34 2023 -0600
NIFI-11245 Corrected reducing Maximum Thread Count while running (#7005)
- Corrected implementation to allow reducing Maximum Thread Count below the
default of 10
---
.../org/apache/nifi/controller/FlowController.java | 19 +--
.../apache/nifi/controller/TestFlowController.java | 21 ++---
2 files changed, 31 insertions(+), 9 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
index ce93d4a474..376bdcc1f9 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
@@ -1567,7 +1567,7 @@ public class FlowController implements
ReportingTaskProvider, Authorizable, Node
public void setMaxTimerDrivenThreadCount(final int maxThreadCount) {
writeLock.lock();
try {
-setMaxThreadCount(maxThreadCount, this.timerDrivenEngineRef.get(),
this.maxTimerDrivenThreads);
+setMaxThreadCount(maxThreadCount, "Timer Driven",
this.timerDrivenEngineRef.get(), this.maxTimerDrivenThreads);
} finally {
writeLock.unlock("setMaxTimerDrivenThreadCount");
}
@@ -1576,7 +1576,7 @@ public class FlowController implements
ReportingTaskProvider, Authorizable, Node
public void setMaxEventDrivenThreadCount(final int maxThreadCount) {
writeLock.lock();
try {
-setMaxThreadCount(maxThreadCount, this.eventDrivenEngineRef.get(),
this.maxEventDrivenThreads);
+setMaxThreadCount(maxThreadCount, "Event Driven",
this.eventDrivenEngineRef.get(), this.maxEventDrivenThreads);
processScheduler.setMaxThreadCount(SchedulingStrategy.EVENT_DRIVEN,
maxThreadCount);
} finally {
writeLock.unlock("setMaxEventDrivenThreadCount");
@@ -1587,16 +1587,23 @@ public class FlowController implements
ReportingTaskProvider, Authorizable, Node
* Updates the number of threads that can be simultaneously used for
executing processors.
* This method must be called while holding the write lock!
*
- * @param maxThreadCount max number of threads
+ * @param maxThreadCount Requested new thread pool size
+ * @param poolName Thread Pool Name
+ * @param engine Flow Engine executor or null when terminated
+ * @param maxThreads Internal tracker for Maximum Threads
*/
-private void setMaxThreadCount(final int maxThreadCount, final FlowEngine
engine, final AtomicInteger maxThreads) {
+private void setMaxThreadCount(final int maxThreadCount, final String
poolName, final FlowEngine engine, final AtomicInteger maxThreads) {
if (maxThreadCount < 1) {
throw new IllegalArgumentException("Cannot set max number of
threads to less than 1");
}
maxThreads.getAndSet(maxThreadCount);
-if (null != engine && engine.getCorePoolSize() < maxThreadCount) {
-engine.setCorePoolSize(maxThreads.intValue());
+if (engine == null) {
+LOG.debug("[{}] Engine not found: Maximum Thread Count not
updated", poolName);
+} else {
+final int previousCorePoolSize = engine.getCorePoolSize();
+engine.setCorePoolSize(maxThreadCount);
+LOG.info("[{}] Maximum Thread Count updated [{}] previous [{}]",
poolName, maxThreadCount, previousCorePoolSize);
}
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java
index d6711cf6df..acc52c9cb9 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.jav
[nifi] branch main updated (9ea0fe7b3d -> 44c70277ea)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 9ea0fe7b3d NIFI-11234: Fix RecordWriter NPE in QuerySalesforceObject add 44c70277ea NIFI-11245 Corrected reducing Maximum Thread Count while running (#7005) No new revisions were added by this update. Summary of changes: .../org/apache/nifi/controller/FlowController.java | 19 +-- .../apache/nifi/controller/TestFlowController.java | 21 ++--- 2 files changed, 31 insertions(+), 9 deletions(-)
[nifi] branch support/nifi-1.x updated: NIFI-11225 Fix toLowerCase and toUpperCase documentation (#6992)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a commit to branch support/nifi-1.x in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/support/nifi-1.x by this push: new e9dba3087d NIFI-11225 Fix toLowerCase and toUpperCase documentation (#6992) e9dba3087d is described below commit e9dba3087dab4a772124d51351c8508a9cee30ed Author: Juldrixx <31806759+juldr...@users.noreply.github.com> AuthorDate: Mon Feb 27 15:29:14 2023 +0100 NIFI-11225 Fix toLowerCase and toUpperCase documentation (#6992) * NIFI-11225 Fixed toLowerCase and toUpperCase documentation --- nifi-docs/src/main/asciidoc/record-path-guide.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nifi-docs/src/main/asciidoc/record-path-guide.adoc b/nifi-docs/src/main/asciidoc/record-path-guide.adoc index 17fd585fba..b2ef0f5ba7 100644 --- a/nifi-docs/src/main/asciidoc/record-path-guide.adoc +++ b/nifi-docs/src/main/asciidoc/record-path-guide.adoc @@ -758,7 +758,7 @@ and a record such as: } -The following record path expression would remove extraneous whitespace: +The following record path expression change lower case letters to upper case: |== | RecordPath | Return value @@ -787,11 +787,11 @@ and a record such as: } -The following record path expression would remove extraneous whitespace: +The following record path expression change upper case letters to lower case: |== | RecordPath | Return value -| `trim(/message)` | hello world +| `toLowerCase(/message)` | hello world |== === base64Encode
[nifi] branch main updated (61b87e007b -> 055cf62c70)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 61b87e007b NIFI-11209: Include newly-added columns in output for UpdateTable processors when Update Field Names is true add 055cf62c70 NIFI-11225 Fix toLowerCase and toUpperCase documentation (#6992) No new revisions were added by this update. Summary of changes: nifi-docs/src/main/asciidoc/record-path-guide.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
[nifi] branch main updated: NIFI-11175 Enabled system-tests for framework changes (#6947)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/main by this push: new 1014359c64 NIFI-11175 Enabled system-tests for framework changes (#6947) 1014359c64 is described below commit 1014359c64ca61a1659accd0eb1551ca4edabba9 Author: exceptionfactory AuthorDate: Mon Feb 13 16:07:39 2023 -0600 NIFI-11175 Enabled system-tests for framework changes (#6947) --- .github/workflows/system-tests.yml | 8 1 file changed, 8 insertions(+) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 7808ab00be..30b72249dc 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -23,10 +23,18 @@ on: paths: - '.github/workflows/system-tests.yml' - 'nifi-system-tests/**' + - 'nifi-api/**' + - 'nifi-framework-api/**' + - 'nifi-nar-bundles/nifi-framework-bundle/**' + - 'nifi-stateless/**' pull_request: paths: - '.github/workflows/system-tests.yml' - 'nifi-system-tests/**' + - 'nifi-api/**' + - 'nifi-framework-api/**' + - 'nifi-nar-bundles/nifi-framework-bundle/**' + - 'nifi-stateless/**' env: DEFAULT_MAVEN_OPTS: >-
[nifi] branch support/nifi-1.x updated: NIFI-11159 Fixing connections with source having reassigned id
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new f432d98166 NIFI-11159 Fixing connections with source having reassigned
id
f432d98166 is described below
commit f432d98166e3b095b388620e250b08f79e11175e
Author: Bence Simon
AuthorDate: Thu Feb 9 19:32:14 2023 +0100
NIFI-11159 Fixing connections with source having reassigned id
---
.../StandardVersionedComponentSynchronizer.java| 23 ++
1 file changed, 23 insertions(+)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
index cf9d6c5052..519089f048 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
@@ -614,16 +614,39 @@ public class StandardVersionedComponentSynchronizer
implements VersionedComponen
private void removeMissingConnections(final ProcessGroup group, final
VersionedProcessGroup proposed, final Map
connectionsByVersionedId) {
final Set connectionsRemoved = new
HashSet<>(connectionsByVersionedId.keySet());
+final Set connectionsRemovedDueToChangingSourceId = new
HashSet<>();
for (final VersionedConnection proposedConnection :
proposed.getConnections()) {
connectionsRemoved.remove(proposedConnection.getIdentifier());
}
+// Check for any case where there's an existing connection whose ID
matches the proposed connection, but whose source doesn't match
+// the proposed source ID. The source of a Connection should never
change from one component to another. However, there are cases
+// in which the Versioned Component ID might change, in order to avoid
conflicts with sibling Process Groups. In such a case, we must remove
+// the connection and create a new one, since we cannot simply change
the source in the same way that we can change the destination.
+for (final VersionedConnection proposedConnection :
proposed.getConnections()) {
+final Connection existingConnection =
connectionsByVersionedId.get(proposedConnection.getIdentifier());
+
+if (existingConnection != null) {
+final String proposedSourceId =
proposedConnection.getSource().getId();
+final String existingSourceId =
existingConnection.getSource().getVersionedComponentId().orElse(null);
+
+if (!Objects.equals(proposedSourceId, existingSourceId)) {
+
connectionsRemovedDueToChangingSourceId.add(proposedConnection.getIdentifier());
+connectionsRemoved.add(proposedConnection.getIdentifier());
+}
+}
+}
+
for (final String removedVersionedId : connectionsRemoved) {
final Connection connection =
connectionsByVersionedId.get(removedVersionedId);
LOG.info("Removing {} from {}", connection, group);
group.removeConnection(connection);
}
+
+for (final String removedVersionedId :
connectionsRemovedDueToChangingSourceId) {
+connectionsByVersionedId.remove(removedVersionedId);
+}
}
private void synchronizeConnections(final ProcessGroup group, final
VersionedProcessGroup proposed, final Map
connectionsByVersionedId) {
[nifi] branch main updated (a66727c4d1 -> b0ec28a452)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from a66727c4d1 NIFI-11160 Upgraded Apache Tika from 2.6.0 to 2.7.0 add b0ec28a452 NIFI-11159 Fixing connections with source having reassigned id No new revisions were added by this update. Summary of changes: .../StandardVersionedComponentSynchronizer.java| 23 ++ 1 file changed, 23 insertions(+)
[nifi] branch main updated (bda1bd326d -> ae66ebdc22)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
from bda1bd326d NIFI-4 Thise closes #6906. Upgraded OWASP Dependency
Check from 7.4.4 to 8.0.2
add ae66ebdc22 NIFI-10973 Introduce differention of comparing embedded
versioned flows for UI and synchronisation purposes (#6870)
No new revisions were added by this update.
Summary of changes:
.../StandardVersionedComponentSynchronizer.java| 3 ++-
.../apache/nifi/groups/StandardProcessGroup.java | 4 +++-
.../flow/StandardVersionControlInformation.java| 5 +++--
.../serialization/VersionedFlowSynchronizer.java | 3 ++-
.../nifi/integration/versioned/ImportFlowIT.java | 3 ++-
.../apache/nifi/web/StandardNiFiServiceFacade.java | 5 +++--
.../org/apache/nifi/web/api/VersionsResource.java | 1 +
...n.java => FlowComparatorVersionedStrategy.java} | 22 +-
.../registry/flow/diff/StandardFlowComparator.java | 14 ++
.../flow/diff/TestStandardFlowComparator.java | 2 +-
.../nifi/registry/service/RegistryService.java | 3 ++-
11 files changed, 42 insertions(+), 23 deletions(-)
copy
nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/{FlowComparison.java
=> FlowComparatorVersionedStrategy.java} (64%)
[nifi] branch main updated: NIFI-11076: Resolving deadlock issue in StandardParameterContext (#6865)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 29618ae5c6 NIFI-11076: Resolving deadlock issue in
StandardParameterContext (#6865)
29618ae5c6 is described below
commit 29618ae5c64a210fb9b497512fcc5d2e26575c75
Author: Joe Gresock
AuthorDate: Mon Jan 23 10:54:38 2023 -0500
NIFI-11076: Resolving deadlock issue in StandardParameterContext (#6865)
* NIFI-11076: Resolving deadlock issue in StandardParameterContext
* NIFI-11076: Resolving deadlock issue in StandardParameterContext
---
.../nifi/parameter/StandardParameterContext.java | 22 +-
1 file changed, 9 insertions(+), 13 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java
index 63aa3e50d6..339317fdd9 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java
@@ -47,6 +47,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Stack;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -61,7 +62,7 @@ public class StandardParameterContext implements
ParameterContext {
private final Authorizable parentAuthorizable;
private String name;
-private long version = 0L;
+private AtomicLong version = new AtomicLong(0L);
private final Map parameters = new
LinkedHashMap<>();
private final List inheritedParameterContexts = new
ArrayList<>();
private ParameterProvider parameterProvider;
@@ -105,7 +106,7 @@ public class StandardParameterContext implements
ParameterContext {
public void setName(final String name) {
writeLock.lock();
try {
-this.version++;
+this.version.incrementAndGet();
this.name = name;
} finally {
writeLock.unlock();
@@ -125,8 +126,9 @@ public class StandardParameterContext implements
ParameterContext {
@Override
public void setParameters(final Map updatedParameters) {
writeLock.lock();
+final Map parameterUpdates = new HashMap<>();
try {
-this.version++;
+this.version.incrementAndGet();
final Map
currentEffectiveParameters = getEffectiveParameters();
final Map
effectiveProposedParameters =
getEffectiveParameters(getProposedParameters(updatedParameters));
@@ -139,12 +141,11 @@ public class StandardParameterContext implements
ParameterContext {
updateParameters(parameters, updatedParameters, true);
// Get a list of all effective updates in order to alert
referencing components
-final Map parameterUpdates = new
HashMap<>(updateParameters(currentEffectiveParameters,
effectiveParameterUpdates, false));
-
-alertReferencingComponents(parameterUpdates);
+
parameterUpdates.putAll(updateParameters(currentEffectiveParameters,
effectiveParameterUpdates, false));
} finally {
writeLock.unlock();
}
+alertReferencingComponents(parameterUpdates);
}
private Map getProposedParameters(final
Map proposedParameterUpdates) {
@@ -270,12 +271,7 @@ public class StandardParameterContext implements
ParameterContext {
@Override
public long getVersion() {
-readLock.lock();
-try {
-return version;
-} finally {
-readLock.unlock();
-}
+return version.get();
}
public Optional getParameter(final String parameterName) {
@@ -597,7 +593,7 @@ public class StandardParameterContext implements
ParameterContext {
writeLock.lock();
try {
-this.version++;
+this.version.incrementAndGet();
final Map
currentEffectiveParameters = getEffectiveParameters();
final Map
effectiveProposedParameters =
getEffectiveParameters(inheritedParameterContexts);
[nifi] branch main updated: NIFI-11028 Handle controller service 'Scheduled State' change (#6831)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 9ad1d8a7ae NIFI-11028 Handle controller service 'Scheduled State'
change (#6831)
9ad1d8a7ae is described below
commit 9ad1d8a7aece34ae78363d265492edec5a4c3bd8
Author: krisztina-zsihovszki
<76993418+krisztina-zsihovs...@users.noreply.github.com>
AuthorDate: Wed Jan 18 00:40:47 2023 +0100
NIFI-11028 Handle controller service 'Scheduled State' change (#6831)
---
.../org/apache/nifi/util/FlowDifferenceFilters.java| 8
.../apache/nifi/util/TestFlowDifferenceFilters.java| 18 ++
2 files changed, 22 insertions(+), 4 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
index b052af8110..c9e07622e0 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
@@ -23,7 +23,6 @@ import org.apache.nifi.controller.flow.FlowManager;
import org.apache.nifi.controller.label.StandardLabel;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.flow.ComponentType;
-import org.apache.nifi.flow.ScheduledState;
import org.apache.nifi.flow.VersionedComponent;
import org.apache.nifi.flow.VersionedConnection;
import org.apache.nifi.flow.VersionedFlowCoordinates;
@@ -308,11 +307,12 @@ public class FlowDifferenceFilters {
return false;
}
-// If Scheduled State transitions from null to ENABLED or ENABLED to
null, consider it a "new" scheduled state.
-if (fd.getValueA() == null &&
ScheduledState.ENABLED.equals(fd.getValueB())) {
+// If Scheduled State transitions from null to a real state or
+// from a real state to null, consider it a "new" scheduled state.
+if (fd.getValueA() == null && fd.getValueB() != null) {
return true;
}
-if (fd.getValueB() == null && "ENABLED".equals(fd.getValueA())) {
+if (fd.getValueB() == null && fd.getValueA() != null) {
return true;
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
index a606769cea..b87bebc01c 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
@@ -17,6 +17,8 @@
package org.apache.nifi.util;
import org.apache.nifi.flow.ComponentType;
+import org.apache.nifi.flow.ScheduledState;
+import org.apache.nifi.flow.VersionedControllerService;
import org.apache.nifi.flow.VersionedFlowCoordinates;
import org.apache.nifi.flow.VersionedPort;
import org.apache.nifi.flow.VersionedProcessGroup;
@@ -169,5 +171,21 @@ public class TestFlowDifferenceFilters {
assertFalse(FlowDifferenceFilters.FILTER_PUBLIC_PORT_NAME_CHANGES.test(flowDifference));
}
+
+@Test
+public void
testFilterControllerServiceStatusChangeWhenNewStateIntroduced() {
+final VersionedControllerService controllerServiceA = new
VersionedControllerService();
+final VersionedControllerService controllerServiceB = new
VersionedControllerService();
+controllerServiceA.setScheduledState(null);
+controllerServiceB.setScheduledState(ScheduledState.DISABLED);
+
+final StandardFlowDifference flowDifference = new
StandardFlowDifference(
+DifferenceType.SCHEDULED_STATE_CHANGED,
+controllerServiceA, controllerServiceB,
+controllerServiceA.getScheduledState(),
controllerServiceB.getScheduledState(),
+"");
+
+assertTrue(FlowDifferenceFilters.isScheduledStateNew(flowDifference));
+}
}
[nifi] branch main updated: NIFI-10980 - correct ordering of property descriptor allowable values (#6781)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new b69721cac9 NIFI-10980 - correct ordering of property descriptor
allowable values (#6781)
b69721cac9 is described below
commit b69721cac9d24fb645aea461c7dd16e517dcfb34
Author: greyp9
AuthorDate: Thu Jan 5 14:11:03 2023 -0500
NIFI-10980 - correct ordering of property descriptor allowable values
(#6781)
NIFI-10980 - correct ordering of property descriptor allowable values
---
.../org/apache/nifi/web/api/dto/DtoFactory.java| 1 -
.../apache/nifi/web/api/dto/DtoFactoryTest.java| 112 +
2 files changed, 112 insertions(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 2039eab5a2..eb1e69c951 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -4247,7 +4247,6 @@ public final class DtoFactory {
allowableValueDto.setDescription(allowableValue.getDescription());
allowableValues.add(entityFactory.createAllowableValueEntity(allowableValueDto,
true));
}
-allowableValues.sort(Comparator.comparing(e ->
e.getAllowableValue().getDisplayName()));
dto.setAllowableValues(allowableValues);
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/dto/DtoFactoryTest.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/dto/DtoFactoryTest.java
new file mode 100644
index 00..358d9d2ff9
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/dto/DtoFactoryTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.service.ControllerServiceNode;
+import org.apache.nifi.controller.service.ControllerServiceProvider;
+import org.apache.nifi.nar.StandardExtensionDiscoveringManager;
+import org.apache.nifi.nar.SystemBundle;
+import org.apache.nifi.web.api.entity.AllowableValueEntity;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DtoFactoryTest {
+private final Logger logger = LoggerFactory.getLogger(getClass());
+
+@Test
+void testAllowableValuesControllerService() {
+final Set csIdentifiers = new
HashSet<>(Arrays.asList("uuid-2", "uuid-3", "uuid-1"));
+final ControllerServiceProvider controllerServiceProvider =
mock(ControllerServiceProvider.class);
+when(controllerServiceProvider.getControllerServiceIdentifiers(any(),
any())).thenReturn(csIdentifiers);
+final ControllerServiceNode service1 =
mock(ControllerServiceNode.class);
+final ControllerServiceNode service2 =
mock(ControllerServiceNode.class);
+final ControllerServiceNode service3 =
mock(Control
[nifi] branch main updated (a8974253ce -> 5c3ca9d537)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
from a8974253ce NIFI-10989 Removed SHA-1 and MD5 from TestHashContent
add 5c3ca9d537 NIFI-10981 Ensure NarAutoLoader starts after provider
retrieves NARs,… (#6785)
No new revisions were added by this update.
Summary of changes:
.github/workflows/ci-workflow.yml | 1 +
.gitignore | 1 +
.../java/org/apache/nifi/nar/NarAutoLoader.java| 12 ++-
.../org/apache/nifi/web/server/JettyServer.java| 22 ++---
.../nar/provider/LocalDirectoryNarProvider.java| 67 +++
...he.nifi.flow.resource.ExternalResourceProvider} | 19 +
.../nifi-nar-provider-assembly/pom.xml | 83 +++
.../src/main/assembly/dependencies.xml}| 34
.../nifi-nar-provider-processors-nar}/pom.xml | 37 ++---
.../nifi-nar-provider-processors/pom.xml | 68
.../nifi/nar/provider/GetClassLoaderInfo.java | 94 ++
.../services/org.apache.nifi.processor.Processor} | 20 +
.../nifi-nar-provider-service-api-nar}/pom.xml | 37 ++---
.../nifi-nar-provider-service-api}/pom.xml | 28 ---
.../nifi/nar/provider/ClassLoaderInfoService.java | 29 +++
.../nifi-nar-provider-service-nar}/pom.xml | 37 ++---
.../nifi-nar-provider-service/pom.xml | 67 +++
.../provider/StandardClassLoaderInfoService.java | 61 ++
.../org.apache.nifi.controller.ControllerService} | 20 +
.../pom.xml| 25 +++---
nifi-system-tests/nifi-system-test-suite/pom.xml | 26 +-
.../src/test/assembly/dependencies.xml | 1 +
.../SpawnedStandaloneNiFiInstanceFactory.java | 6 ++
.../system/nar/NarProviderAndAutoLoaderIT.java | 82 +++
.../src/test/resources/conf/default/bootstrap.conf | 2 +-
nifi-system-tests/pom.xml | 1 +
26 files changed, 732 insertions(+), 148 deletions(-)
create mode 100644
nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/nar/provider/LocalDirectoryNarProvider.java
copy
nifi-system-tests/{nifi-system-test-suite/src/test/resources/conf/default/bootstrap.conf
=>
nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.flow.resource.ExternalResourceProvider}
(57%)
create mode 100644
nifi-system-tests/nifi-system-test-nar-provider-bundles/nifi-nar-provider-assembly/pom.xml
copy nifi-system-tests/{pom.xml =>
nifi-system-test-nar-provider-bundles/nifi-nar-provider-assembly/src/main/assembly/dependencies.xml}
(50%)
copy nifi-system-tests/{ =>
nifi-system-test-nar-provider-bundles/nifi-nar-provider-processors-nar}/pom.xml
(50%)
create mode 100644
nifi-system-tests/nifi-system-test-nar-provider-bundles/nifi-nar-provider-processors/pom.xml
create mode 100644
nifi-system-tests/nifi-system-test-nar-provider-bundles/nifi-nar-provider-processors/src/main/java/org/apache/nifi/nar/provider/GetClassLoaderInfo.java
copy
nifi-system-tests/{nifi-system-test-suite/src/test/resources/conf/default/bootstrap.conf
=>
nifi-system-test-nar-provider-bundles/nifi-nar-provider-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor}
(57%)
copy nifi-system-tests/{ =>
nifi-system-test-nar-provider-bundles/nifi-nar-provider-service-api-nar}/pom.xml
(50%)
copy nifi-system-tests/{ =>
nifi-system-test-nar-provider-bundles/nifi-nar-provider-service-api}/pom.xml
(63%)
create mode 100644
nifi-system-tests/nifi-system-test-nar-provider-bundles/nifi-nar-provider-service-api/src/main/java/org/apache/nifi/nar/provider/ClassLoaderInfoService.java
copy nifi-system-tests/{ =>
nifi-system-test-nar-provider-bundles/nifi-nar-provider-service-nar}/pom.xml
(51%)
create mode 100644
nifi-system-tests/nifi-system-test-nar-provider-bundles/nifi-nar-provider-service/pom.xml
create mode 100644
nifi-system-tests/nifi-system-test-nar-provider-bundles/nifi-nar-provider-service/src/main/java/org/apache/nifi/nar/provider/StandardClassLoaderInfoService.java
copy
nifi-system-tests/{nifi-system-test-suite/src/test/resources/conf/default/bootstrap.conf
=>
nifi-system-test-nar-provider-bundles/nifi-nar-provider-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService}
(57%)
copy nifi-system-tests/{ => nifi-system-test-nar-provider-bundles}/pom.xml
(56%)
create mode 100644
nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/nar/NarProviderAndAutoLoaderIT.java
[nifi] branch main updated (45a31c7286 -> b744fac479)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 45a31c7286 NIFI-10899 Added SameSite Policy to Application Cookies add b744fac479 NIFI-10956: Fix inference issues with mixed arrays (#6763) No new revisions were added by this update. Summary of changes: .../serialization/record/util/DataTypeUtils.java | 22 - .../nifi-record-serialization-services/pom.xml | 2 + .../org/apache/nifi/json/JsonSchemaInference.java | 5 ++ .../inference/HierarchicalSchemaInference.java | 94 +++--- .../nifi/xml/inference/XmlSchemaInference.java | 5 ++ .../apache/nifi/json/TestJsonSchemaInference.java | 77 +++--- .../org/apache/nifi/json/TestWriteJsonResult.java | 44 ++ .../choice-of-array-empty-or-array-record.json | 59 ++ .../choice-of-array-string-or-array-record.json| 9 +++ .../src/test/resources/json/empty-arrays.json | 13 +++ 10 files changed, 305 insertions(+), 25 deletions(-) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/choice-of-array-empty-or-array-record.json create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/choice-of-array-string-or-array-record.json create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/empty-arrays.json
[nifi] branch main updated (9a574c875e -> 570fc7f1fa)
This is an automated email from the ASF dual-hosted git repository. markap14 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from 9a574c875e NIFI-10909 Upgraded Jackson from 2.14.0 to 2.14.1 add 570fc7f1fa NIFI-10914 Adjusting input check for loading nested versioned flows (#6741) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-)
[nifi] branch main updated: NIFI-10901 - PublishKafka headers not sent in ProducerRecord (#6731)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 282c56b5ce NIFI-10901 - PublishKafka headers not sent in
ProducerRecord (#6731)
282c56b5ce is described below
commit 282c56b5ceca47caf711898c5929e35b964bb7fe
Author: greyp9
AuthorDate: Wed Nov 30 13:23:23 2022 -0500
NIFI-10901 - PublishKafka headers not sent in ProducerRecord (#6731)
---
.../processors/kafka/pubsub/PublisherLease.java| 3 +-
.../additionalDetails.html | 2 +-
.../pubsub/TestPublishKafkaMockParameterized.java | 97 +-
...> TestPublishKafkaRecordMockParameterized.java} | 49 +--
.../Publish/parameterized/flowfileInput1.json | 8 ++
.../Publish/parameterized/flowfileInputA.json | 12 +++
.../Publish/parameterized/kafkaOutput1A.json | 18
.../Publish/parameterized/kafkaOutput1B.json | 18
.../Publish/parameterized/kafkaOutputA1.json | 22 +
.../Publish/parameterized/kafkaOutputA2.json | 22 +
.../parameterized/flowfileInputDoc1V.json | 8 ++
.../parameterized/flowfileInputDoc1W.json | 15
.../parameterized/flowfileInputDoc2W.json | 15
.../parameterized/kafkaOutputDoc1V.json| 21 +
.../parameterized/kafkaOutputDoc1W.json| 18
.../parameterized/kafkaOutputDoc2W.json| 16
16 files changed, 275 insertions(+), 69 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
index 0949e537b6..da37db0319 100644
---
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
+++
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
@@ -431,7 +431,8 @@ public class PublisherLease implements Closeable {
}
protected void publish(final FlowFile flowFile, final byte[] messageKey,
final byte[] messageContent, final String topic, final InFlightMessageTracker
tracker, final Integer partition) {
-publish(flowFile, Collections.emptyList(), messageKey, messageContent,
topic, tracker, partition);
+final List headers = toHeaders(flowFile,
Collections.emptyMap());
+publish(flowFile, headers, messageKey, messageContent, topic, tracker,
partition);
}
protected void publish(final FlowFile flowFile, final List
headers, final byte[] messageKey, final byte[] messageContent,
diff --git
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html
index ce87abc73d..e125b0f067 100644
---
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html
+++
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html
@@ -375,7 +375,7 @@
Record Key
-Acme Accounts
+Acme Holdings
Record Value
diff --git
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaMockParameterized.java
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaMockParameterized.java
index 4e97a6da84..b15f5d55a1 100644
---
a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaMockParameterized.java
+++
b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaMockParameterized.java
@@ -31,17 +31,12 @@ import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.header.Header;
import org.apache.kafka.common.header.internals.RecordHeader;
-import org.apache.nifi.json.JsonRecordSetWriter;
-imp
