[nifi] branch main updated: NIFI-11158 PutSalesforceObject processor improvements
This is an automated email from the ASF dual-hosted git repository.
turcsanyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 60c02225d5 NIFI-11158 PutSalesforceObject processor improvements
60c02225d5 is described below
commit 60c02225d5694efe8f6dab151aa441448fb215c7
Author: krisztina-zsihovszki
AuthorDate: Wed Feb 15 12:37:31 2023 +0100
NIFI-11158 PutSalesforceObject processor improvements
This closes #6959.
Reviewed-by: Lehel
Reviewed-by: Mark Bathori
Signed-off-by: Peter Turcsanyi
---
.../nifi/util/StandardProcessorTestRunner.java | 13 -
.../main/java/org/apache/nifi/util/TestRunner.java | 7 +++
.../processors/salesforce/PutSalesforceObject.java | 34 ---
.../salesforce/QuerySalesforceObject.java | 8 ++-
.../processors/salesforce/util/RecordExtender.java | 4 +-
.../salesforce/util/SalesforceRestService.java | 15 -
.../salesforce/PutSalesforceObjectIT.java | 67 +++---
.../salesforce/QuerySalesforceObjectIT.java| 2 +
8 files changed, 126 insertions(+), 24 deletions(-)
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
index 63290be359..c113817411 100644
---
a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
+++
b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
@@ -40,6 +40,7 @@ import org.apache.nifi.processor.ProcessSessionFactory;
import org.apache.nifi.processor.Processor;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.registry.VariableDescriptor;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.state.MockStateManager;
@@ -72,6 +73,7 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toSet;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class StandardProcessorTestRunner implements TestRunner {
@@ -366,7 +368,7 @@ public class StandardProcessorTestRunner implements
TestRunner {
.collect(Collectors.toMap(Map.Entry::getKey,
Map.Entry::getValue))
)
-.collect(Collectors.toSet());
+.collect(toSet());
assertEquals(expectedAttributes, actualAttributes);
}
@@ -1055,4 +1057,13 @@ public class StandardProcessorTestRunner implements
TestRunner {
public void setRunSchedule(long runSchedule) {
this.runSchedule = runSchedule;
}
+
+@Override
+public void assertProvenanceEvent(final ProvenanceEventType eventType) {
+Set expectedEventTypes =
Collections.singleton(eventType);
+Set actualEventTypes =
getProvenanceEvents().stream()
+.map(ProvenanceEventRecord::getEventType)
+.collect(toSet());
+assertEquals(expectedEventTypes, actualEventTypes);
+}
}
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
index 7b01ed9709..dbe31e73c8 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
@@ -28,6 +28,7 @@ import org.apache.nifi.processor.ProcessSessionFactory;
import org.apache.nifi.processor.Processor;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.state.MockStateManager;
@@ -1062,4 +1063,10 @@ public interface TestRunner {
*/
void setRunSchedule(long runSchedule);
+/**
+ * Assert that provenance event was created with the specified event type.
+ *
+ * @param eventType Provenance event type
+ */
+ void assertProvenanceEvent(ProvenanceEventType eventType);
}
diff --git
a/nifi-nar-bundles/nifi-salesforce-bundle/nifi-salesforce-processors/src/main/java/org/apache/nifi/processors/salesforce/PutSalesforceObject.java
b/nifi-nar-bundles/nifi-salesforce-bundle/nifi-salesforce-processors/src/main/java/org/apache/nifi/processors/salesforce/PutSalesforceObject.java
index 00d0c2b783..83605800f9 100644
---
a/nifi-nar-bundles/nifi-salesforce-bundle/nifi-salesforce-processors/src/main/java/org/apache/nifi/processors/salesforce/PutSalesforceObject.java
+++
b/nifi-nar-bundles/nifi-salesforce-bundle/nifi-salesforce-processors/src/main/java/org/apache/nifi/processors/salesforce/PutSalesforceObject.java
@@ -20,7 +20,9 @@ import
[nifi] branch NIFI-7060 created (now 6c1913af56)
This is an automated email from the ASF dual-hosted git repository. chriss pushed a change to branch NIFI-7060 in repository https://gitbox.apache.org/repos/asf/nifi.git at 6c1913af56 NIFI-7060 update NiFi and NiFi Registry Docker Image start scripts to populate properties from env vars This branch includes the following new commits: new abfea95ba5 Replace properties defined as env variables new 4424b2ae46 Update environment variable names new 6c1913af56 NIFI-7060 update NiFi and NiFi Registry Docker Image start scripts to populate properties from env vars The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[nifi] 03/03: NIFI-7060 update NiFi and NiFi Registry Docker Image start scripts to populate properties from env vars
This is an automated email from the ASF dual-hosted git repository.
chriss pushed a commit to branch NIFI-7060
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 6c1913af56471a8b762f42237860882ba4b2e445
Author: Chris Sampson
AuthorDate: Tue Feb 28 18:17:30 2023 +
NIFI-7060 update NiFi and NiFi Registry Docker Image start scripts to
populate properties from env vars
---
nifi-docker/dockerhub/sh/common.sh | 8 +-
nifi-docker/dockerhub/sh/nifi_env_from_file.sh | 102 +
nifi-docker/dockerhub/sh/secure.sh | 56 ++---
nifi-docker/dockerhub/sh/start.sh | 228 -
.../sh/update_cluster_state_management.sh | 4 +-
nifi-docker/dockerhub/sh/update_oidc_properties.sh | 21 +-
nifi-docker/dockermaven/pom.xml| 2 +-
.../nifi-registry-docker/dockerhub/sh/common.sh| 14 ++
.../dockerhub/sh/nifi_registry_env_from_file.sh| 98 +
.../nifi-registry-docker/dockerhub/sh/secure.sh| 37 ++--
.../nifi-registry-docker/dockerhub/sh/start.sh | 62 --
.../dockerhub/sh/update_database.sh| 14 +-
.../dockerhub/sh/update_flow_provider.sh | 9 +-
.../dockerhub/sh/update_login_providers.sh | 1 +
.../dockerhub/sh/update_oidc_properties.sh | 18 +-
.../dockermaven/integration-test.sh| 26 +--
.../nifi-registry-docker-maven/dockermaven/pom.xml | 18 +-
.../dockermaven/sh/common.sh | 28 ---
.../dockermaven/sh/secure.sh | 57 --
.../dockermaven/sh/start.sh| 63 --
.../dockermaven/sh/update_bundle_provider.sh | 48 -
.../dockermaven/sh/update_database.sh | 24 ---
.../dockermaven/sh/update_flow_provider.sh | 47 -
.../dockermaven/sh/update_login_providers.sh | 48 -
.../dockermaven/sh/update_oidc_properties.sh | 27 ---
25 files changed, 481 insertions(+), 579 deletions(-)
diff --git a/nifi-docker/dockerhub/sh/common.sh
b/nifi-docker/dockerhub/sh/common.sh
index e3ab9d8369..49d1c0c76b 100755
--- a/nifi-docker/dockerhub/sh/common.sh
+++ b/nifi-docker/dockerhub/sh/common.sh
@@ -20,13 +20,14 @@
prop_replace () {
target_file="${3:-${nifi_props_file}}"
echo "File [${target_file}] replacing [${1}]"
- sed -i -e "s|^$1=.*$|$1=$2|" "${target_file}"
+ # use case-insensitive match for the property name to support mixed-case
properties (e.g. keystoreType)
+ sed -i -e "s|^($1)=.*$|\1=$2|i" "${target_file}"
}
uncomment() {
target_file="${2}"
echo "File [${target_file}] uncommenting [${1}]"
- sed -i -e "s|^\#$1|$1|" "${target_file}"
+ sed -i -e "s|^\#($1)|\1|i" "${target_file}"
}
# 1 - property key to add or replace
@@ -34,7 +35,8 @@ uncomment() {
# 3 - file to perform replacement inline
prop_add_or_replace () {
target_file="${3:-${nifi_props_file}}"
- property_found=$(awk -v property="${1}" 'index($0, property) == 1')
+ # case-insensitive matching of property name (e.g. keystoreType)
+ property_found=$(awk -v property="${1}" 'index(toLower($0), property) == 1')
if [ -z "${property_found}" ]; then
echo "File [${target_file}] adding [${1}]"
echo "$1=$2" >> "${target_file}"
diff --git a/nifi-docker/dockerhub/sh/nifi_env_from_file.sh
b/nifi-docker/dockerhub/sh/nifi_env_from_file.sh
new file mode 100755
index 00..01d5a713e1
--- /dev/null
+++ b/nifi-docker/dockerhub/sh/nifi_env_from_file.sh
@@ -0,0 +1,102 @@
+#!/bin/bash -e
+
+#Licensed to the Apache Software Foundation (ASF) under one or more
+#contributor license agreements. See the NOTICE file distributed with
+#this work for additional information regarding copyright ownership.
+#The ASF licenses this file to You under the Apache License, Version 2.0
+#(the "License"); you may not use this file except in compliance with
+#the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -o pipefail
+
+# Allow environment variables to be set by creating a file with the
+# contents, and setting an environment variable with the suffix _FILE to
+# point to it. This can be used to provide secrets to a container, without
+# the values being specified explicitly when running the container.
+#
+# Note that only supported environment variables are processed, in order
+# to avoid unexpected failures when an environment sets a "*_FILE" variable
+# that doesn't contain a filename.
+#
+# This script is intended to be sourced, not executed, and modifies the
+# environment.
+
[nifi] 02/03: Update environment variable names
This is an automated email from the ASF dual-hosted git repository.
chriss pushed a commit to branch NIFI-7060
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 4424b2ae4653480e468fd3e571122ef13a2b320c
Author: Adrian Martin
AuthorDate: Wed Jan 22 18:53:38 2020 +
Update environment variable names
Set the environment variable names to their counterpart in the
nifi.properties file
---
minifi/minifi-docker/dockerhub/sh/start.sh| 4 ++--
nifi-docker/dockerhub/sh/common.sh| 12 ++--
nifi-docker/dockerhub/sh/secure.sh| 3 ++-
nifi-docker/dockerhub/sh/toolkit.sh | 2 +-
.../dockerhub/sh/update_cluster_state_management.sh | 4 ++--
nifi-docker/dockerhub/sh/update_login_providers.sh| 4 ++--
.../nifi-registry-docker/dockerhub/sh/common.sh | 11 ++-
.../nifi-registry-docker/dockerhub/sh/secure.sh | 19 ++-
.../nifi-registry-docker/dockerhub/sh/start.sh| 7 ---
.../dockerhub/sh/update_bundle_provider.sh| 8
.../dockerhub/sh/update_flow_provider.sh | 10 +-
.../dockerhub/sh/update_login_providers.sh| 6 +++---
.../docker/sh/docker-entrypoint.sh| 12 ++--
13 files changed, 53 insertions(+), 49 deletions(-)
diff --git a/minifi/minifi-docker/dockerhub/sh/start.sh
b/minifi/minifi-docker/dockerhub/sh/start.sh
index f1f19f9467..d5cdd49fab 100755
--- a/minifi/minifi-docker/dockerhub/sh/start.sh
+++ b/minifi/minifi-docker/dockerhub/sh/start.sh
@@ -20,7 +20,7 @@ tail -F "${MINIFI_HOME}/logs/minifi-app.log" &
"${MINIFI_HOME}/bin/minifi.sh" run &
minifi_pid="$!"
-trap "echo Received trapped signal, beginning shutdown...;" KILL TERM HUP INT
EXIT;
+trap "echo Received trapped signal, beginning shutdown...;" TERM HUP INT EXIT;
echo MiNiFi running with PID ${minifi_pid}.
-wait ${minifi_pid}
\ No newline at end of file
+wait ${minifi_pid}
diff --git a/nifi-docker/dockerhub/sh/common.sh
b/nifi-docker/dockerhub/sh/common.sh
index f2121abec8..e3ab9d8369 100755
--- a/nifi-docker/dockerhub/sh/common.sh
+++ b/nifi-docker/dockerhub/sh/common.sh
@@ -18,13 +18,13 @@
# 2 - value to replace
# 3 - file to perform replacement inline
prop_replace () {
- target_file=${3:-${nifi_props_file}}
+ target_file="${3:-${nifi_props_file}}"
echo "File [${target_file}] replacing [${1}]"
sed -i -e "s|^$1=.*$|$1=$2|" "${target_file}"
}
uncomment() {
- target_file=${2}
+ target_file="${2}"
echo "File [${target_file}] uncommenting [${1}]"
sed -i -e "s|^\#$1|$1|" "${target_file}"
}
@@ -33,7 +33,7 @@ uncomment() {
# 2 - property value to use
# 3 - file to perform replacement inline
prop_add_or_replace () {
- target_file=${3:-${nifi_props_file}}
+ target_file="${3:-${nifi_props_file}}"
property_found=$(awk -v property="${1}" 'index($0, property) == 1')
if [ -z "${property_found}" ]; then
echo "File [${target_file}] adding [${1}]"
@@ -44,8 +44,8 @@ prop_add_or_replace () {
}
# NIFI_HOME is defined by an ENV command in the backing Dockerfile
-export nifi_bootstrap_file=${NIFI_HOME}/conf/bootstrap.conf
-export nifi_props_file=${NIFI_HOME}/conf/nifi.properties
-export nifi_toolkit_props_file=${HOME}/.nifi-cli.nifi.properties
+export nifi_bootstrap_file="${NIFI_HOME}/conf/bootstrap.conf"
+export nifi_props_file="${NIFI_HOME}/conf/nifi.properties"
+export nifi_toolkit_props_file="${HOME}/.nifi-cli.nifi.properties"
hostname=$(hostname)
export hostname
diff --git a/nifi-docker/dockerhub/sh/secure.sh
b/nifi-docker/dockerhub/sh/secure.sh
index dd021ccd76..70622a6905 100755
--- a/nifi-docker/dockerhub/sh/secure.sh
+++ b/nifi-docker/dockerhub/sh/secure.sh
@@ -17,6 +17,7 @@
scripts_dir='/opt/nifi/scripts'
+# shellcheck source=./common.sh
[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
# Perform idempotent changes of configuration to support secure environments
@@ -54,7 +55,7 @@ prop_replace 'keyPasswd'
"${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" "${nifi_
prop_replace 'truststore' "${TRUSTSTORE_PATH}"
"${nifi_toolkit_props_file}"
prop_replace 'truststoreType' "${TRUSTSTORE_TYPE}"
"${nifi_toolkit_props_file}"
# shellcheck disable=SC2086
-prop_replace 'truststorePasswd' "${TRUSTSTORE_PASSWORD}"
${nifi_toolkit_props_file}
+prop_replace 'truststorePasswd' "${TRUSTSTORE_PASSWORD}"
"${nifi_toolkit_props_file}"
# Disable HTTP and enable HTTPS
prop_replace 'nifi.web.http.port' ''
diff --git a/nifi-docker/dockerhub/sh/toolkit.sh
b/nifi-docker/dockerhub/sh/toolkit.sh
index 465b38400f..2697620be3 100755
--- a/nifi-docker/dockerhub/sh/toolkit.sh
+++ b/nifi-docker/dockerhub/sh/toolkit.sh
@@ -29,5 +29,5 @@ proxiedEntity=
EOT
cat < "${HOME}/.nifi-cli.config"
-nifi.props=${nifi_toolkit_props_file}
+nifi.props="${nifi_toolkit_props_file}"
EOT
diff --git
[nifi] 01/03: Replace properties defined as env variables
This is an automated email from the ASF dual-hosted git repository.
chriss pushed a commit to branch NIFI-7060
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit abfea95ba5f76563b3bb2975c6972d12f865737a
Author: Adrian Martin
AuthorDate: Wed Jan 22 18:35:33 2020 +
Replace properties defined as env variables
---
nifi-docker/dockerhub/sh/common.sh | 11 +++--
nifi-docker/dockerhub/sh/secure.sh | 38 +++
nifi-docker/dockerhub/sh/start.sh | 54 +-
nifi-docker/dockerhub/sh/toolkit.sh| 7 +--
.../sh/update_cluster_state_management.sh | 4 +-
nifi-docker/dockerhub/sh/update_login_providers.sh | 2 +-
6 files changed, 66 insertions(+), 50 deletions(-)
diff --git a/nifi-docker/dockerhub/sh/common.sh
b/nifi-docker/dockerhub/sh/common.sh
index d78aec494c..f2121abec8 100755
--- a/nifi-docker/dockerhub/sh/common.sh
+++ b/nifi-docker/dockerhub/sh/common.sh
@@ -20,13 +20,13 @@
prop_replace () {
target_file=${3:-${nifi_props_file}}
echo "File [${target_file}] replacing [${1}]"
- sed -i -e "s|^$1=.*$|$1=$2|" ${target_file}
+ sed -i -e "s|^$1=.*$|$1=$2|" "${target_file}"
}
uncomment() {
target_file=${2}
echo "File [${target_file}] uncommenting [${1}]"
- sed -i -e "s|^\#$1|$1|" ${target_file}
+ sed -i -e "s|^\#$1|$1|" "${target_file}"
}
# 1 - property key to add or replace
@@ -37,9 +37,9 @@ prop_add_or_replace () {
property_found=$(awk -v property="${1}" 'index($0, property) == 1')
if [ -z "${property_found}" ]; then
echo "File [${target_file}] adding [${1}]"
-echo "$1=$2" >> ${target_file}
+echo "$1=$2" >> "${target_file}"
else
-prop_replace $1 $2 $3
+prop_replace "$1" "$2" "$3"
fi
}
@@ -47,4 +47,5 @@ prop_add_or_replace () {
export nifi_bootstrap_file=${NIFI_HOME}/conf/bootstrap.conf
export nifi_props_file=${NIFI_HOME}/conf/nifi.properties
export nifi_toolkit_props_file=${HOME}/.nifi-cli.nifi.properties
-export hostname=$(hostname)
+hostname=$(hostname)
+export hostname
diff --git a/nifi-docker/dockerhub/sh/secure.sh
b/nifi-docker/dockerhub/sh/secure.sh
index 4fff214e8c..dd021ccd76 100755
--- a/nifi-docker/dockerhub/sh/secure.sh
+++ b/nifi-docker/dockerhub/sh/secure.sh
@@ -22,21 +22,21 @@ scripts_dir='/opt/nifi/scripts'
# Perform idempotent changes of configuration to support secure environments
echo 'Configuring environment with SSL settings'
-: ${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."}
+: "${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being
used."}"
if [ ! -f "${KEYSTORE_PATH}" ]; then
echo "Keystore file specified (${KEYSTORE_PATH}) does not exist."
exit 1
fi
-: ${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of
the keystore being used."}
-: ${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."}
+: "${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of
the keystore being used."}"
+: "${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being
used."}"
-: ${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being
used."}
+: "${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being
used."}"
if [ ! -f "${TRUSTSTORE_PATH}" ]; then
echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist."
exit 1
fi
-: ${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM)
of the truststore being used."}
-: ${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being
used."}
+: "${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM)
of the truststore being used."}"
+: "${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being
used."}"
prop_replace 'nifi.security.keystore' "${KEYSTORE_PATH}"
prop_replace 'nifi.security.keystoreType' "${KEYSTORE_TYPE}"
@@ -46,36 +46,38 @@ prop_replace 'nifi.security.truststore'
"${TRUSTSTORE_PATH}"
prop_replace 'nifi.security.truststoreType' "${TRUSTSTORE_TYPE}"
prop_replace 'nifi.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}"
-prop_replace 'keystore' "${KEYSTORE_PATH}"
${nifi_toolkit_props_file}
-prop_replace 'keystoreType' "${KEYSTORE_TYPE}"
${nifi_toolkit_props_file}
-prop_replace 'keystorePasswd' "${KEYSTORE_PASSWORD}"
${nifi_toolkit_props_file}
-prop_replace 'keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}"
${nifi_toolkit_props_file}
-prop_replace 'truststore' "${TRUSTSTORE_PATH}"
${nifi_toolkit_props_file}
-prop_replace 'truststoreType' "${TRUSTSTORE_TYPE}"
${nifi_toolkit_props_file}
+# shellcheck disable=SC2154
+prop_replace 'keystore' "${KEYSTORE_PATH}"
"${nifi_toolkit_props_file}"
+prop_replace 'keystoreType'
[nifi] branch main updated: NIFI-11161 Simplified KeyedCipherPropertyEncryptor
This is an automated email from the ASF dual-hosted git repository.
greyp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 9773105841 NIFI-11161 Simplified KeyedCipherPropertyEncryptor
9773105841 is described below
commit 977310584195a59401ef7472cf01a677be8cd465
Author: exceptionfactory
AuthorDate: Thu Feb 9 16:19:05 2023 -0600
NIFI-11161 Simplified KeyedCipherPropertyEncryptor
- Replaced KeyedCipherProvider references with direct Cipher instantiation
This closes #6939
Signed-off-by: Paul Grey
---
.../nifi/encrypt/KeyedCipherPropertyEncryptor.java | 48 +-
.../nifi/encrypt/PropertyEncryptorBuilder.java | 6 +--
.../encrypt/KeyedCipherPropertyEncryptorTest.java | 17 +++-
3 files changed, 26 insertions(+), 45 deletions(-)
diff --git
a/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java
b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java
index 7bcd002b39..5c56fd2dd4 100644
---
a/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java
+++
b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java
@@ -16,30 +16,24 @@
*/
package org.apache.nifi.encrypt;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.apache.nifi.security.util.crypto.KeyedCipherProvider;
-
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
+import javax.crypto.spec.GCMParameterSpec;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Objects;
/**
- * Property Encryptor implementation using Keyed Cipher Provider
+ * Property Encryptor implementation using AES-GCM
*/
class KeyedCipherPropertyEncryptor extends CipherPropertyEncryptor {
private static final int INITIALIZATION_VECTOR_LENGTH = 16;
-private static final int ARRAY_START = 0;
-
-private static final boolean ENCRYPT = true;
+private static final int GCM_TAG_LENGTH_BITS = 128;
-private static final boolean DECRYPT = false;
-
-private final KeyedCipherProvider cipherProvider;
+private static final int ARRAY_START = 0;
-private final EncryptionMethod encryptionMethod;
+private static final String CIPHER_ALGORITHM = "AES/GCM/NoPadding";
private final SecretKey secretKey;
@@ -47,16 +41,11 @@ class KeyedCipherPropertyEncryptor extends
CipherPropertyEncryptor {
private final String description;
-protected KeyedCipherPropertyEncryptor(final KeyedCipherProvider
cipherProvider,
- final EncryptionMethod
encryptionMethod,
- final SecretKey secretKey) {
-this.cipherProvider = cipherProvider;
-this.encryptionMethod = encryptionMethod;
+protected KeyedCipherPropertyEncryptor(final SecretKey secretKey) {
this.secretKey = secretKey;
this.secureRandom = new SecureRandom();
-this.description = String.format("%s Encryption Method [%s] Key
Algorithm [%s] Key Bytes [%d]",
+this.description = String.format("%s Key Algorithm [%s] Key Bytes
[%d]",
getClass().getSimpleName(),
-encryptionMethod.getAlgorithm(),
secretKey.getAlgorithm(),
secretKey.getEncoded().length);
}
@@ -70,7 +59,7 @@ class KeyedCipherPropertyEncryptor extends
CipherPropertyEncryptor {
@Override
protected Cipher getDecryptionCipher(final byte[] encryptedBinary) {
final byte[] initializationVector =
readInitializationVector(encryptedBinary);
-return getCipher(initializationVector, DECRYPT);
+return getCipher(initializationVector, Cipher.DECRYPT_MODE);
}
/**
@@ -81,7 +70,7 @@ class KeyedCipherPropertyEncryptor extends
CipherPropertyEncryptor {
*/
@Override
protected Cipher getEncryptionCipher(byte[] encodedParameters) {
-return getCipher(encodedParameters, ENCRYPT);
+return getCipher(encodedParameters, Cipher.ENCRYPT_MODE);
}
/**
@@ -107,11 +96,14 @@ class KeyedCipherPropertyEncryptor extends
CipherPropertyEncryptor {
return initializationVector;
}
-private Cipher getCipher(final byte[] initializationVector, final boolean
encrypt) {
+private Cipher getCipher(final byte[] initializationVector, final int
cipherMode) {
try {
-return cipherProvider.getCipher(encryptionMethod, secretKey,
initializationVector, encrypt);
+final Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM);
+final GCMParameterSpec parameterSpec = new
GCMParameterSpec(GCM_TAG_LENGTH_BITS, initializationVector);
+cipher.init(cipherMode, secretKey, parameterSpec);
[nifi-minifi-cpp] branch main updated (2c7f989ae -> f27e349b5)
This is an automated email from the ASF dual-hosted git repository. szaszm pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git from 2c7f989ae MINIFICPP-2007 Add rocksdb compression options new c4c80e630 MINIFICPP-2047 added reverseDnsLookup to EL new f27e349b5 MINIFICPP-1887 Add default connection size limits The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: EXPRESSIONS.md | 26 + extensions/expression-language/Expression.cpp | 20 .../tests/ExpressionLanguageTests.cpp | 64 .../tests/unit/FlowJsonTests.cpp | 4 +- .../tests/unit/ProcessorTests.cpp | 10 +- .../tests/unit/YamlConnectionParserTest.cpp| 4 +- libminifi/include/Connection.h | 110 - libminifi/include/core/state/ConnectionStore.h | 4 +- .../include/core/state/nodes/FlowInformation.h | 4 +- libminifi/include/core/state/nodes/QueueMetrics.h | 4 +- libminifi/include/utils/net/DNS.h | 7 ++ libminifi/src/Connection.cpp | 19 ++-- libminifi/src/core/ProcessSession.cpp | 2 +- libminifi/src/core/Processor.cpp | 8 +- .../src/core/flow/StructuredConfiguration.cpp | 4 +- .../src/core/flow/StructuredConnectionParser.cpp | 4 +- libminifi/src/utils/net/DNS.cpp| 36 +++ .../test/persistence-tests/PersistenceTests.cpp| 4 +- libminifi/test/rocksdb-tests/RepoTests.cpp | 2 +- libminifi/test/unit/ConnectionTests.cpp| 41 libminifi/test/unit/MetricsTests.cpp | 4 +- libminifi/test/unit/NetUtilsTest.cpp | 34 ++- libminifi/test/unit/ResponseNodeLoaderTests.cpp| 2 +- 23 files changed, 304 insertions(+), 113 deletions(-)
[nifi-minifi-cpp] 01/02: MINIFICPP-2047 added reverseDnsLookup to EL
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit c4c80e63012fa92c00c7a4f3b5ea383f1ffa55d3
Author: Martin Zink
AuthorDate: Tue Feb 28 13:10:01 2023 +0100
MINIFICPP-2047 added reverseDnsLookup to EL
Closes #1510
Signed-off-by: Marton Szasz
---
EXPRESSIONS.md | 26 +
extensions/expression-language/Expression.cpp | 20 +++
.../tests/ExpressionLanguageTests.cpp | 64 ++
libminifi/include/utils/net/DNS.h | 7 +++
libminifi/src/utils/net/DNS.cpp| 36
libminifi/test/unit/NetUtilsTest.cpp | 34 +++-
6 files changed, 185 insertions(+), 2 deletions(-)
diff --git a/EXPRESSIONS.md b/EXPRESSIONS.md
index 79b9e0c11..97ba68341 100644
--- a/EXPRESSIONS.md
+++ b/EXPRESSIONS.md
@@ -224,6 +224,7 @@ token, filename.
- [`hostname`](#hostname)
- [`UUID`](#uuid)
- [`literal`](#literal)
+- [`reverseDnsLookup`](#reversednslookup)
### Evaluating Multiple Attributes
@@ -1590,6 +1591,31 @@ to evaluate additional functions against.
${allMatchingAttributes('a.*'):count()} ):gt(3)}` returns true if there are
more than 3 attributes whose names begin with the letter a.
+### reverseDnsLookup
+
+**Description**: Performs a reverse DNS lookup on an ip address, and returns
the corresponding hostname.
+
+**Subject Type**: No subject
+
+**Arguments**:
+
+| Argument | Description
|
+|---|--|
+| IP address| The ip address to perform the reverse DNS
lookup on.
|
+| Timeout duration milliseconds | Optional parameter that specifies the
timeout duration of the operation in milliseconds. If not specified, defaults
to 5000. |
+
+
+**Return Type**: String
+
+**Examples**:
+
+| Expression | Value|
+||--|
+| `${reverseDnsLookup('127.0.0.1')}` | `localhost` |
+| `${reverseDnsLookup('::1')}` | `localhost` |
+| `${reverseDnsLookup('2001:4860:4860::'), 100}` | `dns.google` |
+
+
## Evaluating Multiple Attributes
When it becomes necessary to evaluate the same conditions against multiple
diff --git a/extensions/expression-language/Expression.cpp
b/extensions/expression-language/Expression.cpp
index c52411c74..f1570775c 100644
--- a/extensions/expression-language/Expression.cpp
+++ b/extensions/expression-language/Expression.cpp
@@ -64,6 +64,10 @@
#include "Driver.h"
#include "date/tz.h"
+#include "utils/net/DNS.h"
+#include "utils/expected.h"
+
+using namespace std::literals::chrono_literals;
namespace org::apache::nifi::minifi::expression {
@@ -183,6 +187,20 @@ Value expr_ip(const std::vector& /*args*/) {
return {};
}
+Value expr_reverseDnsLookup(const std::vector& args) {
+ std::string ip_address_str = args[0].asString();
+
+ std::chrono::steady_clock::duration timeout_duration = 5s;
+ if (args.size() > 1) {
+timeout_duration = std::chrono::milliseconds(args[1].asUnsignedLong());
+ }
+
+ return utils::net::addressFromString(ip_address_str)
+ | utils::flatMap([timeout_duration](const auto& ip_address) { return
utils::net::reverseDnsLookup(ip_address, timeout_duration);})
+ | utils::map([](const auto& hostname)-> Value { return Value(hostname);
})
+ | utils::valueOrElse([&](std::error_code error_code) { throw
std::system_error(error_code);});
+}
+
Value expr_uuid(const std::vector& /*args*/) {
return Value(utils::IdGenerator::getIdGenerator()->generate().to_string());
}
@@ -1339,6 +1357,8 @@ Expression make_dynamic_function(const std::string
_name, const std::ve
return make_dynamic_function_incomplete(function_name,
args, 0);
} else if (function_name == "ip") {
return make_dynamic_function_incomplete(function_name, args, 0);
+ } else if (function_name == "reverseDnsLookup") {
+return
make_dynamic_function_incomplete(function_name, args, 1);
} else if (function_name == "UUID") {
return make_dynamic_function_incomplete(function_name, args, 0);
} else if (function_name == "toUpper") {
diff --git a/extensions/expression-language/tests/ExpressionLanguageTests.cpp
b/extensions/expression-language/tests/ExpressionLanguageTests.cpp
index 77cd383af..477000981 100644
--- a/extensions/expression-language/tests/ExpressionLanguageTests.cpp
+++ b/extensions/expression-language/tests/ExpressionLanguageTests.cpp
@@ -41,6 +41,7 @@
#include
[nifi-minifi-cpp] 02/02: MINIFICPP-1887 Add default connection size limits
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit f27e349b508c502218ca702b78fad2ef931f4f83
Author: Martin Zink
AuthorDate: Tue Feb 28 13:11:23 2023 +0100
MINIFICPP-1887 Add default connection size limits
default limits: 2000 flow files / 100MB data size
Closes #1501
Signed-off-by: Marton Szasz
---
.../tests/unit/FlowJsonTests.cpp | 4 +-
.../tests/unit/ProcessorTests.cpp | 10 +-
.../tests/unit/YamlConnectionParserTest.cpp| 4 +-
libminifi/include/Connection.h | 110 -
libminifi/include/core/state/ConnectionStore.h | 4 +-
.../include/core/state/nodes/FlowInformation.h | 4 +-
libminifi/include/core/state/nodes/QueueMetrics.h | 4 +-
libminifi/src/Connection.cpp | 19 ++--
libminifi/src/core/ProcessSession.cpp | 2 +-
libminifi/src/core/Processor.cpp | 8 +-
.../src/core/flow/StructuredConfiguration.cpp | 4 +-
.../src/core/flow/StructuredConnectionParser.cpp | 4 +-
.../test/persistence-tests/PersistenceTests.cpp| 4 +-
libminifi/test/rocksdb-tests/RepoTests.cpp | 2 +-
libminifi/test/unit/ConnectionTests.cpp| 41
libminifi/test/unit/MetricsTests.cpp | 4 +-
libminifi/test/unit/ResponseNodeLoaderTests.cpp| 2 +-
17 files changed, 119 insertions(+), 111 deletions(-)
diff --git a/extensions/standard-processors/tests/unit/FlowJsonTests.cpp
b/extensions/standard-processors/tests/unit/FlowJsonTests.cpp
index b7d1a47f8..495965122 100644
--- a/extensions/standard-processors/tests/unit/FlowJsonTests.cpp
+++ b/extensions/standard-processors/tests/unit/FlowJsonTests.cpp
@@ -150,8 +150,8 @@ TEST_CASE("NiFi flow json format is correctly parsed") {
REQUIRE(connection1->getSource() == proc);
REQUIRE(connection1->getDestination() == funnel);
REQUIRE(connection1->getRelationships() ==
(std::set{{"a", ""}, {"b", ""}}));
- REQUIRE(connection1->getMaxQueueSize() == 7);
- REQUIRE(connection1->getMaxQueueDataSize() == 11_KiB);
+ REQUIRE(connection1->getBackpressureThresholdCount() == 7);
+ REQUIRE(connection1->getBackpressureThresholdDataSize() == 11_KiB);
REQUIRE(13s == connection1->getFlowExpirationDuration());
auto connection2 = connection_map.at("----0008");
diff --git a/extensions/standard-processors/tests/unit/ProcessorTests.cpp
b/extensions/standard-processors/tests/unit/ProcessorTests.cpp
index b7afc647c..c3b43856a 100644
--- a/extensions/standard-processors/tests/unit/ProcessorTests.cpp
+++ b/extensions/standard-processors/tests/unit/ProcessorTests.cpp
@@ -245,7 +245,7 @@ TEST_CASE("TestConnectionFull", "[ConnectionFull]") {
std::shared_ptr repo =
std::static_pointer_cast(test_repo);
std::shared_ptr connection =
std::make_shared(test_repo, content_repo, "GFF2Connection");
- connection->setMaxQueueSize(5);
+ connection->setBackpressureThresholdCount(5);
connection->addRelationship(core::Relationship("success", "description"));
@@ -270,8 +270,8 @@ TEST_CASE("TestConnectionFull", "[ConnectionFull]") {
auto session = std::make_shared(context);
- REQUIRE(session->outgoingConnectionsFull("success") == false);
- REQUIRE(connection->isFull() == false);
+ CHECK_FALSE(session->outgoingConnectionsFull("success"));
+ CHECK_FALSE(connection->backpressureThresholdReached());
processor->incrementActiveTasks();
processor->setScheduledState(core::ScheduledState::RUNNING);
@@ -279,8 +279,8 @@ TEST_CASE("TestConnectionFull", "[ConnectionFull]") {
session->commit();
- REQUIRE(connection->isFull());
- REQUIRE(session->outgoingConnectionsFull("success"));
+ CHECK(connection->backpressureThresholdReached());
+ CHECK(session->outgoingConnectionsFull("success"));
}
TEST_CASE("LogAttributeTest", "[getfileCreate3]") {
diff --git
a/extensions/standard-processors/tests/unit/YamlConnectionParserTest.cpp
b/extensions/standard-processors/tests/unit/YamlConnectionParserTest.cpp
index 44d346ed4..0444c71e6 100644
--- a/extensions/standard-processors/tests/unit/YamlConnectionParserTest.cpp
+++ b/extensions/standard-processors/tests/unit/YamlConnectionParserTest.cpp
@@ -183,8 +183,8 @@ TEST_CASE("Connections components are parsed from yaml",
"[YamlConfiguration]")
"drop empty: \n"});
flow::Node connection_node{std::make_shared(yaml_node)};
StructuredConnectionParser yaml_connection_parser(connection_node,
"test_node", parent_ptr, logger);
-CHECK(0 == yaml_connection_parser.getWorkQueueSize());
-CHECK(0 == yaml_connection_parser.getWorkQueueDataSize());
+CHECK(minifi::Connection::DEFAULT_BACKPRESSURE_THRESHOLD_COUNT ==
yaml_connection_parser.getWorkQueueSize());
+
