(nifi) branch support/nifi-1.x updated: NIFI-12890: Refactor HadoopDBCPConnectionPool to extend AbstractDBCPConnectionPool
This is an automated email from the ASF dual-hosted git repository. turcsanyi pushed a commit to branch support/nifi-1.x in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/support/nifi-1.x by this push: new b2842fd00a NIFI-12890: Refactor HadoopDBCPConnectionPool to extend AbstractDBCPConnectionPool b2842fd00a is described below commit b2842fd00af1728bc479b899fd99b8a092e82884 Author: lehelb AuthorDate: Wed Mar 13 14:22:19 2024 -0500 NIFI-12890: Refactor HadoopDBCPConnectionPool to extend AbstractDBCPConnectionPool This closes #8623. Signed-off-by: Peter Turcsanyi --- .../nifi-hadoop-dbcp-service/pom.xml | 5 + .../apache/nifi/dbcp/HadoopDBCPConnectionPool.java | 370 +++-- .../nifi/dbcp/HadoopDBCPConnectionPoolTest.java| 15 +- 3 files changed, 126 insertions(+), 264 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml index f2624a37f5..b138e1cd62 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml @@ -28,6 +28,11 @@ 1.26.0-SNAPSHOT provided + +org.apache.nifi +nifi-dbcp-base +1.26.0-SNAPSHOT + org.apache.nifi nifi-api diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java index 8293386564..17917f8489 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java @@ -30,15 +30,15 @@ import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnDisabled; import org.apache.nifi.annotation.lifecycle.OnEnabled; import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.RequiredPermission; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.resource.ResourceCardinality; import org.apache.nifi.components.resource.ResourceType; -import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.controller.ControllerServiceInitializationContext; +import org.apache.nifi.dbcp.utils.DBCPProperties; +import org.apache.nifi.dbcp.utils.DataSourceConfiguration; import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.hadoop.KerberosProperties; import org.apache.nifi.hadoop.SecurityUtil; @@ -50,7 +50,6 @@ import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.security.krb.KerberosKeytabUser; import org.apache.nifi.security.krb.KerberosLoginException; import org.apache.nifi.security.krb.KerberosPasswordUser; -import org.apache.nifi.security.krb.KerberosUser; import javax.security.auth.login.LoginException; import java.io.File; @@ -58,22 +57,38 @@ import java.io.IOException; import java.lang.reflect.UndeclaredThrowableException; import java.security.PrivilegedExceptionAction; import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import static org.apache.nifi.dbcp.utils.DBCPProperties.DATABASE_URL; +import static org.apache.nifi.dbcp.utils.DBCPProperties.DB_DRIVERNAME; +import static org.apache.nifi.dbcp.utils.DBCPProperties.DB_PASSWORD; +import static org.apache.nifi.dbcp.utils.DBCPProperties.DB_USER; +import static org.apache.nifi.dbcp.utils.DBCPProperties.EVICTION_RUN_PERIOD; +import static org.apache.nifi.dbcp.utils.DBCPProperties.KERBEROS_USER_SERVICE; +import static org.apache.nifi.dbcp.utils.DBCPProperties.MAX_CONN_LIFETIME; +import static org.apache.nifi.dbcp.utils.DBCPProperties.MAX_IDLE; +import static org.apache.nifi.dbcp.utils.DBCPProperties.MAX_TOTAL_CONNECTIONS; +import static
(nifi) branch main updated: NIFI-12890: Refactor HadoopDBCPConnectionPool to extend AbstractDBCPConnectionPool
This is an automated email from the ASF dual-hosted git repository. turcsanyi pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/main by this push: new 419a9cc73b NIFI-12890: Refactor HadoopDBCPConnectionPool to extend AbstractDBCPConnectionPool 419a9cc73b is described below commit 419a9cc73bfe5410c0ff399f4feb8e6350b7609d Author: lehelb AuthorDate: Wed Mar 13 14:22:19 2024 -0500 NIFI-12890: Refactor HadoopDBCPConnectionPool to extend AbstractDBCPConnectionPool This closes #8619. Signed-off-by: Peter Turcsanyi --- .../nifi-hadoop-dbcp-service/pom.xml | 5 + .../apache/nifi/dbcp/HadoopDBCPConnectionPool.java | 374 +++-- .../nifi/dbcp/HadoopDBCPConnectionPoolTest.java| 15 +- 3 files changed, 135 insertions(+), 259 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml index 37a0370672..396c82bd1a 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml @@ -28,6 +28,11 @@ 2.0.0-SNAPSHOT provided + +org.apache.nifi +nifi-dbcp-base +2.0.0-SNAPSHOT + org.apache.nifi nifi-api diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java index 5d75daf2ca..09c6302990 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java @@ -16,21 +16,6 @@ */ package org.apache.nifi.dbcp; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.UndeclaredThrowableException; -import java.security.PrivilegedExceptionAction; -import java.sql.Connection; -import java.sql.SQLException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import javax.security.auth.login.LoginException; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -45,15 +30,15 @@ import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnDisabled; import org.apache.nifi.annotation.lifecycle.OnEnabled; import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.RequiredPermission; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.resource.ResourceCardinality; import org.apache.nifi.components.resource.ResourceType; -import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.controller.ControllerServiceInitializationContext; +import org.apache.nifi.dbcp.utils.DBCPProperties; +import org.apache.nifi.dbcp.utils.DataSourceConfiguration; import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.hadoop.KerberosProperties; import org.apache.nifi.hadoop.SecurityUtil; @@ -64,15 +49,45 @@ import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.security.krb.KerberosKeytabUser; import org.apache.nifi.security.krb.KerberosLoginException; import org.apache.nifi.security.krb.KerberosPasswordUser; -import org.apache.nifi.security.krb.KerberosUser; + +import javax.security.auth.login.LoginException; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.PrivilegedExceptionAction; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.nifi.dbcp.utils.DBCPProperties.DATABASE_URL;
(nifi) branch main updated: General improvements (#8627)
This is an automated email from the ASF dual-hosted git repository. scottyaslan pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/main by this push: new d78e817fe8 General improvements (#8627) d78e817fe8 is described below commit d78e817fe87c0168be49f71b871da2ef463b17e5 Author: Rob Fellows AuthorDate: Thu Apr 11 13:48:36 2024 -0400 General improvements (#8627) * [NIFI-13021] - Fix - Status History dialog changes size when switching selected metric * [NIFI-13018] - Fix - Refreshing Node Status History when open for the cluster, makes the incorrect backend call * [NIFI-13023] - Fix - Right click on canvas while search results open shows browser right-click menu * [NIFI-13024] - Improvement - Support Enter to create selected extension type * [NIFI-13000] - Improvement - Prevent text selection in header, flow status, canvas, extension creation dialog table. * prettier * color udpates for new theme classes, turn of text selection on the status history charts --- .../flow-designer/ui/canvas/canvas.component.html | 5 +- .../header/flow-status/flow-status.component.html | 2 +- .../ui/canvas/header/header.component.scss | 4 ++ .../new-canvas-item/new-canvas-item.component.scss | 4 ++ .../ui/canvas/header/search/search.component.html | 20 .../ui/canvas/header/search/search.component.ts| 4 +- .../change-version-progress-dialog.html| 4 +- .../edit-parameter-context.component.html | 2 +- .../flowfile-table/flowfile-table.component.html | 2 +- .../parameter-groups-table.component.html | 4 +- .../parameter-providers-table.component.html | 4 +- .../state/status-history/status-history.effects.ts | 36 - .../component-state/component-state.component.html | 4 +- .../controller-service-references.component.html | 10 +--- .../disable-controller-service.component.html | 6 +-- .../enable-controller-service.component.html | 6 +-- .../extension-creation.component.html | 33 .../extension-creation.component.ts| 60 -- .../ui/common/navigation/navigation.component.html | 2 +- .../parameter-references.component.html| 10 +--- .../status-history-chart.component.html| 2 +- .../status-history/status-history.component.scss | 5 ++ 22 files changed, 161 insertions(+), 68 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.html index 9054ddb323..85fb83e8ab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.html @@ -18,7 +18,10 @@ - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/flow-status/flow-status.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/flow-status/flow-status.component.html index ff5d3bd30f..6be27a229f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/flow-status/flow-status.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/flow-status/flow-status.component.html @@ -15,7 +15,7 @@ ~ limitations under the License. --> - + @if (clusterSummary?.clustered) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/header.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/header.component.scss index 2944f98194..f0dfba977a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/header.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/header.component.scss @@ -14,3 +14,7 @@ * See
(nifi-minifi-cpp) 03/05: MINIFICPP-2282 Support re-encryption of sensitive properties
This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git commit 40bc3b16ad8516fa83e13eb826415600daabca03 Author: Ferenc Gerlits AuthorDate: Thu Apr 11 18:40:57 2024 +0200 MINIFICPP-2282 Support re-encryption of sensitive properties Added a new --re-encrypt option to the flow-config action of the encrypt-config binary, which can decrypt the sensitive properties using the .old key and re-encrypt them with a new key (either supported by the user or auto-generated). Closes #1739 Signed-off-by: Marton Szasz --- encrypt-config/EncryptConfig.cpp | 26 ++-- encrypt-config/EncryptConfig.h | 5 +- encrypt-config/EncryptConfigMain.cpp | 20 ++- encrypt-config/FlowConfigEncryptor.cpp | 156 + encrypt-config/FlowConfigEncryptor.h | 20 ++- .../include/core/flow/AdaptiveConfiguration.h | 3 +- libminifi/include/core/flow/FlowSerializer.h | 17 ++- libminifi/include/core/json/JsonFlowSerializer.h | 4 +- libminifi/include/core/yaml/YamlFlowSerializer.h | 4 +- .../include/utils/crypto/EncryptionProvider.h | 2 +- libminifi/include/utils/crypto/EncryptionUtils.h | 4 + libminifi/include/utils/crypto/ciphers/XSalsa20.h | 6 +- libminifi/src/core/flow/AdaptiveConfiguration.cpp | 7 +- libminifi/src/core/flow/FlowSerializer.cpp | 52 +++ .../src/core/flow/StructuredConfiguration.cpp | 3 + libminifi/src/core/json/JsonFlowSerializer.cpp | 25 ++-- libminifi/src/core/yaml/YamlFlowSerializer.cpp | 23 +-- libminifi/src/utils/crypto/EncryptionUtils.cpp | 14 +- libminifi/test/unit/JsonFlowSerializerTests.cpp| 127 ++--- libminifi/test/unit/YamlFlowSerializerTests.cpp| 121 +--- 20 files changed, 489 insertions(+), 150 deletions(-) diff --git a/encrypt-config/EncryptConfig.cpp b/encrypt-config/EncryptConfig.cpp index a1a3c89c9..6e0b07296 100644 --- a/encrypt-config/EncryptConfig.cpp +++ b/encrypt-config/EncryptConfig.cpp @@ -57,7 +57,7 @@ EncryptConfig::EncryptConfig(const std::string& minifi_home) : minifi_home_(mini std::filesystem::current_path(minifi_home_); } -bool EncryptConfig::isReencrypting() const { +bool EncryptConfig::isReEncrypting() const { encrypt_config::ConfigFile bootstrap_file{std::ifstream{bootstrapFilePath()}}; std::string decryption_key_name = utils::string::join_pack(ENCRYPTION_KEY_PROPERTY_NAME, ".old"); @@ -200,16 +200,20 @@ void EncryptConfig::encryptSensitiveValuesInMinifiProperties() const { } void EncryptConfig::encryptSensitiveValuesInFlowConfig( -const std::optional& component_id, const std::optional& property_name, const std::optional& property_value) const { - if (!component_id && !property_name && !property_value) { -EncryptionKeys keys = getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME); -flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys, minifi_home_, flowConfigPath()); - } else if (component_id && property_name && property_value) { -EncryptionKeys keys = getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME); -flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys, minifi_home_, flowConfigPath(), *component_id, *property_name, *property_value); - } else { -throw std::runtime_error("either all of --component-id, --property-name and --property-value should be given (for batch mode) or none of them (for interactive mode)"); - } +bool re_encrypt, const std::optional& component_id, const std::optional& property_name, const std::optional& property_value) const { + EncryptionKeys keys = getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME); + flow_config_encryptor::EncryptionRequest request_type = [&] { +if (re_encrypt) { + return flow_config_encryptor::EncryptionRequest{flow_config_encryptor::EncryptionType::ReEncrypt}; +} else if (!component_id && !property_name && !property_value) { + return flow_config_encryptor::EncryptionRequest{flow_config_encryptor::EncryptionType::Interactive}; +} else if (component_id && property_name && property_value) { + return flow_config_encryptor::EncryptionRequest{*component_id, *property_name, *property_value}; +} else { + throw std::runtime_error("either all of --component-id, --property-name and --property-value should be given (for batch mode) or none of them (for interactive mode)"); +} + }(); + flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys, minifi_home_, flowConfigPath(), request_type); } } // namespace org::apache::nifi::minifi::encrypt_config diff --git a/encrypt-config/EncryptConfig.h b/encrypt-config/EncryptConfig.h index 1f48f48fb..5d32fa500 100644 --- a/encrypt-config/EncryptConfig.h +++
(nifi-minifi-cpp) 02/05: MINIFICPP-2293 Support installing python dependencies defined inline
This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git commit 24d30aa0eebdd423ed2c66153f13a6cf76a2e632 Author: Gabor Gyimesi AuthorDate: Thu Apr 11 18:40:19 2024 +0200 MINIFICPP-2293 Support installing python dependencies defined inline Closes #1727 Signed-off-by: Marton Szasz --- docker/test/integration/cluster/ContainerStore.py | 3 ++ .../test/integration/cluster/DockerTestCluster.py | 3 ++ docker/test/integration/cluster/ImageStore.py | 37 ++-- .../cluster/containers/MinifiContainer.py | 7 ++- .../features/MiNiFi_integration_test_driver.py | 3 ++ docker/test/integration/features/python.feature| 1 + docker/test/integration/features/steps/steps.py| 2 + extensions/python/ExecutePythonProcessor.cpp | 1 + extensions/python/PYTHON.md| 7 ++- extensions/python/PythonCreator.h | 7 +++ extensions/python/PythonDependencyInstaller.cpp| 42 +- extensions/python/PythonDependencyInstaller.h | 2 + .../utils/inline_dependency_installer.py | 51 ++ msi/WixWin.wsi.in | 6 +++ 14 files changed, 154 insertions(+), 18 deletions(-) diff --git a/docker/test/integration/cluster/ContainerStore.py b/docker/test/integration/cluster/ContainerStore.py index 0dad14fe0..8b17ee6ba 100644 --- a/docker/test/integration/cluster/ContainerStore.py +++ b/docker/test/integration/cluster/ContainerStore.py @@ -358,6 +358,9 @@ class ContainerStore: def use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi(self): self.minifi_options.use_nifi_python_processors_with_virtualenv_packages_installed = True +def remove_python_requirements_txt_in_minifi(self): +self.minifi_options.remove_python_requirements_txt = True + def set_yaml_in_minifi(self): self.minifi_options.config_format = "yaml" diff --git a/docker/test/integration/cluster/DockerTestCluster.py b/docker/test/integration/cluster/DockerTestCluster.py index 4114b3f6d..5451f67b6 100644 --- a/docker/test/integration/cluster/DockerTestCluster.py +++ b/docker/test/integration/cluster/DockerTestCluster.py @@ -116,6 +116,9 @@ class DockerTestCluster: def use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi(self): self.container_store.use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi() +def remove_python_requirements_txt_in_minifi(self): +self.container_store.remove_python_requirements_txt_in_minifi() + def set_yaml_in_minifi(self): self.container_store.set_yaml_in_minifi() diff --git a/docker/test/integration/cluster/ImageStore.py b/docker/test/integration/cluster/ImageStore.py index bddc9e417..7a6e6cad6 100644 --- a/docker/test/integration/cluster/ImageStore.py +++ b/docker/test/integration/cluster/ImageStore.py @@ -24,6 +24,12 @@ from textwrap import dedent import os +class PythonOptions: +REQUIREMENTS_FILE = 0 +SYSTEM_INSTALLED_PACKAGES = 1 +INLINE_DEFINED_PACKAGES = 2 + + class ImageStore: def __init__(self): self.client = docker.from_env() @@ -43,9 +49,11 @@ class ImageStore: if container_engine == "minifi-cpp-sql": image = self.__build_minifi_cpp_sql_image() elif container_engine == "minifi-cpp-nifi-python": -image = self.__build_minifi_cpp_image_with_nifi_python_processors() +image = self.__build_minifi_cpp_image_with_nifi_python_processors(PythonOptions.REQUIREMENTS_FILE) elif container_engine == "minifi-cpp-nifi-python-system-python-packages": -image = self.__build_minifi_cpp_image_with_nifi_python_processors('RUN pip3 install "langchain<=0.17.0"') +image = self.__build_minifi_cpp_image_with_nifi_python_processors(PythonOptions.SYSTEM_INSTALLED_PACKAGES) +elif container_engine == "minifi-cpp-nifi-with-inline-python-dependencies": +image = self.__build_minifi_cpp_image_with_nifi_python_processors(PythonOptions.INLINE_DEFINED_PACKAGES) elif container_engine == "http-proxy": image = self.__build_http_proxy_image() elif container_engine == "postgresql-server": @@ -97,10 +105,25 @@ class ImageStore: return self.__build_image(dockerfile) -def __build_minifi_cpp_image_with_nifi_python_processors(self, additional_cmd=""): +def __build_minifi_cpp_image_with_nifi_python_processors(self, python_option): parse_document_url = "https://raw.githubusercontent.com/apache/nifi/rel/nifi-; + NifiContainer.NIFI_VERSION + "/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ParseDocument.py" chunk_document_url = "https://raw.githubusercontent.com/apache/nifi/rel/nifi-; +
(nifi-minifi-cpp) branch main updated (f533eb024 -> 33549357a)
This is an automated email from the ASF dual-hosted git repository. szaszm pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git from f533eb024 MINIFICPP-2321 Update Splunk tests for new docker image new fe0dcd0af MINIFICPP-2277 Add virtualenv support for python processors new 24d30aa0e MINIFICPP-2293 Support installing python dependencies defined inline new 40bc3b16a MINIFICPP-2282 Support re-encryption of sensitive properties new e3041e075 MINIFICPP-2324 Make start after install optional new 33549357a MINIFICPP-2327 Fix Loki protobuf code generation on Windows using msbuild The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: cmake/Grpc.cmake | 9 +- conf/minifi.properties | 7 +- docker/python-verify/conda.Dockerfile | 6 +- docker/python-verify/installed.Dockerfile | 2 +- docker/python-verify/venv.Dockerfile | 2 +- docker/test/integration/cluster/ContainerStore.py | 13 +- .../test/integration/cluster/DockerTestCluster.py | 13 +- docker/test/integration/cluster/ImageStore.py | 54 +- .../cluster/containers/MinifiContainer.py | 19 ++- .../features/MiNiFi_integration_test_driver.py | 13 +- docker/test/integration/features/environment.py| 1 - docker/test/integration/features/python.feature| 11 +- docker/test/integration/features/steps/steps.py| 15 ++ encrypt-config/EncryptConfig.cpp | 26 +-- encrypt-config/EncryptConfig.h | 5 +- encrypt-config/EncryptConfigMain.cpp | 20 ++- encrypt-config/FlowConfigEncryptor.cpp | 156 ++--- encrypt-config/FlowConfigEncryptor.h | 20 ++- encrypt-config/tests/ConfigFileEncryptorTests.cpp | 2 +- encrypt-config/tests/ConfigFileTests.cpp | 8 +- encrypt-config/tests/resources/minifi.properties | 7 +- ...th-additional-sensitive-props.minifi.properties | 7 +- extensions/python/ExecutePythonProcessor.cpp | 1 + extensions/python/ExecutePythonProcessor.h | 8 - extensions/python/PYTHON.md| 43 - ...PythonScriptException.h => PythonConfigState.h} | 21 ++- extensions/python/PythonCreator.h | 12 +- extensions/python/PythonDependencyInstaller.cpp| 188 + extensions/python/PythonDependencyInstaller.h | 52 ++ extensions/python/PythonInterpreter.cpp| 78 + .../python/{PyException.h => PythonInterpreter.h} | 30 +++- extensions/python/PythonObjectFactory.h| 8 - extensions/python/PythonScriptEngine.cpp | 60 +-- extensions/python/PythonScriptEngine.h | 57 +-- .../utils/inline_dependency_installer.py | 51 ++ .../include/core/flow/AdaptiveConfiguration.h | 3 +- libminifi/include/core/flow/FlowSerializer.h | 17 +- libminifi/include/core/json/JsonFlowSerializer.h | 4 +- libminifi/include/core/yaml/YamlFlowSerializer.h | 4 +- libminifi/include/properties/Configuration.h | 3 + .../include/utils/crypto/EncryptionProvider.h | 2 +- libminifi/include/utils/crypto/EncryptionUtils.h | 4 + libminifi/include/utils/crypto/ciphers/XSalsa20.h | 6 +- libminifi/src/Configuration.cpp| 6 +- libminifi/src/core/flow/AdaptiveConfiguration.cpp | 7 +- libminifi/src/core/flow/FlowSerializer.cpp | 52 ++ .../src/core/flow/StructuredConfiguration.cpp | 3 + libminifi/src/core/json/JsonFlowSerializer.cpp | 25 ++- libminifi/src/core/yaml/YamlFlowSerializer.cpp | 23 ++- libminifi/src/utils/crypto/EncryptionUtils.cpp | 14 +- .../test/resources/encrypted.minifi.properties | 7 +- libminifi/test/unit/JsonFlowSerializerTests.cpp| 127 -- libminifi/test/unit/YamlFlowSerializerTests.cpp| 121 +++-- msi/WixWin.wsi.in | 64 --- 54 files changed, 1152 insertions(+), 365 deletions(-) copy extensions/python/{PythonScriptException.h => PythonConfigState.h} (73%) create mode 100644 extensions/python/PythonDependencyInstaller.cpp create mode 100644 extensions/python/PythonDependencyInstaller.h create mode 100644 extensions/python/PythonInterpreter.cpp copy extensions/python/{PyException.h => PythonInterpreter.h} (65%) create mode 100644 extensions/python/pythonprocessors/nifi_python_processors/utils/inline_dependency_installer.py create mode 100644 libminifi/src/core/flow/FlowSerializer.cpp
(nifi-minifi-cpp) 05/05: MINIFICPP-2327 Fix Loki protobuf code generation on Windows using msbuild
This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git commit 33549357ae986be01a826a01464ff6fe1c7648db Author: Gabor Gyimesi AuthorDate: Thu Apr 11 18:50:32 2024 +0200 MINIFICPP-2327 Fix Loki protobuf code generation on Windows using msbuild Closes #1757 Signed-off-by: Marton Szasz --- cmake/Grpc.cmake | 9 ++--- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cmake/Grpc.cmake b/cmake/Grpc.cmake index bdf3870a0..7c881bae7 100644 --- a/cmake/Grpc.cmake +++ b/cmake/Grpc.cmake @@ -42,10 +42,5 @@ add_dependencies(grpc++ OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB) set(GRPC_INCLUDE_DIR "${grpc_SOURCE_DIR}/include" CACHE STRING "" FORCE) set(PROTOBUF_INCLUDE_DIR "${protobuf_SOURCE_DIR}/src" CACHE STRING "" FORCE) -if (WIN32) - set(PROTOBUF_COMPILER "${protobuf_BINARY_DIR}/protoc.exe" CACHE STRING "" FORCE) - set(GRPC_CPP_PLUGIN "${grpc_BINARY_DIR}/grpc_cpp_plugin.exe" CACHE STRING "" FORCE) -else() - set(PROTOBUF_COMPILER "${protobuf_BINARY_DIR}/protoc" CACHE STRING "" FORCE) - set(GRPC_CPP_PLUGIN "${grpc_BINARY_DIR}/grpc_cpp_plugin" CACHE STRING "" FORCE) -endif() +set(PROTOBUF_COMPILER "$" CACHE STRING "" FORCE) +set(GRPC_CPP_PLUGIN "$" CACHE STRING "" FORCE)
(nifi-minifi-cpp) 04/05: MINIFICPP-2324 Make start after install optional
This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git commit e3041e075c8725cb415833bfe9c5095c6fa88af7 Author: Ferenc Gerlits AuthorDate: Thu Apr 11 18:50:05 2024 +0200 MINIFICPP-2324 Make start after install optional In the Windows installer (msi), add a new checkbox, enabled by default, to control whether the service will be started at the end of the installation. To disable auto-start from the command line, use the AUTOSTART="" property in the msiexec.exe command. Closes #1754 Signed-off-by: Marton Szasz --- msi/WixWin.wsi.in | 58 ++- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/msi/WixWin.wsi.in b/msi/WixWin.wsi.in index 0f8a422e7..717bcdea7 100644 --- a/msi/WixWin.wsi.in +++ b/msi/WixWin.wsi.in @@ -50,6 +50,7 @@ ${WIX_EXTRA_FEATURES} + @@ -182,10 +183,10 @@ ${WIX_EXTRA_FEATURES} - - - - + + + + 1 @@ -193,40 +194,44 @@ ${WIX_EXTRA_FEATURES} - + + - - - + + ENABLEC2="1" - - + + ENABLEC2="1" - - + + ENABLEC2="1" - - + + ENABLEC2="1" - - + + @@ -269,12 +274,11 @@ ${WIX_EXTRA_FEATURES} http://localhost:8181/heartbeat; /> http://localhost:8181/acknowledge; /> - + - @@ -351,10 +355,6 @@ ${WIX_EXTRA_FEATURES} RestartServiceDelayInSeconds="60"/> - + - + + + + + + ${WIX_EXTRA_COMPONENTS}
(nifi-minifi-cpp) 01/05: MINIFICPP-2277 Add virtualenv support for python processors
This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git commit fe0dcd0affad9fe58ef67551e06b6b92789ef73c Author: Gabor Gyimesi AuthorDate: Thu Apr 11 18:36:11 2024 +0200 MINIFICPP-2277 Add virtualenv support for python processors Closes #1721 Signed-off-by: Marton Szasz --- conf/minifi.properties | 7 +- docker/python-verify/conda.Dockerfile | 6 +- docker/python-verify/installed.Dockerfile | 2 +- docker/python-verify/venv.Dockerfile | 2 +- docker/test/integration/cluster/ContainerStore.py | 10 +- .../test/integration/cluster/DockerTestCluster.py | 10 +- docker/test/integration/cluster/ImageStore.py | 23 ++- .../cluster/containers/MinifiContainer.py | 16 +- .../features/MiNiFi_integration_test_driver.py | 10 +- docker/test/integration/features/environment.py| 1 - docker/test/integration/features/python.feature| 10 +- docker/test/integration/features/steps/steps.py| 13 ++ encrypt-config/tests/ConfigFileEncryptorTests.cpp | 2 +- encrypt-config/tests/ConfigFileTests.cpp | 8 +- encrypt-config/tests/resources/minifi.properties | 7 +- ...th-additional-sensitive-props.minifi.properties | 7 +- extensions/python/ExecutePythonProcessor.h | 8 - extensions/python/PYTHON.md| 38 - extensions/python/PythonConfigState.h | 37 + extensions/python/PythonCreator.h | 5 +- extensions/python/PythonDependencyInstaller.cpp| 168 + extensions/python/PythonDependencyInstaller.h | 50 ++ extensions/python/PythonInterpreter.cpp| 78 ++ extensions/python/PythonInterpreter.h | 48 ++ extensions/python/PythonObjectFactory.h| 8 - extensions/python/PythonScriptEngine.cpp | 60 +--- extensions/python/PythonScriptEngine.h | 57 +-- libminifi/include/properties/Configuration.h | 3 + libminifi/src/Configuration.cpp| 6 +- .../test/resources/encrypted.minifi.properties | 7 +- 30 files changed, 543 insertions(+), 164 deletions(-) diff --git a/conf/minifi.properties b/conf/minifi.properties index 9ba8a0a08..67de200a3 100644 --- a/conf/minifi.properties +++ b/conf/minifi.properties @@ -135,10 +135,15 @@ nifi.nar.deploy.directory=${MINIFI_HOME}/minifi-jni/nardeploy nifi.nar.docs.directory=${MINIFI_HOME}/minifi-jni/nardocs # must be comma separated nifi.jvm.options=-Xmx1G -nifi.python.processor.dir=${MINIFI_HOME}/minifi-python/ # Publish metrics to external consumers # nifi.metrics.publisher.agent.identifier= # nifi.metrics.publisher.class=PrometheusMetricsPublisher # nifi.metrics.publisher.PrometheusMetricsPublisher.port=9936 # nifi.metrics.publisher.metrics=QueueMetrics,RepositoryMetrics,GetFileMetrics,DeviceInfoNode,FlowInformation + +# Python processor properties +nifi.python.processor.dir=${MINIFI_HOME}/minifi-python/ +nifi.python.virtualenv.directory=${MINIFI_HOME}/minifi-python-env +nifi.python.install.packages.automatically=true +# nifi.python.env.setup.binary=python3 diff --git a/docker/python-verify/conda.Dockerfile b/docker/python-verify/conda.Dockerfile index 911e2bd22..d8bd4b64b 100644 --- a/docker/python-verify/conda.Dockerfile +++ b/docker/python-verify/conda.Dockerfile @@ -31,14 +31,12 @@ USER root RUN wget https://repo.anaconda.com/archive/Anaconda3-2023.09-0-Linux-x86_64.sh -P /tmp \ && echo "6c8a4abb36fbb711dc055b7049a23bbfd61d356de9468b41c5140f8a11abd851 /tmp/Anaconda3-2023.09-0-Linux-x86_64.sh" | sha256sum -c \ && bash /tmp/Anaconda3-2023.09-0-Linux-x86_64.sh -b -p /opt/conda \ -&& chown -R ${USER}:${USER} /opt/conda \ -&& mkdir /home/${USER} \ -&& chown -R ${USER}:${USER} /home/${USER} +&& chown -R ${USER}:${USER} /opt/conda USER ${USER} RUN ${CONDA_HOME}/bin/conda init bash -RUN ${CONDA_HOME}/bin/conda install langchain -c conda-forge +RUN ${CONDA_HOME}/bin/conda install "langchain<=0.17.0" -c conda-forge WORKDIR ${MINIFI_HOME} diff --git a/docker/python-verify/installed.Dockerfile b/docker/python-verify/installed.Dockerfile index c49bd0657..aa2c24abc 100644 --- a/docker/python-verify/installed.Dockerfile +++ b/docker/python-verify/installed.Dockerfile @@ -32,7 +32,7 @@ ENV MINIFI_HOME ${MINIFI_BASE_DIR}/minifi-current ENV MINIFI_VERSIONED_HOME ${MINIFI_BASE_DIR}/nifi-minifi-cpp-${MINIFI_VERSION} -RUN groupadd -g ${GID} ${USER} && useradd -g ${GID} ${USER} && \ +RUN groupadd -g ${GID} ${USER} && useradd -m -g ${GID} ${USER} && \ install -d -o ${USER} -g ${USER} ${MINIFI_BASE_DIR} && ln -s ${MINIFI_VERSIONED_HOME} ${MINIFI_HOME} ADD ${ARCHIVE_LOCATION} ${MINIFI_BASE_DIR} diff --git a/docker/python-verify/venv.Dockerfile
(nifi) branch main updated (bc589e273d -> 44852fb5d5)
This is an automated email from the ASF dual-hosted git repository. rfellows pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git from bc589e273d NIFI-13020 Fix legacy flow C2 update issue add 44852fb5d5 [NIFI-13022] rename semantic color classes to avoid conflict with angular material class names (#8628) No new revisions were added by this update. Summary of changes: .../feature/access-policies.component.html | 2 +- .../add-tenant-to-policy-dialog.component.html | 4 +- .../policy-table/policy-table.component.html | 7 +- ..._component-access-policies.component-theme.scss | 33 .../component-access-policies.component.html | 16 +- .../global-access-policies.component.html | 8 +- .../bulletins/feature/bulletins.component.html | 2 +- .../bulletin-board/bulletin-board.component.html | 7 +- .../pages/counters/feature/counters.component.html | 2 +- .../counter-listing/counter-listing.component.html | 4 +- .../counter-table/counter-table.component.html | 4 +- .../flow-configuration-history.component.html | 2 +- .../action-details/action-details.component.html | 6 +- ...ow-configuration-history-listing.component.html | 6 +- ...flow-configuration-history-table.component.html | 2 +- .../flow-designer/service/canvas-utils.service.ts | 2 +- .../service/manager/port-manager.service.ts| 4 +- .../manager/process-group-manager.service.ts | 24 +-- .../service/manager/processor-manager.service.ts | 2 +- .../remote-process-group-manager.service.ts| 2 +- .../ui/canvas/_canvas.component-theme.scss | 4 - .../ui/canvas/footer/_footer.component-theme.scss | 2 - .../ui/canvas/footer/footer.component.html | 2 +- .../_navigation-control.component-theme.scss | 9 +- .../navigation-control.component.html | 14 +- .../_operation-control.component-theme.scss| 21 +-- .../operation-control.component.html | 27 ++- .../flow-status/_flow-status.component-theme.scss | 5 - .../header/flow-status/flow-status.component.html | 34 ++-- .../header/flow-status/flow-status.component.ts| 12 +- .../header/search/_search.component-theme.scss | 3 - .../ui/canvas/header/search/search.component.html | 4 +- .../destination-funnel.component.html | 4 +- .../destination-output-port.component.html | 4 +- .../destination-process-group.component.html | 2 +- .../destination-processor.component.html | 4 +- ...destination-remote-process-group.component.html | 2 +- .../_prioritizers.component-theme.scss | 2 - .../prioritizers/prioritizers.component.html | 9 +- .../source-funnel/source-funnel.component.html | 4 +- .../source-input-port.component.html | 4 +- .../source-process-group.component.html| 2 +- .../source-processor.component.html| 4 +- .../source-remote-process-group.component.html | 2 +- .../change-version-dialog.html | 10 +- .../change-version-progress-dialog.html| 6 +- .../import-from-registry.component.html| 2 +- .../local-changes-table/local-changes-table.html | 7 +- .../save-version-dialog.component.html | 8 +- .../create-process-group.component.html| 4 +- .../edit-processor/edit-processor.component.html | 6 +- .../relationship-settings.component.html | 2 +- .../run-duration-slider.component.html | 4 +- .../edit-remote-process-group.component.html | 4 +- .../controller-services.component.html | 8 +- .../edit-remote-port.component.html| 2 +- .../manage-remote-ports.component.html | 16 +- .../login/ui/login-form/login-form.component.html | 8 +- .../feature/parameter-contexts.component.html | 2 +- .../edit-parameter-context.component.html | 8 +- ...ameter-context-inheritance.component-theme.scss | 1 - .../parameter-context-inheritance.component.html | 6 +- .../parameter-context-listing.component.html | 6 +- .../parameter-context-table.component.html | 16 +- .../parameter-table/parameter-table.component.html | 20 +- .../process-group-references.component.html| 2 +- .../provenance/feature/provenance.component.html | 2 +- .../lineage/lineage.component.ts | 2 +- .../provenance-event-table.component.html | 20 +- .../flowfile-dialog/flowfile-dialog.component.html | 18 +- .../flowfile-table/flowfile-table.component.html | 12 +- .../ui/queue-listing/queue-listing.component.html | 4 +- .../pages/settings/feature/settings.component.html | 2 +- .../edit-flow-analysis-rule.component.html | 6 +- .../flow-analysis-rule-table.component.html| 30