[GitHub] [nifi-minifi-cpp] lordgamez commented on a diff in pull request #1490: MINIFICPP-2022 Add valid repository size metrics for all repositories
lordgamez commented on code in PR #1490:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1490#discussion_r1172203729
##
libminifi/test/rocksdb-tests/RepoTests.cpp:
##
@@ -515,4 +519,221 @@ TEST_CASE("FlowFileRepository synchronously pushes
existing flow files") {
}
}
+TEST_CASE("Test getting flow file repository size properties",
"[TestGettingRepositorySize]") {
+
LogTestController::getInstance().setDebug();
+
LogTestController::getInstance().setDebug();
+
LogTestController::getInstance().setDebug();
+
LogTestController::getInstance().setDebug();
+ TestController testController;
+ auto dir = testController.createTempDirectory();
+
+ std::shared_ptr repository;
+ auto expected_is_full = false;
+ uint64_t expected_max_repo_size = 0;
+ SECTION("FlowFileRepository") {
+repository = std::make_shared("ff",
dir.string(), 0ms, 0, 1ms);
+ }
+
+ SECTION("ProvenanceRepository") {
+repository =
std::make_shared("ff", dir.string(),
0ms, 0, 1ms);
+ }
+
+ SECTION("VolatileFlowFileRepository") {
+repository =
std::make_shared("ff",
dir.string(), 0ms, 10, 1ms);
+expected_is_full = true;
+expected_max_repo_size = 7;
+ }
+
+ SECTION("VolatileProvenanceRepository") {
+repository =
std::make_shared("ff",
dir.string(), 0ms, 10, 1ms);
+expected_is_full = true;
+expected_max_repo_size = 7;
+ }
+ auto configuration = std::make_shared();
+ repository->initialize(configuration);
+
+ auto flow_file = std::make_shared();
+
+ for (auto i = 0; i < 100; ++i) {
+flow_file->addAttribute("key" + std::to_string(i), "testattributevalue" +
std::to_string(i));
+ }
+
+ auto original_size = repository->getRepositorySize();
+ using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime;
+ REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(5),
[&original_size, &repository] {
+ auto old_size = original_size;
+ original_size = repository->getRepositorySize();
+ return old_size == original_size;
+},
+std::chrono::milliseconds(50)));
+ REQUIRE(true == flow_file->Persist(repository));
+ auto flow_file_2 = std::make_shared();
+ REQUIRE(true == flow_file_2->Persist(repository));
+
+ repository->flush();
+ repository->stop();
+
+ auto new_size = repository->getRepositorySize();
+ REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(5), [&new_size,
&repository] {
+ auto old_size = new_size;
+ new_size = repository->getRepositorySize();
+ return old_size == new_size;
+},
+std::chrono::milliseconds(50)));
+ REQUIRE(new_size > original_size);
+ REQUIRE(expected_is_full == repository->isFull());
+ REQUIRE(expected_max_repo_size == repository->getMaxRepositorySize());
+ REQUIRE(2 == repository->getRepositoryEntryCount());
+}
+
+TEST_CASE("Test getting noop repository size properties",
"[TestGettingRepositorySize]") {
+ TestController testController;
+ auto dir = testController.createTempDirectory();
+
+ auto repository = minifi::core::createRepository("NoOpRepository", "ff");
+
+ repository->initialize(std::make_shared());
+
+ auto flow_file = std::make_shared();
+
+ flow_file->addAttribute("key", "testattributevalue");
+
+ repository->flush();
+ repository->stop();
+
+ REQUIRE(repository->getRepositorySize() == 0);
+ REQUIRE(!repository->isFull());
+ REQUIRE(repository->getMaxRepositorySize() == 0);
+ REQUIRE(repository->getRepositoryEntryCount() == 0);
+}
+
+TEST_CASE("Test getting content repository size properties",
"[TestGettingRepositorySize]") {
+ LogTestController::getInstance().setDebug();
+
LogTestController::getInstance().setDebug();
+
LogTestController::getInstance().setDebug();
+
LogTestController::getInstance().setDebug();
+ TestController testController;
+ auto dir = testController.createTempDirectory();
+
+ auto repository =
std::make_shared("ff", dir.string(), 0ms,
0, 1ms);
+
+ auto content_repo_dir = testController.createTempDirectory();
+ auto configuration = std::make_shared();
+
configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default,
content_repo_dir.string());
+ std::string content = "content";
+
configuration->set(minifi::Configure::nifi_volatile_repository_options_content_max_bytes,
std::to_string(content.size()));
+
+ std::shared_ptr content_repo;
+ auto expected_is_full = false;
+ uint64_t expected_max_repo_size = 0;
+ SECTION("FileSystemRepository") {
+content_repo = std::make_shared();
+ }
+
+ SECTION("VolatileContentRepository") {
+content_repo =
std::make_shared("content");
+expected_is_full = true;
+expected_max_repo_size = content.size();
+ }
+
+ SECTION("DatabaseContentRepository") {
+content_repo =
std::make_shared();
+ }
+
+ content_repo->initialize(configuration);
+
+ repository->initialize(configuration);
+ repository->loadComponent(content_repo);
+ auto original_content_repo_size = content_repo->getRepositorySize();
+
+ auto flow_file = std::mak
[jira] [Updated] (NIFI-11402) PutBigQuery processor case sensitive and Append Record Count issues
[
https://issues.apache.org/jira/browse/NIFI-11402?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Csaba Bejan updated NIFI-11402:
---
Resolution: Fixed
Status: Resolved (was: Patch Available)
> PutBigQuery processor case sensitive and Append Record Count issues
> ---
>
> Key: NIFI-11402
> URL: https://issues.apache.org/jira/browse/NIFI-11402
> Project: Apache NiFi
> Issue Type: Bug
>Affects Versions: 1.18.0, 1.19.0, 1.20.0, 1.19.1, 1.21.0
>Reporter: Julien G.
>Assignee: Pierre Villard
>Priority: Major
> Fix For: 2.0.0, 1.22.0
>
> Time Spent: 50m
> Remaining Estimate: 0h
>
> The {{PutBigQuery}} processor seems to have to some issues. I detected 2
> issues that can be quite blocking.
> For the first one, if you set a hight value in the {{Append Record Count}}
> property in my case 500 000 and that you have a big flowfile (number of
> records and size, in my case 54 000 records for a size of 74MB) you will get
> an error because the message to send is too big. That is quite normal.
> {code:java}
> PutBigQuery[id=16da3694-c886-3b31-929e-0dc81be51bf7] Stream processing
> failed: java.lang.RuntimeException: io.grpc.StatusRuntimeException:
> INVALID_ARGUMENT: MessageSize is too large. Max allow: 1000 Actual:
> 13593340
> - Caused by: io.grpc.StatusRuntimeException: INVALID_ARGUMENT: MessageSize is
> too large. Max allow: 1000 Actual: 13593340
> {code}
> So you replace the value with a smaller one, but the error message remains
> the same. Even if you reduce your flowfile to a single record, you will still
> get the error. The only way to fix this is to delete the processor and readd
> it, then reduce the value of the property before running it. Seems to be an
> issue here.
> It would also be interesting to give information about the limit of the
> message sent in the processor documentation because the limit in the previous
> implementation of the {{PutBigQueryStreaming}} and {{PutBigQueryBatch}}
> processors was quite straightforward and linked to the size of the file sent.
> But now the limit is on the {{Message}} but it doesn't really correspond to
> the size of the FlowFile or the number of records in it.
> The second issue occure if you are using upper case in your field name. For
> example, you have a table with the following schema:
> {code:java}
> timestamp | TIMESTAMP | REQUIRED
> original_payload | STRING | NULLABLE
> error_message | STRING | REQUIRED
> error_type | STRING REQUIRED
> error_subType | STRING | REQUIRED
> {code}
> and try to put the following event in it:
> {code:java}
> {
> "original_payload" : "",
> "error_message" : "XX",
> "error_type" : "XX",
> "error_subType" : "XXX",
> "timestamp" : "2023-04-07T10:31:45Z"
> }
> {code}
> (in my case this event was in Avro)
> You will get the following telling you that the required field
> {{error_subtype}} is missing:
> {code:java}
> Cannot convert record to message:
> com.google.protobuf.UninitializedMessageException: Message missing required
> fields: error_subtype
> {code}
> So to fix it, you need to change your Avro Schema and put {{error_subtype}}
> instead of {{error_subType}} in it.
> BigQuery columns aren't case sensitive so it should be ok to put a field with
> upper case but it's not. In the previous implementation of the
> {{PutBigQueryStreaming}} and {{PutBigQueryBatch}}, we were able to use upper
> case in the schema fields. So it should still be the case.
> {color:#DE350B}If you get this error, the flowfile will not go in the failure
> queue but just disappear.{color}
> Link to the slack thread:
> https://apachenifi.slack.com/archives/C0L9VCD47/p1680866688318739
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
[GitHub] [nifi-minifi-cpp] martinzink commented on a diff in pull request #1560: MINIFICPP-2101 Compilation fix in PutSFTPTests (libc++)
martinzink commented on code in PR #1560:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1560#discussion_r1172250106
##
libminifi/test/unit/FileUtilsTests.cpp:
##
@@ -477,3 +477,27 @@ TEST_CASE("FileUtils::get_relative_path",
"[TestGetRelativePath]") {
REQUIRE(*FileUtils::get_relative_path(path, base_path / "") ==
std::filesystem::path("subdir") / "file.log");
REQUIRE(*FileUtils::get_relative_path(base_path, base_path) == ".");
}
+
+TEST_CASE("file_clock to system_clock conversion tests") {
+ using namespace std::chrono;
+
+ static_assert(system_clock::duration::period::num ==
file_clock::duration::period::num);
+ constexpr auto lowest_den = system_clock::period::den <
file_clock::period::den ? system_clock::period::den : file_clock::period::den;
Review Comment:
sure thing, good idea, I've changed it in
https://github.com/apache/nifi-minifi-cpp/pull/1560/commits/41f3dd4ef1416a1bf5ab38d7052e08678adc2d88
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] martinzink commented on a diff in pull request #1560: MINIFICPP-2101 Compilation fix in PutSFTPTests (libc++)
martinzink commented on code in PR #1560:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1560#discussion_r1172250412
##
libminifi/test/unit/FileUtilsTests.cpp:
##
@@ -477,3 +477,27 @@ TEST_CASE("FileUtils::get_relative_path",
"[TestGetRelativePath]") {
REQUIRE(*FileUtils::get_relative_path(path, base_path / "") ==
std::filesystem::path("subdir") / "file.log");
REQUIRE(*FileUtils::get_relative_path(base_path, base_path) == ".");
}
+
+TEST_CASE("file_clock to system_clock conversion tests") {
+ using namespace std::chrono;
+
+ static_assert(system_clock::duration::period::num ==
file_clock::duration::period::num);
+ constexpr auto lowest_den = system_clock::period::den <
file_clock::period::den ? system_clock::period::den : file_clock::period::den;
+ using LeastPreciseDurationType =
duration, std::ratio>;
Review Comment:
good idea, changed it in
https://github.com/apache/nifi-minifi-cpp/pull/1560/commits/41f3dd4ef1416a1bf5ab38d7052e08678adc2d88
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] martinzink commented on a diff in pull request #1560: MINIFICPP-2101 Compilation fix in PutSFTPTests (libc++)
martinzink commented on code in PR #1560:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1560#discussion_r1172250693
##
extensions/standard-processors/tests/unit/ListFileTests.cpp:
##
@@ -107,8 +104,12 @@ TEST_CASE_METHOD(ListFileTestFixture, "Input Directory is
empty", "[testListFile
REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true),
minifi::Exception);
}
+std::string get_last_modified_time_formatted_string(const
std::filesystem::path& path) {
+ return date::format("%Y-%m-%dT%H:%M:%SZ",
std::chrono::time_point_cast(utils::file::to_sys(*utils::file::last_write_time(path;
Review Comment:
you are right, I've changed it in
https://github.com/apache/nifi-minifi-cpp/pull/1560/commits/41f3dd4ef1416a1bf5ab38d7052e08678adc2d88
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] lordgamez commented on a diff in pull request #1558: MINIFICPP-2100 Add RockyLinux Dockerfile
lordgamez commented on code in PR #1558:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1558#discussion_r1172252957
##
docker/rockylinux/Dockerfile:
##
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM rockylinux:8
+LABEL maintainer="Apache NiFi "
+
+ARG MINIFI_VERSION
+ARG UID=1000
+ARG GID=1000
+
+# MINIFI_OPTIONS will be passed directly to cmake
+# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON)
+ARG MINIFI_OPTIONS=""
+ARG CMAKE_BUILD_TYPE=Release
+ARG DOCKER_SKIP_TESTS=ON
+ARG DOCKER_MAKE_TARGET="all"
+
+ENV MINIFI_BASE_DIR /opt/minifi
+ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION
+ENV USER minificpp
+
+RUN mkdir -p $MINIFI_BASE_DIR
+COPY . ${MINIFI_BASE_DIR}
+
+# Install the system dependencies needed for a build
+# gpsd-devel and ccache are in EPEL
+RUN yum -y install epel-release && yum -y install sudo git which make
libarchive ccache ca-certificates gcc-toolset-11-toolchain && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_GPS=ON"; then yum -y install
gpsd-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_JNI=ON"; then yum -y install
java-1.8.0-openjdk maven; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then yum -y install
libpcap-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_USB_CAMERA=ON"; then yum -y
install libpng-devel libusbx-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_OPENCV=ON" || echo
"$MINIFI_OPTIONS" | grep -q "ENABLE_BUSTACHE=ON"; then yum -y install
boost-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_LUA_SCRIPTING=ON"; then yum -y
install lua-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PYTHON_SCRIPTING=ON"; then yum
-y install python36-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SFTP=ON" && [
"${DOCKER_SKIP_TESTS}" == "OFF" ]; then yum -y install java-1.8.0-openjdk
maven; fi
Review Comment:
Updated in b04a27f9a52eb3f0cdd0c637aa815b5e0183d93f
##
docker/rockylinux/Dockerfile:
##
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM rockylinux:8
+LABEL maintainer="Apache NiFi "
+
+ARG MINIFI_VERSION
+ARG UID=1000
+ARG GID=1000
+
+# MINIFI_OPTIONS will be passed directly to cmake
+# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON)
+ARG MINIFI_OPTIONS=""
+ARG CMAKE_BUILD_TYPE=Release
+ARG DOCKER_SKIP_TESTS=ON
+ARG DOCKER_MAKE_TARGET="all"
+
+ENV MINIFI_BASE_DIR /opt/minifi
+ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION
+ENV USER minificpp
+
+RUN mkdir -p $MINIFI_BASE_DIR
+COPY . ${MINIFI_BASE_DIR}
+
+# Install the system dependencies needed for a build
+# gpsd-devel and ccache are in EPEL
+RUN yum -y install epel-release && yum -y install sudo git which make
libarchive ccache ca-certificates gcc-toolset-11-toolchain && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_GPS=ON"; then yum -y install
gpsd-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_JNI=ON"; then yum -y install
java-1.8.0-openjdk maven; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then yum -y install
libpcap-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_USB_CAMERA=ON"; then yum -y
install libpng-devel libusbx-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_OPENCV=ON" || echo
"$MINIFI_OPTIONS" | grep -q "ENABLE_BUSTACHE=ON"; then yum -y install
boost-devel; fi
[GitHub] [nifi-minifi-cpp] lordgamez commented on a diff in pull request #1562: MINIFICPP-2106 Add 'Use Path Style Access' property to PutS3Object
lordgamez commented on code in PR #1562:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1562#discussion_r1172254661
##
extensions/aws/tests/MockS3RequestSender.h:
##
@@ -250,6 +252,10 @@ class MockS3RequestSender : public
minifi::aws::s3::S3RequestSender {
return client_config_;
}
+ bool getUserVirtualAddressing() const {
Review Comment:
Good catch, fixed in 70dd30bb57534a2eb38d3fc5cb76150900a8c076
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] martinzink commented on a diff in pull request #1562: MINIFICPP-2106 Add 'Use Path Style Access' property to PutS3Object
martinzink commented on code in PR #1562:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1562#discussion_r1171511190
##
extensions/aws/processors/PutS3Object.cpp:
##
@@ -73,6 +73,11 @@ void PutS3Object::onSchedule(const
std::shared_ptr &contex
}
logger_->log_debug("PutS3Object: Server Side Encryption [%s]",
server_side_encryption_);
+ bool use_path_style_access = false;
+ if (context->getProperty(UsePathStyleAccess.getName(),
use_path_style_access)) {
+use_virtual_addressing_ = !use_path_style_access;
+ }
+
Review Comment:
Nitpick but I think we could improve the readability and limit the scope of
the use_path_style_access by using the optional returning getProperty
```suggestion
if (auto use_path_style_access =
context->getProperty(UsePathStyleAccess)) {
use_virtual_addressing_ = !*use_path_style_access;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] lordgamez commented on a diff in pull request #1562: MINIFICPP-2106 Add 'Use Path Style Access' property to PutS3Object
lordgamez commented on code in PR #1562:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1562#discussion_r1172352494
##
extensions/aws/processors/PutS3Object.cpp:
##
@@ -73,6 +73,11 @@ void PutS3Object::onSchedule(const
std::shared_ptr &contex
}
logger_->log_debug("PutS3Object: Server Side Encryption [%s]",
server_side_encryption_);
+ bool use_path_style_access = false;
+ if (context->getProperty(UsePathStyleAccess.getName(),
use_path_style_access)) {
+use_virtual_addressing_ = !use_path_style_access;
+ }
+
Review Comment:
Updated in 0cc5232b2bda754eefbc48f7f9ff99fca40131f1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[jira] [Created] (NIFI-11472) Make PutFTP processor more multithread friendly
Daniel Salwerowicz created NIFI-11472:
-
Summary: Make PutFTP processor more multithread friendly
Key: NIFI-11472
URL: https://issues.apache.org/jira/browse/NIFI-11472
Project: Apache NiFi
Issue Type: Bug
Components: Core Framework
Affects Versions: 1.21.0
Environment: CentOS server, Java 11
Reporter: Daniel Salwerowicz
Problem happens when a PutFTP is set to run several concurrent tasks and two
(or more ) FlowFiles come in and both need to create the same directory. One of
them will create directory and succeed immediately while the other will try to
create directory, but fail since it already exist, throw an error, the FlowFile
will then be penalized and on second run will succeed.
While it is not the biggest error, as files are getting transferred in the end,
but the bulletins and errors are annoying, especially in production environment
where you don't want to get unnecessary errors.
We found that the solution involves a simple change to the {{FTPTransfer.java}}
class in:
{{nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java}}
On line 398 and {{ensureDirectoryExists}} method you can simply add another if
check which double checks that the directory exists when it fails to create one.
{code:java}
final boolean cdSuccessful = setWorkingDirectory(remoteDirectory);
if (!cdSuccessful) {
if (client.makeDirectory(remoteDirectory)) {
logger.debug("Remote Directory not found: created directory [{}]",
remoteDirectory);
} else if (setWorkingDirectory(remoteDirectory)) {
// Double check that the dir exists as it might have been created in
another thread
throw new IOException("Failed to create remote directory " +
remoteDirectory);
}
}{code}
We have created a fix and will submit it as a Pull Request on GitHub
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
[GitHub] [nifi-minifi-cpp] szaszm commented on a diff in pull request #1552: MINIFICPP-2089 prefix EventData in flat JSON output so it doesnt need t…
szaszm commented on code in PR #1552:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1552#discussion_r1172407058
##
extensions/windows-event-log/wel/JSONUtils.cpp:
##
@@ -59,29 +60,50 @@ rapidjson::Value xmlDocumentToJSON(const pugi::xml_node&
node, rapidjson::Docume
return children;
}
-void simplifiedGenericXmlToJson(const pugi::xml_node& source_node,
rapidjson::Value& output_value, rapidjson::Document& allocator_source_document,
bool flatten = false) {
+void simplifiedGenericXmlToJson(const pugi::xml_node& source_node,
+rapidjson::Value& output_value,
+rapidjson::Document::AllocatorType& allocator,
+std::optional prefix_for_flat_structure) {
Review Comment:
We should document somewhere, that this has value when the format is
flattened, and empty when the format is simple json. A comment or a boolean
like `const bool is_flattened = prefix_for_flat_structure.has_value();` could
both achieve this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi] MormonJesus69420 opened a new pull request, #7184: NIFI-11472 Make PutFTP processor more multithread friendly
MormonJesus69420 opened a new pull request, #7184:
URL: https://github.com/apache/nifi/pull/7184
Add an extra check during directory creation to see if directory wasn't
already created in another thread.
From Issue:
Problem happens when a PutFTP is set to run several concurrent tasks and two
(or more ) FlowFiles come in and both need to create the same directory. One of
them will create directory and succeed immediately while the other will try to
create directory, but fail since it already exist, throw an error, the FlowFile
will then be penalized and on second run will succeed.
While it is not the biggest error, as files are getting transferred in the
end, but the bulletins and errors are annoying, especially in production
environment where you don't want to get unnecessary errors.
We found that the solution involves a simple change to the FTPTransfer.java
class in:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java
On line 398 and ensureDirectoryExists method you can simply add another if
check which double checks that the directory exists when it fails to create one.
```java
final boolean cdSuccessful = setWorkingDirectory(remoteDirectory);
if (!cdSuccessful) {
if (client.makeDirectory(remoteDirectory)) {
logger.debug("Remote Directory not found: created directory [{}]",
remoteDirectory);
} else if (setWorkingDirectory(remoteDirectory)) {
// Double check that the dir exists as it might have been created
in another thread
throw new IOException("Failed to create remote directory " +
remoteDirectory);
}
}
```
# Summary
[NIFI-11472](https://issues.apache.org/jira/browse/NIFI-11472)
# Tracking
Please complete the following tracking steps prior to pull request creation.
### Issue Tracking
- [x] [Apache NiFi Jira](https://issues.apache.org/jira/browse/NIFI) issue
created
### Pull Request Tracking
- [x] Pull Request title starts with Apache NiFi Jira issue number, such as
`NIFI-0`
- [x] Pull Request commit message starts with Apache NiFi Jira issue number,
as such `NIFI-0`
### Pull Request Formatting
- [x] Pull Request based on current revision of the `main` branch
- [x] Pull Request refers to a feature branch with one commit containing
changes
# Verification
Please indicate the verification steps performed prior to pull request
creation.
### Build
- [x] Build completed using `mvn clean install -P contrib-check`
- [x] JDK 11
- [ ] JDK 17
### Licensing
- [x] New dependencies are compatible with the [Apache License
2.0](https://apache.org/licenses/LICENSE-2.0) according to the [License
Policy](https://www.apache.org/legal/resolved.html)
- [x] New dependencies are documented in applicable `LICENSE` and `NOTICE`
files
### Documentation
- [x] Documentation formatting appears as expected in rendered files
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[jira] [Updated] (MINIFICPP-1641) Parallel running of docker tests
[ https://issues.apache.org/jira/browse/MINIFICPP-1641?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Martin Zink updated MINIFICPP-1641: --- Description: Currently docker tests (docker-verify target) is run sequentially. This is not scalable. Consider exploring the parallelization options of behave, multiprocess and multithreading under Python. Also pay attention to available CPU cores on GitHub CI and if it is possible to increase them. The main problem is that containers are referenced by name only. We should postfix them based on the scenario and iron out any problems that arise. * attributes_to_json.feature * azure_storage.feature * core_functionality.feature * defragtextflowfiles.feature * elasticsearch.feature * file_system_operations.feature * google_cloud_storage.feature * hashcontent.feature * http.feature * https.feature * kafka.feature * kubernetes_logging.feature * kubernetes_metrics.feature * lua_script.feature * minifi_c2_server.feature * mqtt.feature * network_listener.feature * opcua.feature * opensearch.feature * prometheus.feature * python.feature * python_script.feature * python_with_modules.feature * replace_text.feature * routetext.feature * s2s.feature * s3.feature * splunk.feature * sql.feature * syslog_listener.feature was:Currently docker tests (docker-verify target) is run sequentially. This is not scalable. Consider exploring the parallelization options of behave, multiprocess and multithreading under Python. Also pay attention to available CPU cores on GitHub CI and if it is possible to increase them. > Parallel running of docker tests > > > Key: MINIFICPP-1641 > URL: https://issues.apache.org/jira/browse/MINIFICPP-1641 > Project: Apache NiFi MiNiFi C++ > Issue Type: Test >Reporter: Ádám Markovics >Assignee: Martin Zink >Priority: Minor > > Currently docker tests (docker-verify target) is run sequentially. This is > not scalable. Consider exploring the parallelization options of behave, > multiprocess and multithreading under Python. Also pay attention to available > CPU cores on GitHub CI and if it is possible to increase them. > The main problem is that containers are referenced by name only. We should > postfix them based on the scenario and iron out any problems that arise. > * attributes_to_json.feature > * azure_storage.feature > * core_functionality.feature > * defragtextflowfiles.feature > * elasticsearch.feature > * file_system_operations.feature > * google_cloud_storage.feature > * hashcontent.feature > * http.feature > * https.feature > * kafka.feature > * kubernetes_logging.feature > * kubernetes_metrics.feature > * lua_script.feature > * minifi_c2_server.feature > * mqtt.feature > * network_listener.feature > * opcua.feature > * opensearch.feature > * prometheus.feature > * python.feature > * python_script.feature > * python_with_modules.feature > * replace_text.feature > * routetext.feature > * s2s.feature > * s3.feature > * splunk.feature > * sql.feature > * syslog_listener.feature -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Updated] (MINIFICPP-1641) Parallel running of docker tests
[ https://issues.apache.org/jira/browse/MINIFICPP-1641?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Martin Zink updated MINIFICPP-1641: --- Description: Currently docker tests (docker-verify target) is run sequentially. This is not scalable. Consider exploring the parallelization options of behave, multiprocess and multithreading under Python. Also pay attention to available CPU cores on GitHub CI and if it is possible to increase them. The main problem is that containers are referenced by name only. We should postfix them based on the scenario and iron out any problems that arise. was: Currently docker tests (docker-verify target) is run sequentially. This is not scalable. Consider exploring the parallelization options of behave, multiprocess and multithreading under Python. Also pay attention to available CPU cores on GitHub CI and if it is possible to increase them. The main problem is that containers are referenced by name only. We should postfix them based on the scenario and iron out any problems that arise. * attributes_to_json.feature * azure_storage.feature * core_functionality.feature * defragtextflowfiles.feature * elasticsearch.feature * file_system_operations.feature * google_cloud_storage.feature * hashcontent.feature * http.feature * https.feature * kafka.feature * kubernetes_logging.feature * kubernetes_metrics.feature * lua_script.feature * minifi_c2_server.feature * mqtt.feature * network_listener.feature * opcua.feature * opensearch.feature * prometheus.feature * python.feature * python_script.feature * python_with_modules.feature * replace_text.feature * routetext.feature * s2s.feature * s3.feature * splunk.feature * sql.feature * syslog_listener.feature > Parallel running of docker tests > > > Key: MINIFICPP-1641 > URL: https://issues.apache.org/jira/browse/MINIFICPP-1641 > Project: Apache NiFi MiNiFi C++ > Issue Type: Test >Reporter: Ádám Markovics >Assignee: Martin Zink >Priority: Minor > > Currently docker tests (docker-verify target) is run sequentially. This is > not scalable. Consider exploring the parallelization options of behave, > multiprocess and multithreading under Python. Also pay attention to available > CPU cores on GitHub CI and if it is possible to increase them. > The main problem is that containers are referenced by name only. We should > postfix them based on the scenario and iron out any problems that arise. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Created] (MINIFICPP-2107) make package creates faulty tar.gz on macOS
Martin Zink created MINIFICPP-2107: -- Summary: make package creates faulty tar.gz on macOS Key: MINIFICPP-2107 URL: https://issues.apache.org/jira/browse/MINIFICPP-2107 Project: Apache NiFi MiNiFi C++ Issue Type: Bug Reporter: Martin Zink Assignee: Martin Zink INSTALL_RPATH is set to $ORIGIN, which doesnt seemed to work on macOS (we should use @loader_path) -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Created] (NIFI-11473) Flow version change in NiFi should not stop a component when only position is changed
Timea Barna created NIFI-11473: -- Summary: Flow version change in NiFi should not stop a component when only position is changed Key: NIFI-11473 URL: https://issues.apache.org/jira/browse/NIFI-11473 Project: Apache NiFi Issue Type: Improvement Reporter: Timea Barna When going from one flow version to another when the position of a component is changing, but not its configuration, the component should not be stopped. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Assigned] (NIFI-11473) Flow version change in NiFi should not stop a component when only position is changed
[ https://issues.apache.org/jira/browse/NIFI-11473?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Timea Barna reassigned NIFI-11473: -- Assignee: Timea Barna > Flow version change in NiFi should not stop a component when only position is > changed > - > > Key: NIFI-11473 > URL: https://issues.apache.org/jira/browse/NIFI-11473 > Project: Apache NiFi > Issue Type: Improvement >Reporter: Timea Barna >Assignee: Timea Barna >Priority: Major > > When going from one flow version to another when the position of a component > is changing, but not its configuration, the component should not be stopped. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[GitHub] [nifi-minifi-cpp] martinzink commented on a diff in pull request #1558: MINIFICPP-2100 Add RockyLinux Dockerfile
martinzink commented on code in PR #1558:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1558#discussion_r1172455456
##
docker/rockylinux/Dockerfile:
##
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM rockylinux:8
+LABEL maintainer="Apache NiFi "
+
+ARG MINIFI_VERSION
+ARG UID=1000
+ARG GID=1000
+
+# MINIFI_OPTIONS will be passed directly to cmake
+# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON)
+ARG MINIFI_OPTIONS=""
+ARG CMAKE_BUILD_TYPE=Release
+ARG DOCKER_SKIP_TESTS=ON
+ARG DOCKER_MAKE_TARGET="all"
+
+ENV MINIFI_BASE_DIR /opt/minifi
+ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION
+ENV USER minificpp
+
+RUN mkdir -p $MINIFI_BASE_DIR
+COPY . ${MINIFI_BASE_DIR}
+
+# Install the system dependencies needed for a build
+# gpsd-devel and ccache are in EPEL
+RUN dnf -y install epel-release && dnf -y install sudo git which make
libarchive ccache ca-certificates cmake && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_GPS=ON"; then dnf -y install
gpsd-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_JNI=ON"; then dnf -y install
java-1.8.0-openjdk maven; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then dnf -y install
libpcap-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_USB_CAMERA=ON"; then dnf -y
install libpng-devel libusbx-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_OPENCV=ON" || echo
"$MINIFI_OPTIONS" | grep -q "ENABLE_BUSTACHE=ON"; then dnf -y install
boost-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_LUA_SCRIPTING=ON"; then dnf
--enablerepo=powertools install lua-devel; fi && \
Review Comment:
sry I might have left out the noprompt flag from here
```suggestion
if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_LUA_SCRIPTING=ON"; then dnf
-y --enablerepo=powertools install lua-devel; fi && \
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] martinzink commented on a diff in pull request #1558: MINIFICPP-2100 Add RockyLinux Dockerfile
martinzink commented on code in PR #1558:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1558#discussion_r1172455456
##
docker/rockylinux/Dockerfile:
##
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM rockylinux:8
+LABEL maintainer="Apache NiFi "
+
+ARG MINIFI_VERSION
+ARG UID=1000
+ARG GID=1000
+
+# MINIFI_OPTIONS will be passed directly to cmake
+# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON)
+ARG MINIFI_OPTIONS=""
+ARG CMAKE_BUILD_TYPE=Release
+ARG DOCKER_SKIP_TESTS=ON
+ARG DOCKER_MAKE_TARGET="all"
+
+ENV MINIFI_BASE_DIR /opt/minifi
+ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION
+ENV USER minificpp
+
+RUN mkdir -p $MINIFI_BASE_DIR
+COPY . ${MINIFI_BASE_DIR}
+
+# Install the system dependencies needed for a build
+# gpsd-devel and ccache are in EPEL
+RUN dnf -y install epel-release && dnf -y install sudo git which make
libarchive ccache ca-certificates cmake && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_GPS=ON"; then dnf -y install
gpsd-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_JNI=ON"; then dnf -y install
java-1.8.0-openjdk maven; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then dnf -y install
libpcap-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_USB_CAMERA=ON"; then dnf -y
install libpng-devel libusbx-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_OPENCV=ON" || echo
"$MINIFI_OPTIONS" | grep -q "ENABLE_BUSTACHE=ON"; then dnf -y install
boost-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_LUA_SCRIPTING=ON"; then dnf
--enablerepo=powertools install lua-devel; fi && \
Review Comment:
sry I might have left out the noprompt flag from my earlier suggestion
```suggestion
if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_LUA_SCRIPTING=ON"; then dnf
-y --enablerepo=powertools install lua-devel; fi && \
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[jira] [Updated] (NIFI-11473) Flow version change in NiFi should not stop a component when only position is changed
[ https://issues.apache.org/jira/browse/NIFI-11473?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Timea Barna updated NIFI-11473: --- Description: When going from one flow version to another and the position of a component is changing, but not its configuration, the component should not be stopped. (was: When going from one flow version to another when the position of a component is changing, but not its configuration, the component should not be stopped.) > Flow version change in NiFi should not stop a component when only position is > changed > - > > Key: NIFI-11473 > URL: https://issues.apache.org/jira/browse/NIFI-11473 > Project: Apache NiFi > Issue Type: Improvement >Reporter: Timea Barna >Assignee: Timea Barna >Priority: Major > > When going from one flow version to another and the position of a component > is changing, but not its configuration, the component should not be stopped. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[GitHub] [nifi] timeabarna opened a new pull request, #7185: NIFI-11473 Flow version change in NiFi should not stop a component wh…
timeabarna opened a new pull request, #7185: URL: https://github.com/apache/nifi/pull/7185 …en only position is changed # Summary When going from one flow version to another and the position of a component is changing, but not its configuration, the component should not be stopped. [NIFI-11473](https://issues.apache.org/jira/browse/NIFI-11473) # Tracking Please complete the following tracking steps prior to pull request creation. ### Issue Tracking - [ ] [Apache NiFi Jira](https://issues.apache.org/jira/browse/NIFI) issue created ### Pull Request Tracking - [ ] Pull Request title starts with Apache NiFi Jira issue number, such as `NIFI-0` - [ ] Pull Request commit message starts with Apache NiFi Jira issue number, as such `NIFI-0` ### Pull Request Formatting - [ ] Pull Request based on current revision of the `main` branch - [ ] Pull Request refers to a feature branch with one commit containing changes # Verification Please indicate the verification steps performed prior to pull request creation. ### Build - [ ] Build completed using `mvn clean install -P contrib-check` - [ ] JDK 11 - [ ] JDK 17 ### Licensing - [ ] New dependencies are compatible with the [Apache License 2.0](https://apache.org/licenses/LICENSE-2.0) according to the [License Policy](https://www.apache.org/legal/resolved.html) - [ ] New dependencies are documented in applicable `LICENSE` and `NOTICE` files ### Documentation - [ ] Documentation formatting appears as expected in rendered files -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[jira] [Updated] (NIFI-11473) Flow version change in NiFi should not stop a component when only position is changed
[ https://issues.apache.org/jira/browse/NIFI-11473?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Timea Barna updated NIFI-11473: --- Status: Patch Available (was: In Progress) > Flow version change in NiFi should not stop a component when only position is > changed > - > > Key: NIFI-11473 > URL: https://issues.apache.org/jira/browse/NIFI-11473 > Project: Apache NiFi > Issue Type: Improvement >Reporter: Timea Barna >Assignee: Timea Barna >Priority: Major > Time Spent: 10m > Remaining Estimate: 0h > > When going from one flow version to another and the position of a component > is changing, but not its configuration, the component should not be stopped. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[GitHub] [nifi] arpadboda commented on pull request #7184: NIFI-11472 Make PutFTP processor more multithread friendly
arpadboda commented on PR #7184: URL: https://github.com/apache/nifi/pull/7184#issuecomment-1516193929 Hello @MormonJesus69420 , thanks for your contribution! I'm not sure if using multiple threads to transfer to the same ftp makes any sense, this operation is limited by bandwith anyway. So I would prefer to restrict it to single threaded usage, but the current implementation allows uploading to multiple hosts based on flowfile attributes, in which case multiple threads might make sense, so I'm not against this change. Weak +1, I'm ok to merge with a 2nd approval. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi] MormonJesus69420 commented on pull request #7184: NIFI-11472 Make PutFTP processor more multithread friendly
MormonJesus69420 commented on PR #7184: URL: https://github.com/apache/nifi/pull/7184#issuecomment-1516223097 Hi @arpadboda I might not have described it so well, but the issue we face is when we configure processor to run several concurrent tasks.  When we change the number to two or more we start receiving "errors" about processor being unable to create directory as it already exists. While the issue resolves itself, it is rather distracting to see it, when it's not a "real" error. We have noticed a significant performance boost when using a PutFTP processor with more than one concurrent task. I don't have the numbers on me at the moment, but I think that switching to two or three concurrent tasks significantly sped up the transfer time. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] lordgamez commented on a diff in pull request #1558: MINIFICPP-2100 Add RockyLinux Dockerfile
lordgamez commented on code in PR #1558:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1558#discussion_r1172506942
##
docker/rockylinux/Dockerfile:
##
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM rockylinux:8
+LABEL maintainer="Apache NiFi "
+
+ARG MINIFI_VERSION
+ARG UID=1000
+ARG GID=1000
+
+# MINIFI_OPTIONS will be passed directly to cmake
+# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON)
+ARG MINIFI_OPTIONS=""
+ARG CMAKE_BUILD_TYPE=Release
+ARG DOCKER_SKIP_TESTS=ON
+ARG DOCKER_MAKE_TARGET="all"
+
+ENV MINIFI_BASE_DIR /opt/minifi
+ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION
+ENV USER minificpp
+
+RUN mkdir -p $MINIFI_BASE_DIR
+COPY . ${MINIFI_BASE_DIR}
+
+# Install the system dependencies needed for a build
+# gpsd-devel and ccache are in EPEL
+RUN dnf -y install epel-release && dnf -y install sudo git which make
libarchive ccache ca-certificates cmake && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_GPS=ON"; then dnf -y install
gpsd-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_JNI=ON"; then dnf -y install
java-1.8.0-openjdk maven; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then dnf -y install
libpcap-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_USB_CAMERA=ON"; then dnf -y
install libpng-devel libusbx-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_OPENCV=ON" || echo
"$MINIFI_OPTIONS" | grep -q "ENABLE_BUSTACHE=ON"; then dnf -y install
boost-devel; fi && \
+if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_LUA_SCRIPTING=ON"; then dnf
--enablerepo=powertools install lua-devel; fi && \
Review Comment:
Good catch, updated in 1ca67cbec06fca4ed5fe229740cf0f9d3c7adb11
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi] MormonJesus69420 commented on pull request #7184: NIFI-11472 Make PutFTP processor more multithread friendly
MormonJesus69420 commented on PR #7184: URL: https://github.com/apache/nifi/pull/7184#issuecomment-1516261028 I am sorry, I managed to make the most basic mistake in such a small change, I forgot to add a `!` to the `setWorkingDirectory(remoteDirectory)` method call. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[jira] [Updated] (NIFI-11472) Make PutFTP processor more multithread friendly
[
https://issues.apache.org/jira/browse/NIFI-11472?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Daniel Salwerowicz updated NIFI-11472:
--
Description:
Problem happens when a PutFTP is set to run several concurrent tasks and two
(or more ) FlowFiles come in and both need to create the same directory. One of
them will create directory and succeed immediately while the other will try to
create directory, but fail since it already exist, throw an error, the FlowFile
will then be penalized and on second run will succeed.
While it is not the biggest error, as files are getting transferred in the end,
but the bulletins and errors are annoying, especially in production environment
where you don't want to get unnecessary errors.
We found that the solution involves a simple change to the {{FTPTransfer.java}}
class in:
{{nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java}}
On line 398 and {{ensureDirectoryExists}} method you can simply add another if
check which double checks that the directory exists when it fails to create one.
{code:java}
final boolean cdSuccessful = setWorkingDirectory(remoteDirectory);
if (!cdSuccessful) {
if (client.makeDirectory(remoteDirectory)) {
logger.debug("Remote Directory not found: created directory [{}]",
remoteDirectory);
} else if (!setWorkingDirectory(remoteDirectory)) {
// Double check that the dir exists as it might have been created in
another thread
throw new IOException("Failed to create remote directory " +
remoteDirectory);
}
}{code}
We have created a fix and will submit it as a Pull Request on GitHub
was:
Problem happens when a PutFTP is set to run several concurrent tasks and two
(or more ) FlowFiles come in and both need to create the same directory. One of
them will create directory and succeed immediately while the other will try to
create directory, but fail since it already exist, throw an error, the FlowFile
will then be penalized and on second run will succeed.
While it is not the biggest error, as files are getting transferred in the end,
but the bulletins and errors are annoying, especially in production environment
where you don't want to get unnecessary errors.
We found that the solution involves a simple change to the {{FTPTransfer.java}}
class in:
{{nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java}}
On line 398 and {{ensureDirectoryExists}} method you can simply add another if
check which double checks that the directory exists when it fails to create one.
{code:java}
final boolean cdSuccessful = setWorkingDirectory(remoteDirectory);
if (!cdSuccessful) {
if (client.makeDirectory(remoteDirectory)) {
logger.debug("Remote Directory not found: created directory [{}]",
remoteDirectory);
} else if (setWorkingDirectory(remoteDirectory)) {
// Double check that the dir exists as it might have been created in
another thread
throw new IOException("Failed to create remote directory " +
remoteDirectory);
}
}{code}
We have created a fix and will submit it as a Pull Request on GitHub
> Make PutFTP processor more multithread friendly
> ---
>
> Key: NIFI-11472
> URL: https://issues.apache.org/jira/browse/NIFI-11472
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 1.21.0
> Environment: CentOS server, Java 11
>Reporter: Daniel Salwerowicz
>Priority: Major
> Time Spent: 40m
> Remaining Estimate: 0h
>
> Problem happens when a PutFTP is set to run several concurrent tasks and two
> (or more ) FlowFiles come in and both need to create the same directory. One
> of them will create directory and succeed immediately while the other will
> try to create directory, but fail since it already exist, throw an error, the
> FlowFile will then be penalized and on second run will succeed.
> While it is not the biggest error, as files are getting transferred in the
> end, but the bulletins and errors are annoying, especially in production
> environment where you don't want to get unnecessary errors.
> We found that the solution involves a simple change to the
> {{FTPTransfer.java}} class in:
> {{nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java}}
> On line 398 and {{ensureDirectoryExists}} method you can simply add another
> if check which double checks that the directory exists when it fails to
> create one.
> {code:java}
> final boolean cdSuccessful = setWorkingDirectory(remoteDirectory);
> if (!cdSuccessful) {
> if (client.makeDirec
[jira] [Updated] (MINIFICPP-1641) Parallel running of docker tests
[ https://issues.apache.org/jira/browse/MINIFICPP-1641?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Martin Zink updated MINIFICPP-1641: --- Issue Type: Improvement (was: Test) > Parallel running of docker tests > > > Key: MINIFICPP-1641 > URL: https://issues.apache.org/jira/browse/MINIFICPP-1641 > Project: Apache NiFi MiNiFi C++ > Issue Type: Improvement >Reporter: Ádám Markovics >Assignee: Martin Zink >Priority: Minor > > Currently docker tests (docker-verify target) is run sequentially. This is > not scalable. Consider exploring the parallelization options of behave, > multiprocess and multithreading under Python. Also pay attention to available > CPU cores on GitHub CI and if it is possible to increase them. > The main problem is that containers are referenced by name only. We should > postfix them based on the scenario and iron out any problems that arise. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Updated] (MINIFICPP-2107) make package creates faulty tar.gz on macOS
[ https://issues.apache.org/jira/browse/MINIFICPP-2107?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Martin Zink updated MINIFICPP-2107: --- Priority: Minor (was: Major) > make package creates faulty tar.gz on macOS > --- > > Key: MINIFICPP-2107 > URL: https://issues.apache.org/jira/browse/MINIFICPP-2107 > Project: Apache NiFi MiNiFi C++ > Issue Type: Bug >Reporter: Martin Zink >Assignee: Martin Zink >Priority: Minor > > INSTALL_RPATH is set to $ORIGIN, which doesnt seemed to work on macOS (we > should use @loader_path) -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Updated] (MINIFICPP-1641) Parallel running of docker tests
[ https://issues.apache.org/jira/browse/MINIFICPP-1641?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Martin Zink updated MINIFICPP-1641: --- Priority: Major (was: Minor) > Parallel running of docker tests > > > Key: MINIFICPP-1641 > URL: https://issues.apache.org/jira/browse/MINIFICPP-1641 > Project: Apache NiFi MiNiFi C++ > Issue Type: Improvement >Reporter: Ádám Markovics >Assignee: Martin Zink >Priority: Major > > Currently docker tests (docker-verify target) is run sequentially. This is > not scalable. Consider exploring the parallelization options of behave, > multiprocess and multithreading under Python. Also pay attention to available > CPU cores on GitHub CI and if it is possible to increase them. > The main problem is that containers are referenced by name only. We should > postfix them based on the scenario and iron out any problems that arise. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Created] (NIFI-11474) Add missing endpoint mergers.
Tamas Palfy created NIFI-11474: -- Summary: Add missing endpoint mergers. Key: NIFI-11474 URL: https://issues.apache.org/jira/browse/NIFI-11474 Project: Apache NiFi Issue Type: Bug Reporter: Tamas Palfy The following endpoints and entities don't have corresponding endpoint mergers: * ConfigurationAnalysisEntity * DELETE component endpoints (e.g. in ReportingTaskResource) * GET PropertyDescriptorEntity endpoints (e.g. in ReportingTaskResource) -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Created] (MINIFICPP-2108) Update version number to 0.15.0
Gábor Gyimesi created MINIFICPP-2108: Summary: Update version number to 0.15.0 Key: MINIFICPP-2108 URL: https://issues.apache.org/jira/browse/MINIFICPP-2108 Project: Apache NiFi MiNiFi C++ Issue Type: Task Reporter: Gábor Gyimesi Assignee: Gábor Gyimesi Fix For: 0.15.0 -- This message was sent by Atlassian Jira (v8.20.10#820010)
[GitHub] [nifi-minifi-cpp] lordgamez opened a new pull request, #1563: MINIFICPP-2108 Update version number to 0.15.0
lordgamez opened a new pull request, #1563: URL: https://github.com/apache/nifi-minifi-cpp/pull/1563 https://issues.apache.org/jira/browse/MINIFICPP-2108 --- Thank you for submitting a contribution to Apache NiFi - MiNiFi C++. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### For all changes: - [ ] Is there a JIRA ticket associated with this PR? Is it referenced in the commit message? - [ ] Does your PR title start with MINIFICPP- where is the JIRA number you are trying to resolve? Pay particular attention to the hyphen "-" character. - [ ] Has your PR been rebased against the latest commit within the target branch (typically main)? - [ ] Is your initial contribution a single, squashed commit? ### For code changes: - [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)? - [ ] If applicable, have you updated the LICENSE file? - [ ] If applicable, have you updated the NOTICE file? ### For documentation related changes: - [ ] Have you ensured that format looks appropriate for the output in which it is rendered? ### Note: Please ensure that once the PR is submitted, you check GitHub Actions CI results for build issues and submit an update to your PR as soon as possible. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] fgerlits commented on a diff in pull request #1552: MINIFICPP-2089 prefix EventData in flat JSON output so it doesnt need t…
fgerlits commented on code in PR #1552:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1552#discussion_r1172666551
##
extensions/windows-event-log/wel/JSONUtils.cpp:
##
@@ -135,28 +158,33 @@ rapidjson::Document toJSONImpl(const pugi::xml_node&
root, bool flatten) {
{
auto eventData_xml = event_xml.child("EventData");
-// create EventData subarray even if flatten requested
-doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator());
-for (const auto& data : eventData_xml.children()) {
- auto name_attr = data.attribute("Name");
- rapidjson::Value item(rapidjson::kObjectType);
- item.AddMember("Name", rapidjson::StringRef(name_attr.value()),
doc.GetAllocator());
- item.AddMember("Content", rapidjson::StringRef(data.text().get()),
doc.GetAllocator());
- item.AddMember("Type", rapidjson::StringRef(data.name()),
doc.GetAllocator());
- // we need to query EventData because a reference to it wouldn't be
stable, as we
- // possibly add members to its parent which could result in reallocation
- doc["EventData"].PushBack(item, doc.GetAllocator());
- // check collision
- if (flatten && !name_attr.empty() && !doc.HasMember(name_attr.value())) {
-doc.AddMember(rapidjson::StringRef(name_attr.value()),
rapidjson::StringRef(data.text().get()), doc.GetAllocator());
+if (flatten) {
+ for (const auto& event_data_child : eventData_xml.children()) {
+std::string key = "EventData";
+if (auto name_attr = event_data_child.attribute("Name").value();
strlen(name_attr)) {
+ key = utils::StringUtils::join_pack(key, ".", name_attr);
Review Comment:
would
```suggestion
if (auto name_attr = event_data_child.attribute("Name");
!name_attr.empty()) {
key = utils::StringUtils::join_pack(key, ".", name_attr.value());
```
work? if it does, I think that would be slightly nicer
##
extensions/windows-event-log/wel/JSONUtils.cpp:
##
@@ -135,28 +158,33 @@ rapidjson::Document toJSONImpl(const pugi::xml_node&
root, bool flatten) {
{
auto eventData_xml = event_xml.child("EventData");
-// create EventData subarray even if flatten requested
-doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator());
-for (const auto& data : eventData_xml.children()) {
- auto name_attr = data.attribute("Name");
- rapidjson::Value item(rapidjson::kObjectType);
- item.AddMember("Name", rapidjson::StringRef(name_attr.value()),
doc.GetAllocator());
- item.AddMember("Content", rapidjson::StringRef(data.text().get()),
doc.GetAllocator());
- item.AddMember("Type", rapidjson::StringRef(data.name()),
doc.GetAllocator());
- // we need to query EventData because a reference to it wouldn't be
stable, as we
- // possibly add members to its parent which could result in reallocation
- doc["EventData"].PushBack(item, doc.GetAllocator());
- // check collision
- if (flatten && !name_attr.empty() && !doc.HasMember(name_attr.value())) {
-doc.AddMember(rapidjson::StringRef(name_attr.value()),
rapidjson::StringRef(data.text().get()), doc.GetAllocator());
+if (flatten) {
+ for (const auto& event_data_child : eventData_xml.children()) {
+std::string key = "EventData";
+if (auto name_attr = event_data_child.attribute("Name").value();
strlen(name_attr)) {
+ key = utils::StringUtils::join_pack(key, ".", name_attr);
+}
+
+doc.AddMember(rapidjson::Value(createUniqueKey(key, doc),
doc.GetAllocator()).Move(),
rapidjson::StringRef(event_data_child.text().get()), doc.GetAllocator());
+ }
+} else {
+ auto& event_data = doc.AddMember("EventData", rapidjson::kArrayType,
doc.GetAllocator());
+ for (const auto& event_data_child : eventData_xml.children()) {
+auto name_attr = event_data_child.attribute("Name");
+rapidjson::Value item(rapidjson::kObjectType);
+item.AddMember("Name", rapidjson::StringRef(name_attr.value()),
doc.GetAllocator());
+item.AddMember("Content",
rapidjson::StringRef(event_data_child.text().get()), doc.GetAllocator());
+item.AddMember("Type", rapidjson::StringRef(event_data_child.name()),
doc.GetAllocator());
+doc["EventData"].PushBack(item, doc.GetAllocator());
Review Comment:
should this be
```suggestion
event_data.PushBack(item, doc.GetAllocator());
```
?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] martinzink commented on a diff in pull request #1552: MINIFICPP-2089 prefix EventData in flat JSON output so it doesnt need t…
martinzink commented on code in PR #1552:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1552#discussion_r1172785348
##
extensions/windows-event-log/wel/JSONUtils.cpp:
##
@@ -135,28 +158,33 @@ rapidjson::Document toJSONImpl(const pugi::xml_node&
root, bool flatten) {
{
auto eventData_xml = event_xml.child("EventData");
-// create EventData subarray even if flatten requested
-doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator());
-for (const auto& data : eventData_xml.children()) {
- auto name_attr = data.attribute("Name");
- rapidjson::Value item(rapidjson::kObjectType);
- item.AddMember("Name", rapidjson::StringRef(name_attr.value()),
doc.GetAllocator());
- item.AddMember("Content", rapidjson::StringRef(data.text().get()),
doc.GetAllocator());
- item.AddMember("Type", rapidjson::StringRef(data.name()),
doc.GetAllocator());
- // we need to query EventData because a reference to it wouldn't be
stable, as we
- // possibly add members to its parent which could result in reallocation
- doc["EventData"].PushBack(item, doc.GetAllocator());
- // check collision
- if (flatten && !name_attr.empty() && !doc.HasMember(name_attr.value())) {
-doc.AddMember(rapidjson::StringRef(name_attr.value()),
rapidjson::StringRef(data.text().get()), doc.GetAllocator());
+if (flatten) {
+ for (const auto& event_data_child : eventData_xml.children()) {
+std::string key = "EventData";
+if (auto name_attr = event_data_child.attribute("Name").value();
strlen(name_attr)) {
+ key = utils::StringUtils::join_pack(key, ".", name_attr);
+}
+
+doc.AddMember(rapidjson::Value(createUniqueKey(key, doc),
doc.GetAllocator()).Move(),
rapidjson::StringRef(event_data_child.text().get()), doc.GetAllocator());
+ }
+} else {
+ auto& event_data = doc.AddMember("EventData", rapidjson::kArrayType,
doc.GetAllocator());
+ for (const auto& event_data_child : eventData_xml.children()) {
+auto name_attr = event_data_child.attribute("Name");
+rapidjson::Value item(rapidjson::kObjectType);
+item.AddMember("Name", rapidjson::StringRef(name_attr.value()),
doc.GetAllocator());
+item.AddMember("Content",
rapidjson::StringRef(event_data_child.text().get()), doc.GetAllocator());
+item.AddMember("Type", rapidjson::StringRef(event_data_child.name()),
doc.GetAllocator());
+doc["EventData"].PushBack(item, doc.GetAllocator());
Review Comment:
For some reason the event_data reference is unstable (rapidjson probably
reallocates when we add children to it) I've removed the event_data variable so
it doesnt tempt us, and readded the comment that explains why in
https://github.com/apache/nifi-minifi-cpp/pull/1552/commits/1aa0603b2118b0237fe0c90f085ca608c78bcbe6
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] martinzink commented on a diff in pull request #1552: MINIFICPP-2089 prefix EventData in flat JSON output so it doesnt need t…
martinzink commented on code in PR #1552:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1552#discussion_r1172786389
##
extensions/windows-event-log/wel/JSONUtils.cpp:
##
@@ -135,28 +158,33 @@ rapidjson::Document toJSONImpl(const pugi::xml_node&
root, bool flatten) {
{
auto eventData_xml = event_xml.child("EventData");
-// create EventData subarray even if flatten requested
-doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator());
-for (const auto& data : eventData_xml.children()) {
- auto name_attr = data.attribute("Name");
- rapidjson::Value item(rapidjson::kObjectType);
- item.AddMember("Name", rapidjson::StringRef(name_attr.value()),
doc.GetAllocator());
- item.AddMember("Content", rapidjson::StringRef(data.text().get()),
doc.GetAllocator());
- item.AddMember("Type", rapidjson::StringRef(data.name()),
doc.GetAllocator());
- // we need to query EventData because a reference to it wouldn't be
stable, as we
- // possibly add members to its parent which could result in reallocation
- doc["EventData"].PushBack(item, doc.GetAllocator());
- // check collision
- if (flatten && !name_attr.empty() && !doc.HasMember(name_attr.value())) {
-doc.AddMember(rapidjson::StringRef(name_attr.value()),
rapidjson::StringRef(data.text().get()), doc.GetAllocator());
+if (flatten) {
+ for (const auto& event_data_child : eventData_xml.children()) {
+std::string key = "EventData";
+if (auto name_attr = event_data_child.attribute("Name").value();
strlen(name_attr)) {
+ key = utils::StringUtils::join_pack(key, ".", name_attr);
Review Comment:
good idea, I've included this 👍
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] szaszm closed pull request #1552: MINIFICPP-2089 prefix EventData in flat JSON output so it doesnt need t…
szaszm closed pull request #1552: MINIFICPP-2089 prefix EventData in flat JSON output so it doesnt need t… URL: https://github.com/apache/nifi-minifi-cpp/pull/1552 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] szaszm closed pull request #1562: MINIFICPP-2106 Add 'Use Path Style Access' property to PutS3Object
szaszm closed pull request #1562: MINIFICPP-2106 Add 'Use Path Style Access' property to PutS3Object URL: https://github.com/apache/nifi-minifi-cpp/pull/1562 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi-minifi-cpp] szaszm closed pull request #1539: MINIFICPP-2037 Build SSLContext from minifi.properties for SSL commuincation on C2
szaszm closed pull request #1539: MINIFICPP-2037 Build SSLContext from minifi.properties for SSL commuincation on C2 URL: https://github.com/apache/nifi-minifi-cpp/pull/1539 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[jira] [Resolved] (MINIFICPP-2037) Build SSLContext from minifi.properties for SSL communication on C2
[ https://issues.apache.org/jira/browse/MINIFICPP-2037?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marton Szasz resolved MINIFICPP-2037. - Resolution: Done > Build SSLContext from minifi.properties for SSL communication on C2 > --- > > Key: MINIFICPP-2037 > URL: https://issues.apache.org/jira/browse/MINIFICPP-2037 > Project: Apache NiFi MiNiFi C++ > Issue Type: Improvement >Reporter: Gábor Gyimesi >Assignee: Gábor Gyimesi >Priority: Major > Time Spent: 20m > Remaining Estimate: 0h > > If we want to use SSL for C2 communication we have to define an > SSLContextService in the flow configuration with the certificate information > used for the SSL connection. If the flow fetched from the C2 server and the > agent is started with an empty flow configuration this is not possible. We > should be able to retrieve the needed SSL information from the > minifi.properties file as well if we are not able to provide it in the flow > configuration. There is a similar use case for this in the > RemoteProcessorGroupPort where the > SSLContextService is built manually from the information provided in the > minifi.properties file. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Resolved] (MINIFICPP-2106) Add 'Use Path Style Access' property to PutS3Object
[ https://issues.apache.org/jira/browse/MINIFICPP-2106?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marton Szasz resolved MINIFICPP-2106. - Resolution: Done > Add 'Use Path Style Access' property to PutS3Object > --- > > Key: MINIFICPP-2106 > URL: https://issues.apache.org/jira/browse/MINIFICPP-2106 > Project: Apache NiFi MiNiFi C++ > Issue Type: New Feature >Reporter: Gábor Gyimesi >Assignee: Gábor Gyimesi >Priority: Major > Time Spent: 1h > Remaining Estimate: 0h > > Add a similar property to PutS3Object processor as NiFi's ' Use Path Style > Access' property described as: > "Path-style access can be enforced by setting this property to true. Set it > to true if your endpoint does not support virtual-hosted-style requests, only > path-style requests." -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Updated] (MINIFICPP-2089) Revisit CWEL EventData format in flattened JSON output
[ https://issues.apache.org/jira/browse/MINIFICPP-2089?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marton Szasz updated MINIFICPP-2089: Resolution: Done Status: Resolved (was: Patch Available) > Revisit CWEL EventData format in flattened JSON output > -- > > Key: MINIFICPP-2089 > URL: https://issues.apache.org/jira/browse/MINIFICPP-2089 > Project: Apache NiFi MiNiFi C++ > Issue Type: Improvement >Reporter: Marton Szasz >Assignee: Martin Zink >Priority: Major > Time Spent: 1h 50m > Remaining Estimate: 0h > > In flattened JSON, the data in an EventData tag is duplicated: Once in the > normal flattened output in the main object, and once in a list of subobjects > in a slightly different format. The reason for this was to avoid losing data, > even if the inner keys of EventData are clashing with keys of the outer > System object, which happens in practice, but there may be a better way to > ensure this. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Updated] (MINIFICPP-2106) Add 'Use Path Style Access' property to PutS3Object
[ https://issues.apache.org/jira/browse/MINIFICPP-2106?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Gábor Gyimesi updated MINIFICPP-2106: - Fix Version/s: 0.15.0 > Add 'Use Path Style Access' property to PutS3Object > --- > > Key: MINIFICPP-2106 > URL: https://issues.apache.org/jira/browse/MINIFICPP-2106 > Project: Apache NiFi MiNiFi C++ > Issue Type: New Feature >Reporter: Gábor Gyimesi >Assignee: Gábor Gyimesi >Priority: Major > Fix For: 0.15.0 > > Time Spent: 1h > Remaining Estimate: 0h > > Add a similar property to PutS3Object processor as NiFi's ' Use Path Style > Access' property described as: > "Path-style access can be enforced by setting this property to true. Set it > to true if your endpoint does not support virtual-hosted-style requests, only > path-style requests." -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Updated] (MINIFICPP-2037) Build SSLContext from minifi.properties for SSL communication on C2
[ https://issues.apache.org/jira/browse/MINIFICPP-2037?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Gábor Gyimesi updated MINIFICPP-2037: - Fix Version/s: 0.15.0 > Build SSLContext from minifi.properties for SSL communication on C2 > --- > > Key: MINIFICPP-2037 > URL: https://issues.apache.org/jira/browse/MINIFICPP-2037 > Project: Apache NiFi MiNiFi C++ > Issue Type: Improvement >Reporter: Gábor Gyimesi >Assignee: Gábor Gyimesi >Priority: Major > Fix For: 0.15.0 > > Time Spent: 20m > Remaining Estimate: 0h > > If we want to use SSL for C2 communication we have to define an > SSLContextService in the flow configuration with the certificate information > used for the SSL connection. If the flow fetched from the C2 server and the > agent is started with an empty flow configuration this is not possible. We > should be able to retrieve the needed SSL information from the > minifi.properties file as well if we are not able to provide it in the flow > configuration. There is a similar use case for this in the > RemoteProcessorGroupPort where the > SSLContextService is built manually from the information provided in the > minifi.properties file. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Updated] (MINIFICPP-2089) Revisit CWEL EventData format in flattened JSON output
[ https://issues.apache.org/jira/browse/MINIFICPP-2089?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marton Szasz updated MINIFICPP-2089: Fix Version/s: 0.15.0 > Revisit CWEL EventData format in flattened JSON output > -- > > Key: MINIFICPP-2089 > URL: https://issues.apache.org/jira/browse/MINIFICPP-2089 > Project: Apache NiFi MiNiFi C++ > Issue Type: Improvement >Reporter: Marton Szasz >Assignee: Martin Zink >Priority: Major > Fix For: 0.15.0 > > Time Spent: 1h 50m > Remaining Estimate: 0h > > In flattened JSON, the data in an EventData tag is duplicated: Once in the > normal flattened output in the main object, and once in a list of subobjects > in a slightly different format. The reason for this was to avoid losing data, > even if the inner keys of EventData are clashing with keys of the outer > System object, which happens in practice, but there may be a better way to > ensure this. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Commented] (NIFI-4385) Adjust the QueryDatabaseTable processor for handling big tables.
[ https://issues.apache.org/jira/browse/NIFI-4385?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17714726#comment-17714726 ] Rohit Pawar commented on NIFI-4385: --- [~mattyb149] [~pwicks] Hello, I have question regarding QueryDatabaseTable ,Actually In mysql I am trying to use server side cursor streaming by enabling useCursorFetch=true in connection url , For Implementing batch thing and save nifi from memory-overflow like thinks ,Example for what I am expecting : data-fetched and send parallely to next processor(which write data in DB) so that load of memory not increase . QueryDatabaseTable - > PutDatabaseRecord Problem : Througth QueryDatabaseTable I got all data of query about 50k get in halfsec and placed to Queue and then next processor starts processing it . Why QueryDatabaseTable place all data at once ? because I have set the values for fetchrow=50,batch=5,flowfile=5 Anything I am missing? > Adjust the QueryDatabaseTable processor for handling big tables. > > > Key: NIFI-4385 > URL: https://issues.apache.org/jira/browse/NIFI-4385 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.3.0 >Reporter: Tim Späth >Priority: Major > > When querying large database tables, the *QueryDatabaseTable* processor does > not perform very well. > The processor will always perform the full query and then transfer all > flowfiles as a list instead of > transferring them particularly after the *ResultSet* is fetching the next > rows(If a fetch size is given). > If you want to query a billion rows from a table, > the processor will add all flowfiles in an ArrayList in memory > before transferring the whole list after the last row is fetched by the > ResultSet. > I've checked the code in > *org.apache.nifi.processors.standard.QueryDatabaseTable.java* > and in my opinion, it would be no big deal to move the session.transfer to a > proper position in the code (into the while loop where the flowfile is added > to the list) to > achieve a real _stream support_. There was also a bug report for this problem > which resulted in adding the new property *Maximum Number of Fragments*, > but this property will just limit the results. > Now you have to multiply *Max Rows Per Flow File* with *Maximum Number of > Fragments* to get your limit, > which is not really a solution for the original problem imho. > Also the workaround with GenerateTableFetch and/or ExecuteSQL processors is > much slower than using a database cursor or a ResultSet > and stream the rows in flowfiles directly in the queue. -- This message was sent by Atlassian Jira (v8.20.10#820010)
[jira] [Created] (NIFI-11475) Azure Blob v12 processors are broken after Azure BOM 1.2.11 upgrade
Peter Turcsanyi created NIFI-11475:
--
Summary: Azure Blob v12 processors are broken after Azure BOM
1.2.11 upgrade
Key: NIFI-11475
URL: https://issues.apache.org/jira/browse/NIFI-11475
Project: Apache NiFi
Issue Type: Bug
Reporter: Peter Turcsanyi
Assignee: Peter Turcsanyi
{code:java}
2023-04-20 15:57:08,982 ERROR [reactor-http-nio-1]
c.a.c.i.jackson.XmlMapperFactory Failed to retrieve MethodHandles used to
create XmlMapper. XML serialization won't be supported until
'com.fasterxml.jackson.dataformat:jackson-dataformat-xml' is added to the
classpath or updated to a supported version. Package versions:
jackson-core=2.14.2, jackson-databind=2.14.2, jackson-dataformat-xml=unknown,
jackson-datatype-jsr310=2.14.2, azure-core=1.37.0, Troubleshooting version
conflicts: https://aka.ms/azsdk/java/dependency/troubleshoot
{code}
Azure BOM 1.2.11 upgrade in NIFI-11407 bumped {{azure-core}} to 1.37.0 which
[does not depend on {{jackson-dataformat-xml}}
anymore|https://github.com/Azure/azure-sdk-for-java/issues/7694#issuecomment-1462713526].
It needs to be added explicitly in the service api nar module.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
[GitHub] [nifi] turcsanyip opened a new pull request, #7186: NIFI-11475: Fixed missing jackson-dataformat-xml dependency in nifi-a…
turcsanyip opened a new pull request, #7186: URL: https://github.com/apache/nifi/pull/7186 …zure-services-api module # Summary [NIFI-11475](https://issues.apache.org/jira/browse/NIFI-11475) # Tracking Please complete the following tracking steps prior to pull request creation. ### Issue Tracking - [ ] [Apache NiFi Jira](https://issues.apache.org/jira/browse/NIFI) issue created ### Pull Request Tracking - [ ] Pull Request title starts with Apache NiFi Jira issue number, such as `NIFI-0` - [ ] Pull Request commit message starts with Apache NiFi Jira issue number, as such `NIFI-0` ### Pull Request Formatting - [ ] Pull Request based on current revision of the `main` branch - [ ] Pull Request refers to a feature branch with one commit containing changes # Verification Please indicate the verification steps performed prior to pull request creation. ### Build - [ ] Build completed using `mvn clean install -P contrib-check` - [ ] JDK 11 - [ ] JDK 17 ### Licensing - [ ] New dependencies are compatible with the [Apache License 2.0](https://apache.org/licenses/LICENSE-2.0) according to the [License Policy](https://www.apache.org/legal/resolved.html) - [ ] New dependencies are documented in applicable `LICENSE` and `NOTICE` files ### Documentation - [ ] Documentation formatting appears as expected in rendered files -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[jira] [Updated] (NIFI-11475) Azure Blob v12 processors are broken after Azure BOM 1.2.11 upgrade
[
https://issues.apache.org/jira/browse/NIFI-11475?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Peter Turcsanyi updated NIFI-11475:
---
Status: Patch Available (was: Open)
> Azure Blob v12 processors are broken after Azure BOM 1.2.11 upgrade
> ---
>
> Key: NIFI-11475
> URL: https://issues.apache.org/jira/browse/NIFI-11475
> Project: Apache NiFi
> Issue Type: Bug
>Reporter: Peter Turcsanyi
>Assignee: Peter Turcsanyi
>Priority: Major
> Time Spent: 10m
> Remaining Estimate: 0h
>
> {code:java}
> 2023-04-20 15:57:08,982 ERROR [reactor-http-nio-1]
> c.a.c.i.jackson.XmlMapperFactory Failed to retrieve MethodHandles used to
> create XmlMapper. XML serialization won't be supported until
> 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml' is added to the
> classpath or updated to a supported version. Package versions:
> jackson-core=2.14.2, jackson-databind=2.14.2, jackson-dataformat-xml=unknown,
> jackson-datatype-jsr310=2.14.2, azure-core=1.37.0, Troubleshooting version
> conflicts: https://aka.ms/azsdk/java/dependency/troubleshoot
> {code}
> Azure BOM 1.2.11 upgrade in NIFI-11407 bumped {{azure-core}} to 1.37.0 which
> [does not depend on {{jackson-dataformat-xml}}
> anymore|https://github.com/Azure/azure-sdk-for-java/issues/7694#issuecomment-1462713526].
> It needs to be added explicitly in the service api nar module.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
[GitHub] [nifi] MikeThomsen commented on pull request #7171: NIFI-11441 Removed OpenCypher client service because the core depende…
MikeThomsen commented on PR #7171: URL: https://github.com/apache/nifi/pull/7171#issuecomment-1516950599 @exceptionfactory removed the test. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi] MikeThomsen commented on pull request #7169: NIFI-8161 NiFi EL: migration from SimpleDateFormat to DateTimeFormatter: rebased to 2.0
MikeThomsen commented on PR #7169: URL: https://github.com/apache/nifi/pull/7169#issuecomment-1516967002 @arkadius you raise some good questions. I would recommend not closing the PR. I'll raise the issue on the developer mailing list (which you are free to join, as it's public). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi] mtien-apache commented on a diff in pull request #7117: NIFI-11287: detect dependent properties when the property it depends on references a parameter (UI)
mtien-apache commented on code in PR #7117:
URL: https://github.com/apache/nifi/pull/7117#discussion_r1173195571
##
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js:
##
@@ -2001,6 +2020,45 @@
propertyData.setItems([]);
};
+/**
+ * Gets all the referenced parameters from the {parameterContext} based on
the value of {parameterReference} with matching {sensitive} property
+ *
+ * @param {string} parameterReference
+ * @param {ParameterContextEntity} parameterContext
+ * @param {boolean} sensitive
+ * @returns {ParameterEntity[]}
+ */
+var getExistingParametersReferenced = function (parameterReference,
parameterContext, sensitive) {
Review Comment:
@exceptionfactory I've simplified the function by removing the `sensitive`
and `parameterContext` parameters.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi] mtien-apache commented on a diff in pull request #7117: NIFI-11287: detect dependent properties when the property it depends on references a parameter (UI)
mtien-apache commented on code in PR #7117:
URL: https://github.com/apache/nifi/pull/7117#discussion_r1173203769
##
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js:
##
@@ -2261,9 +2319,32 @@
* @argument {object} propertiesThe properties
* @argument {map} descriptors The property descriptors
(property name -> property descriptor)
* @argument {map} history
+ * @argument {object} options
*/
-loadProperties: function (properties, descriptors, history) {
-return this.each(function () {
+loadProperties: function (properties, descriptors, history, options) {
+var self = this;
+var groupId = null;
+
+var loadParameterContext = function (options) {
+if (typeof options.getFullParameterContextDeferred ===
'function') {
Review Comment:
@exceptionfactory I added additional comments for the `options` argument.
The `options` would only be necessary when `loadProperties` is called from
processor configurations because it requires access to a parameter context in
order to reference parameters. `loadProperties` can be called from other parts
of NiFi, such as the Controller Service or Reporting Task that don't require
additional options. In order to know this was called from processor configs, we
can check for the parameter context options.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi] satishpops opened a new pull request, #7187: Update/satish/nifi 1.19.2
satishpops opened a new pull request, #7187: URL: https://github.com/apache/nifi/pull/7187 # Summary [NIFI-0](https://issues.apache.org/jira/browse/NIFI-0) # Tracking Please complete the following tracking steps prior to pull request creation. ### Issue Tracking - [ ] [Apache NiFi Jira](https://issues.apache.org/jira/browse/NIFI) issue created ### Pull Request Tracking - [ ] Pull Request title starts with Apache NiFi Jira issue number, such as `NIFI-0` - [ ] Pull Request commit message starts with Apache NiFi Jira issue number, as such `NIFI-0` ### Pull Request Formatting - [ ] Pull Request based on current revision of the `main` branch - [ ] Pull Request refers to a feature branch with one commit containing changes # Verification Please indicate the verification steps performed prior to pull request creation. ### Build - [ ] Build completed using `mvn clean install -P contrib-check` - [ ] JDK 11 - [ ] JDK 17 ### Licensing - [ ] New dependencies are compatible with the [Apache License 2.0](https://apache.org/licenses/LICENSE-2.0) according to the [License Policy](https://www.apache.org/legal/resolved.html) - [ ] New dependencies are documented in applicable `LICENSE` and `NOTICE` files ### Documentation - [ ] Documentation formatting appears as expected in rendered files -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi] satishpops closed pull request #7187: Update/satish/nifi 1.19.2
satishpops closed pull request #7187: Update/satish/nifi 1.19.2 URL: https://github.com/apache/nifi/pull/7187 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [nifi] MormonJesus69420 commented on pull request #7184: NIFI-11472 Make PutFTP processor more multithread friendly
MormonJesus69420 commented on PR #7184: URL: https://github.com/apache/nifi/pull/7184#issuecomment-1517320690 Strange, I don't understand why it failed on the Windows action. I don't have the ability to test it on Windows either, since we use Linux for development at work. Also my branch is based off of the latest nifi/main branch. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
