(nifi) branch main updated: NIFI-12831: Add PutOpenSearchVector and QueryOpenSearchVector processors
This is an automated email from the ASF dual-hosted git repository.
pvillard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new b608e5a2f0 NIFI-12831: Add PutOpenSearchVector and
QueryOpenSearchVector processors
b608e5a2f0 is described below
commit b608e5a2f0002a0f9c6d6121815b54f6dae67dc4
Author: Mark Bathori
AuthorDate: Wed Feb 21 15:13:47 2024 +0100
NIFI-12831: Add PutOpenSearchVector and QueryOpenSearchVector processors
Signed-off-by: Pierre Villard
This closes #8441.
---
.../python/vectorstores/OpenSearchVectorUtils.py | 142
.../python/vectorstores/PutOpenSearchVector.py | 245 +
.../python/vectorstores/QueryOpenSearchVector.py | 219 ++
.../src/main/python/vectorstores/requirements.txt | 3 +
4 files changed, 609 insertions(+)
diff --git
a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/OpenSearchVectorUtils.py
b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/OpenSearchVectorUtils.py
new file mode 100644
index 00..a10eaba7c9
--- /dev/null
+++
b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/OpenSearchVectorUtils.py
@@ -0,0 +1,142 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nifiapi.properties import PropertyDescriptor, StandardValidators,
ExpressionLanguageScope, PropertyDependency
+from EmbeddingUtils import OPENAI, HUGGING_FACE, EMBEDDING_MODEL
+
+# Space types
+L2 = ("L2 (Euclidean distance)", "l2")
+L1 = ("L1 (Manhattan distance)", "l1")
+LINF = ("L-infinity (chessboard) distance", "linf")
+COSINESIMIL = ("Cosine similarity", "cosinesimil")
+
+HUGGING_FACE_API_KEY = PropertyDescriptor(
+name="HuggingFace API Key",
+description="The API Key for interacting with HuggingFace",
+required=True,
+sensitive=True,
+validators=[StandardValidators.NON_EMPTY_VALIDATOR],
+dependencies=[PropertyDependency(EMBEDDING_MODEL, HUGGING_FACE)]
+)
+HUGGING_FACE_MODEL = PropertyDescriptor(
+name="HuggingFace Model",
+description="The name of the HuggingFace model to use",
+default_value="sentence-transformers/all-MiniLM-L6-v2",
+required=True,
+validators=[StandardValidators.NON_EMPTY_VALIDATOR],
+dependencies=[PropertyDependency(EMBEDDING_MODEL, HUGGING_FACE)]
+)
+OPENAI_API_KEY = PropertyDescriptor(
+name="OpenAI API Key",
+description="The API Key for OpenAI in order to create embeddings",
+required=True,
+sensitive=True,
+validators=[StandardValidators.NON_EMPTY_VALIDATOR],
+dependencies=[PropertyDependency(EMBEDDING_MODEL, OPENAI)]
+)
+OPENAI_API_MODEL = PropertyDescriptor(
+name="OpenAI Model",
+description="The API Key for OpenAI in order to create embeddings",
+default_value="text-embedding-ada-002",
+required=True,
+validators=[StandardValidators.NON_EMPTY_VALIDATOR],
+dependencies=[PropertyDependency(EMBEDDING_MODEL, OPENAI)]
+)
+HTTP_HOST = PropertyDescriptor(
+name="HTTP Host",
+description="URL where OpenSearch is hosted.",
+default_value="http://localhost:9200;,
+required=True,
+validators=[StandardValidators.URL_VALIDATOR]
+)
+USERNAME = PropertyDescriptor(
+name="Username",
+description="The username to use for authenticating to OpenSearch server",
+required=False,
+validators=[StandardValidators.NON_EMPTY_VALIDATOR]
+)
+PASSWORD = PropertyDescriptor(
+name="Password",
+description="The password to use for authenticating to OpenSearch server",
+required=False,
+sensitive=True,
+validators=[StandardValidators.NON_EMPTY_VALIDATOR]
+)
+INDEX_NAME = PropertyDescriptor(
+name="Index Name",
+description="The name of the OpenSearch index.",
+sensitive=False,
+required=True,
+validators=[StandardValidators.NON_EMPTY_VALIDATOR],
+expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES
+)
+VECTOR_FIELD = PropertyDescriptor(
+name="Vector Field Name",
+description="The name of field in the document where the embeddings are
stored. This field need to be a
(nifi) branch main updated: [NIFI-13104] Unknown Route handling (#8725)
This is an automated email from the ASF dual-hosted git repository.
mcgilman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new f87a0f47ef [NIFI-13104] Unknown Route handling (#8725)
f87a0f47ef is described below
commit f87a0f47ef8a6a7038b371ca8e2205e49530a89a
Author: Rob Fellows
AuthorDate: Thu May 2 14:26:24 2024 -0400
[NIFI-13104] Unknown Route handling (#8725)
* [NIFI-13104] - Unknown route handling
* Add method to the ErrorHelper that safely gets an error string from an
HttpErrorResponse object
* review feedback
* use errorHelper.fullScreenError in favor of directly calling the
fullScreenError action
This closes #8725
---
.../src/main/nifi/src/app/app-routing.module.ts| 7
.../state/access-policy/access-policy.effects.ts | 16 +
.../policy-component/policy-component.effects.ts | 6 ++--
.../state/tenants/tenants.effects.ts | 6 ++--
.../cluster-listing/cluster-listing.actions.ts | 3 +-
.../cluster-listing/cluster-listing.effects.ts | 14
.../counter-listing/counter-listing.actions.ts | 3 +-
.../counter-listing/counter-listing.effects.ts | 8 +++--
.../flow-configuration-history-listing.effects.ts | 4 ++-
.../feature/flow-designer-routing.module.ts| 2 +-
.../service/parameter-helper.service.ts| 12 +--
...ntroller-service-advanced-ui-params.resolver.ts | 4 ++-
.../processor-advanced-ui-params.resolver.ts | 4 ++-
.../controller-services.effects.ts | 28 +++
.../pages/flow-designer/state/flow/flow.effects.ts | 10 +++---
.../manage-remote-ports.effects.ts | 6 ++--
.../state/parameter/parameter.effects.ts | 14 +---
.../flow-designer/state/queue/queue.effects.ts | 17 +
.../app/pages/login/state/access/access.effects.ts | 17 +
.../parameter-context-listing.effects.ts | 12 +++
.../provenance/state/lineage/lineage.effects.ts| 4 +--
.../provenance-event-listing.effects.ts| 24 +
.../state/queue-listing/queue-listing.effects.ts | 10 +++---
.../feature/route-not-found-routing.module.ts | 28 +++
.../feature/route-not-found.component.html | 22
.../feature/route-not-found.component.scss | 20 +++
.../feature/route-not-found.component.spec.ts | 39
.../feature/route-not-found.component.ts | 25 +
.../feature/route-not-found.module.ts | 29 +++
...ntroller-service-advanced-ui-params.resolver.ts | 4 ++-
...rameter-provider-advanced-ui-params.resolver.ts | 4 ++-
.../reporting-task-advanced-ui-params.resolver.ts | 4 ++-
.../flow-analysis-rules.effects.ts | 20 +++
.../settings/state/general/general.effects.ts | 4 +--
.../management-controller-services.effects.ts | 14 +---
.../parameter-providers.effects.ts | 42 +-
.../registry-clients/registry-clients.effects.ts | 10 +++---
.../reporting-tasks/reporting-tasks.effects.ts | 20 +++
.../component-cluster-status.effects.ts| 8 +++--
.../state/user-listing/user-listing.effects.ts | 38
.../nifi/src/app/service/error-helper.service.ts | 22 ++--
.../app/service/property-table-helper.service.ts | 27 +++---
.../main/nifi/src/app/state/about/about.effects.ts | 6 ++--
.../cluster-summary/cluster-summary.effects.ts | 18 ++
.../component-state/component-state.effects.ts | 25 +++--
.../controller-service-state.effects.ts| 21 ++-
.../extension-types/extension-types.effects.ts | 13 ---
.../flow-configuration.effects.ts | 11 +++---
.../state/status-history/status-history.effects.ts | 9 ++---
.../system-diagnostics.effects.ts | 31 +++-
50 files changed, 544 insertions(+), 201 deletions(-)
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/app-routing.module.ts
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/app-routing.module.ts
index 9415713b81..8e38042532 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/app-routing.module.ts
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/app-routing.module.ts
@@ -18,6 +18,7 @@
import { NgModule } from '@angular/core';
import { RouterModule, Routes } from '@angular/router';
import { authenticationGuard } from './service/guard/authentication.guard';
+import { RouteNotFound } from
'./pages/route-not-found/feature/route-not-found.component';
const routes: Routes = [
(nifi-minifi-cpp) branch MINIFICPP-2311 updated (91bbdf008 -> 2b531b86c)
This is an automated email from the ASF dual-hosted git repository.
martinzink pushed a change to branch MINIFICPP-2311
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
discard 91bbdf008 MINIFICPP-2311 Initial support for recordset readers and
writers
add df42dd374 MINIFICPP-2339 Fix sccache path in CI workflow
add ce2163403 MINIFICPP-2333 Don't register example python processors
add cd35edd01 MINIFICPP-2334 win: smb and loki extensions on by default
add cb944b192 MINIFICPP-2332 Fix data size conversion in NiFi python
processors
add 58518e434 MINIFICPP-2330 Copy the python examples from the source into
the msi
add 68ff1f8c0 MINIFICPP-2337 Fix OpenSSL toggle in bootstrap.sh
add 1198ba74a MINIFICPP-1886 - Enable msi upgrade
add 4248c5cd3 MINIFICPP-2344 Update version number to 0.99.0
add 8d7c6c56a MINIFICPP-2355 Fix build if CMAKE_BUILD_TYPE is undefined
add 2b531b86c MINIFICPP-2311 Initial support for recordset readers and
writers
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (91bbdf008)
\
N -- N -- N refs/heads/MINIFICPP-2311 (2b531b86c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/ci.yml | 2 +-
CMakeLists.txt | 30 +--
README.md | 4 +-
bootstrap.sh | 5 +-
docker/test/integration/cluster/ContainerStore.py | 3 +
.../test/integration/cluster/DockerTestCluster.py | 3 +
docker/test/integration/cluster/ImageStore.py | 14 ++-
.../cluster/containers/MinifiContainer.py | 3 +
.../features/MiNiFi_integration_test_driver.py | 3 +
docker/test/integration/features/python.feature| 23 -
docker/test/integration/features/steps/steps.py| 5 ++
...gForwarder.py => SpecialPropertyTypeChecker.py} | 7 +-
.../resources/python/SpecialPropertyTypeChecker.py | 99 ++
examples/azure_storage_config.nifi.schema.json | 4 +-
examples/consumejournald_config.nifi.schema.json | 4 +-
examples/http_post_config.nifi.schema.json | 8 +-
examples/lists3_fetchs3_config.nifi.schema.json| 10 +--
examples/querydbtable_config.nifi.schema.json | 8 +-
examples/site_to_site_config.nifi.schema.json | 2 +-
extensions/python/CMakeLists.txt | 17 +++-
.../AddPythonAttribute.py | 0
.../CountingProcessor.py | 0
.../GaussianDistributionWithNumpy.py | 0
.../MoveContentToJson.py | 0
.../RemoveFlowFile.py | 0
.../SentimentAnalysis.py | 0
.../python/pythonprocessors/nifiapi/properties.py | 12 +--
libminifi/CMakeLists.txt | 2 +-
msi/WixWin.wsi.in | 46 +-
29 files changed, 248 insertions(+), 66 deletions(-)
copy docker/test/integration/minifi/processors/{RotatingForwarder.py =>
SpecialPropertyTypeChecker.py} (84%)
create mode 100644
docker/test/integration/resources/python/SpecialPropertyTypeChecker.py
rename extensions/python/{pythonprocessors/examples =>
pythonprocessor-examples}/AddPythonAttribute.py (100%)
rename extensions/python/{pythonprocessors/examples =>
pythonprocessor-examples}/CountingProcessor.py (100%)
rename extensions/python/{pythonprocessors/examples =>
pythonprocessor-examples}/GaussianDistributionWithNumpy.py (100%)
rename extensions/python/{pythonprocessors/examples =>
pythonprocessor-examples}/MoveContentToJson.py (100%)
rename extensions/python/{pythonprocessors/examples =>
pythonprocessor-examples}/RemoveFlowFile.py (100%)
rename extensions/python/{pythonprocessors/examples =>
pythonprocessor-examples}/SentimentAnalysis.py (100%)
(nifi-minifi-cpp) branch MINIFICPP-2311 created (now 91bbdf008)
This is an automated email from the ASF dual-hosted git repository. martinzink pushed a change to branch MINIFICPP-2311 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git at 91bbdf008 MINIFICPP-2311 Initial support for recordset readers and writers No new revisions were added by this update.
(nifi) branch main updated: NIFI-13097 Set project.version in Python Extension Processors
This is an automated email from the ASF dual-hosted git repository. pvillard pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/main by this push: new 49c8d1b490 NIFI-13097 Set project.version in Python Extension Processors 49c8d1b490 is described below commit 49c8d1b4903ac9c23b10cbb89f1ff8b135b5bcf8 Author: exceptionfactory AuthorDate: Wed Apr 24 22:50:30 2024 -0500 NIFI-13097 Set project.version in Python Extension Processors - Enabled resource filtering for Python Extension modules Signed-off-by: Pierre Villard This closes #8700. --- nifi-python-extensions/nifi-openai-module/pom.xml | 1 + .../nifi-openai-module/src/main/python/PromptChatGPT.py | 2 +- nifi-python-extensions/nifi-text-embeddings-module/pom.xml | 1 + .../nifi-text-embeddings-module/src/main/python/ChunkDocument.py| 2 +- .../nifi-text-embeddings-module/src/main/python/ParseDocument.py| 2 +- .../src/main/python/vectorstores/PutChroma.py | 2 +- .../src/main/python/vectorstores/PutPinecone.py | 2 +- .../src/main/python/vectorstores/QueryChroma.py | 2 +- .../src/main/python/vectorstores/QueryPinecone.py | 2 +- 9 files changed, 9 insertions(+), 7 deletions(-) diff --git a/nifi-python-extensions/nifi-openai-module/pom.xml b/nifi-python-extensions/nifi-openai-module/pom.xml index 084bf997cc..bb45dcd1ba 100644 --- a/nifi-python-extensions/nifi-openai-module/pom.xml +++ b/nifi-python-extensions/nifi-openai-module/pom.xml @@ -35,6 +35,7 @@ src/main/python +true **/ diff --git a/nifi-python-extensions/nifi-openai-module/src/main/python/PromptChatGPT.py b/nifi-python-extensions/nifi-openai-module/src/main/python/PromptChatGPT.py index a372a20690..3b1f3fe3c7 100644 --- a/nifi-python-extensions/nifi-openai-module/src/main/python/PromptChatGPT.py +++ b/nifi-python-extensions/nifi-openai-module/src/main/python/PromptChatGPT.py @@ -30,7 +30,7 @@ class PromptChatGPT(FlowFileTransform): implements = ['org.apache.nifi.python.processor.FlowFileTransform'] class ProcessorDetails: -version = '2.0.0-SNAPSHOT' +version = '@project.version@' description = "Submits a prompt to ChatGPT, writing the results either to a FlowFile attribute or to the contents of the FlowFile" tags = ["text", "chatgpt", "gpt", "machine learning", "ML", "artificial intelligence", "ai", "document", "langchain"] dependencies = ['langchain==0.1.2', 'openai==1.9.0', 'jsonpath-ng'] diff --git a/nifi-python-extensions/nifi-text-embeddings-module/pom.xml b/nifi-python-extensions/nifi-text-embeddings-module/pom.xml index d82c2be03b..60a46cbc36 100644 --- a/nifi-python-extensions/nifi-text-embeddings-module/pom.xml +++ b/nifi-python-extensions/nifi-text-embeddings-module/pom.xml @@ -35,6 +35,7 @@ src/main/python +true **/ diff --git a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ChunkDocument.py b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ChunkDocument.py index 0300868593..db658df5bd 100644 --- a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ChunkDocument.py +++ b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ChunkDocument.py @@ -104,7 +104,7 @@ class ChunkDocument(FlowFileTransform): class Java: implements = ['org.apache.nifi.python.processor.FlowFileTransform'] class ProcessorDetails: -version = '2.0.0-SNAPSHOT' +version = '@project.version@' description = """Chunks incoming documents that are formatted as JSON Lines into chunks that are appropriately sized for creating Text Embeddings. The input is expected to be in "json-lines" format, with each line having a 'text' and a 'metadata' element. Each line will then be split into one or more lines in the output.""" diff --git a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ParseDocument.py b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ParseDocument.py index 796694f51a..7210015e95 100644 --- a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ParseDocument.py +++ b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/ParseDocument.py @@ -45,7 +45,7 @@ class ParseDocument(FlowFileTransform): implements =
(nifi-minifi-cpp) branch MINIFICPP-2354-RC2 created (now 63e213efe)
This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a change to branch MINIFICPP-2354-RC2 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git at 63e213efe MINIFICPP-2356 Fix Kafka broker connection issue to Zookeeper No new revisions were added by this update.
(nifi-minifi-cpp) annotated tag minifi-cpp-0.99.0-RC2 created (now 511cf23ae)
This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a change to annotated tag minifi-cpp-0.99.0-RC2 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git at 511cf23ae (tag) tagging 63e213efeefae112e8094e2db7292bdb03fb6ce3 (commit) replaces rel/minifi-cpp-0.15.0 by Gabor Gyimesi on Thu May 2 17:30:08 2024 +0200 - Log - MINIFICPP-2354 RC2 release candidate of NiFi MiNiFi C++ 0.99.0 -BEGIN PGP SIGNATURE- iQIzBAABCgAdFiEELgxn3pAME5QEd7bG/NCaSWHTupsFAmYzsYAACgkQ/NCaSWHT upuWeRAAvPTWbQAs5Y2E87bw3IOJ1a1DyVrEN5mEsqgazF8CgZ9qDfBmM9UkiPwe 68LuT3ieO4H0RNNTyD3IacamKs+qXrPnFHT7zjPbAfvNEfYkbOhhcC1yiOc//q49 Dv8QrVF2YtOi6tjRs3nJyKPwDVjD2nj0ste56eOuSjQfGqgy2tAp9woIrTJAjh5D gmahks+wJbhLOGYIwfqGTDJKRYGkjnzqw8hFngy88JSPh99ve9yvNZ5wOBDOCg6N O93vbdrWp2AosD31kqy3xg6UpodKF6y0iqxzPCX/UjWgUYn4CJ+vqyzPMccTwQAU j6VwT41dv26dhLYcTBTUqBLlPyh1tFbOkaXi7TYKwbxWP2Nwonn1nShwS6oDo+Mp pycogBY8pbHIFEICGTYxwod+QXOmPA5bWFe/s9LvoVgmSWuvJpTMcj0xlHOhNvTF W68Vb7B/qC2vdwilmjDaNgy2BBtrtm2T15n+R4pEU4sc3Of9kIb6wa6rr54yTpqD xsn6frcIO7A3LP+CbtJJoBUHamZ4SXO94q1QlCs88f27T5Yvokoq4lELyCWRoGHr fePACocKywizI4sCI0lLnNf+gj+H0HdhRjMa4gSzfPy0drylIA7fqchA7/wtC1nD nPg08g2j4rJ+GDk/EUOZoD6RnzfoBqfhsZwFHGUM1biVQI9HXlg= =ywkt -END PGP SIGNATURE- --- No new revisions were added by this update.
svn commit: r68921 - /dev/nifi/nifi-minifi-cpp/0.99.0/
Author: lordgamez Date: Thu May 2 15:29:22 2024 New Revision: 68921 Log: Update MiNiFi C++ 0.99.0 to RC2 source, binaries, hashes, and signature files Modified: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.asc dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha256 dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha512 dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.asc dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.sha256 dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.sha512 Modified: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz == Binary files - no diff available. Modified: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.asc == --- dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.asc (original) +++ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.asc Thu May 2 15:29:22 2024 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEELgxn3pAME5QEd7bG/NCaSWHTupsFAmYzlLQACgkQ/NCaSWHT -upuAqg//Z/MSSFrcJEFVSn2w3oh/ZUJUD5Irv69Ixtxfy3ylN5Cam7g5N0Hgnn8i -aG2gvH89SVo0PknwZGcid7Q3hqCB+qg0rgGf78ZNI/jJ0S3SLlxO3IZchET5lo4p -Pl9WMqUlmRSvgIi4gVvPnH9GeU+f/zB+IrVzeJMbU/fEn6maE9O33O5lcvULXh0K -YI4cv9aj34G4gpBJ3zPYELGWkNHPB0TIvKZ/P5GrVvsR4bhPL7ZSDkpPEBxK71/S -i/mHLqaQWlq9KjyFtz4lXFXt1ZxRPYNO4F7K92eg2kD4iJXmOSgYKDQ/XtehN6xr -BUtUTbtMWFDZ5kvIDRRJb4oUQuIF0Sy7c9I9K/8kgckNFotR+Bszf1+ADTI59jcA -OB0HwOlqtASIl8hBHWqS4m88NaAA5p2Xbh86pJeqtzcNaHC3zIV6RmIyrRWVTEXr -8Yylhxtx6fU3lEAAw/31Vcu+5m+18eULHHAjpEYi0FC5eVLn4tI3PpEPZFcZpTFb -UahpslOp9YAcjI0kawgqkspzoE/kk3PyVoeBoth3QJ2HiKPNvkFw+OZTo4HO/M+w -3L9UPC1NvHON6kig9LhdMKWZcL7copArsRxBhn/k5WAy75X78skTitxWMepSQLiC -xLB4LqEgH/nO6GKRjNVYZRdAKu4+COu/qSL8w6l863SadSCc32o= -=HTrB +iQIzBAABCgAdFiEELgxn3pAME5QEd7bG/NCaSWHTupsFAmYzsEcACgkQ/NCaSWHT +uptsAA//ZicWh8wEte2KAPH0hHf83CDO7O6Ta7Yk7UFMbayUMvYOhfUA1SkbXQdZ +V9l7yCXQx2+irHBZVikc9pdC4vU02QDOZByZbkYekkcyHnfLSwQq4HnGDjGDi5zU +S1dazMwuCqDZQTLVUnOLT5HgQ+XijdKRud1cozq4EUr72NyR5aLr77LEiFIRlQgO +YZcnveeGXCme5UThUqGQ67Rtc2A8G3cDFh+E3seMj6Ytou+3TOCJJ3E49Mc6a4mv +gQD6M4t8NHWrW4ZPc3SWRf+d7+HJ1IjPdZMs+u8xbe2cja2mMUiUxbd3sfV2Oa6h +D8SUs4Ve2DUmEDFtpGjDiL+3L39kIbPRL6RLP+HUlJBsMNMNSBdhKKMJ5WJFyp/m +bACIyL1kEzn5yt2zoC0d5cDfsYdG2njZh5ONaO+6P1+EmNFWjqHp90MAzW1/wr+S +JXIfldZxS5mhy3XUJnitok347X4/yI9ge9jKbcfcnYnYVVCi35jUf3gpOO9UtGGt +OtOdiQ+Jb3H4lt1CUcWlasUC9KZJqcG38sGBWCBdmnOQDt1mu3ppdDKxcF6xdQn+ +/ljCCQRUbW9MYEONsNSnnvjKV0tvLTFWR74Yd38tNj/sO58J2Gj/wLAMzKuDzzxn +2CAxPI+E3hG28ZKtTlAdSxWnyMYT0UKNCB8dm7511qtmNOkP1nw= +=7ChE -END PGP SIGNATURE- Modified: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha256 == --- dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha256 (original) +++ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha256 Thu May 2 15:29:22 2024 @@ -1 +1 @@ -f8591b5b580290058fed7a48e2d99200abe5588f83d8becd6b3fe6fca79ecbf3 +00d7041fa012984d6f3dd0088fd1b1d6593a7d5aab0b9e7554598e178484897d Modified: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha512 == --- dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha512 (original) +++ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha512 Thu May 2 15:29:22 2024 @@ -1 +1 @@ -af75244eaaa5467f603c4ebddb3d3007ff6388b7e1c5744c32b4ba222165458ceb40784c4936547be0a6444684eed8cc32449390b48c6fe4ddf0f8641a436848 +f550ddf0d5926844b7fe74ed76505599158e9ec8720ac596fd6069c464e2676e983a4f0dcea82d5c8546ba05553e7b16719eaa47fd8270f96102bd0958770028 Modified: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz == Binary files - no diff available. Modified: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.asc == --- dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.asc (original) +++ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.asc Thu May 2 15:29:22 2024 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEELgxn3pAME5QEd7bG/NCaSWHTupsFAmYzlLUACgkQ/NCaSWHT -uput6RAA2yyjUoS/MjR0Y1xpJ6MxOIQ1YBvPCoX4bLfoVqaH3KxpzNgRSRbdAqMq -SYSrW7rGya4IXRp7jAnVO4iolL+uTLSppA5yk9iIU0a3srAQy5NzyGYAvJQ3Muu7
svn commit: r68912 - /dev/nifi/nifi-minifi-cpp/0.99.0/
Author: lordgamez Date: Thu May 2 14:09:51 2024 New Revision: 68912 Log: MINIFICPP-2354 RC1 release candidate of NiFi MiNiFi C++ 0.99.0 Added: dev/nifi/nifi-minifi-cpp/0.99.0/ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz (with props) dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.asc dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha256 dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha512 dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz (with props) dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.asc dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.sha256 dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.sha512 dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0.msi (with props) dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0.msi.asc dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0.msi.sha256 dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0.msi.sha512 Added: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz == Binary file - no diff available. Propchange: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz -- svn:mime-type = application/octet-stream Added: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.asc == --- dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.asc (added) +++ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.asc Thu May 2 14:09:51 2024 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEELgxn3pAME5QEd7bG/NCaSWHTupsFAmYzlLQACgkQ/NCaSWHT +upuAqg//Z/MSSFrcJEFVSn2w3oh/ZUJUD5Irv69Ixtxfy3ylN5Cam7g5N0Hgnn8i +aG2gvH89SVo0PknwZGcid7Q3hqCB+qg0rgGf78ZNI/jJ0S3SLlxO3IZchET5lo4p +Pl9WMqUlmRSvgIi4gVvPnH9GeU+f/zB+IrVzeJMbU/fEn6maE9O33O5lcvULXh0K +YI4cv9aj34G4gpBJ3zPYELGWkNHPB0TIvKZ/P5GrVvsR4bhPL7ZSDkpPEBxK71/S +i/mHLqaQWlq9KjyFtz4lXFXt1ZxRPYNO4F7K92eg2kD4iJXmOSgYKDQ/XtehN6xr +BUtUTbtMWFDZ5kvIDRRJb4oUQuIF0Sy7c9I9K/8kgckNFotR+Bszf1+ADTI59jcA +OB0HwOlqtASIl8hBHWqS4m88NaAA5p2Xbh86pJeqtzcNaHC3zIV6RmIyrRWVTEXr +8Yylhxtx6fU3lEAAw/31Vcu+5m+18eULHHAjpEYi0FC5eVLn4tI3PpEPZFcZpTFb +UahpslOp9YAcjI0kawgqkspzoE/kk3PyVoeBoth3QJ2HiKPNvkFw+OZTo4HO/M+w +3L9UPC1NvHON6kig9LhdMKWZcL7copArsRxBhn/k5WAy75X78skTitxWMepSQLiC +xLB4LqEgH/nO6GKRjNVYZRdAKu4+COu/qSL8w6l863SadSCc32o= +=HTrB +-END PGP SIGNATURE- Added: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha256 == --- dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha256 (added) +++ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha256 Thu May 2 14:09:51 2024 @@ -0,0 +1 @@ +f8591b5b580290058fed7a48e2d99200abe5588f83d8becd6b3fe6fca79ecbf3 Added: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha512 == --- dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha512 (added) +++ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-bin-linux.tar.gz.sha512 Thu May 2 14:09:51 2024 @@ -0,0 +1 @@ +af75244eaaa5467f603c4ebddb3d3007ff6388b7e1c5744c32b4ba222165458ceb40784c4936547be0a6444684eed8cc32449390b48c6fe4ddf0f8641a436848 Added: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz == Binary file - no diff available. Propchange: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz -- svn:mime-type = application/octet-stream Added: dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.asc == --- dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.asc (added) +++ dev/nifi/nifi-minifi-cpp/0.99.0/nifi-minifi-cpp-0.99.0-source.tar.gz.asc Thu May 2 14:09:51 2024 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEELgxn3pAME5QEd7bG/NCaSWHTupsFAmYzlLUACgkQ/NCaSWHT +uput6RAA2yyjUoS/MjR0Y1xpJ6MxOIQ1YBvPCoX4bLfoVqaH3KxpzNgRSRbdAqMq +SYSrW7rGya4IXRp7jAnVO4iolL+uTLSppA5yk9iIU0a3srAQy5NzyGYAvJQ3Muu7 +pWDp7Mwso0vbYC3FQzhgIg4ldLmO3GpmM/0nLRU3ZTDGjJYNUCtBuF//yI724Kd3 +yl5E1q89MAzx3pKzZ9k14DvX5uoNlcT3nZEaw6a795MkoEzDOqKokpIoTeyeaEzc +bF7947jyPiwDERW0YbqpPeyoEiGvOsANeQcTkKTNuoDxTDS4WB5KKT/NHXYKja6b +ZGEiWzBbwXJg6/O+Q3+hg46bK4jvEtLhVmKSfoDgQPcBC7Pp4m/A3l7fwvzDifa4
(nifi-minifi-cpp) annotated tag minifi-cpp-0.99.0-RC1 created (now 00531ec98)
This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a change to annotated tag minifi-cpp-0.99.0-RC1 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git at 00531ec98 (tag) tagging 63e213efeefae112e8094e2db7292bdb03fb6ce3 (commit) replaces rel/minifi-cpp-0.15.0 by Gabor Gyimesi on Thu May 2 14:34:20 2024 +0200 - Log - MINIFICPP-2354 RC1 release candidate of NiFi MiNiFi C++ 0.99.0 -BEGIN PGP SIGNATURE- iQIzBAABCgAdFiEELgxn3pAME5QEd7bG/NCaSWHTupsFAmYziEwACgkQ/NCaSWHT upt/FQ//SuoJiJvfsM80AjNZJM5h2Su65iUPFLJrQQs3tQ149jZcb3ASytolSFzC b69kxugOgoi0HQLoeGdoTVKZPS5FRsyzib8SiZPb3VmFzuR/noQSD/NK2RaCMolB IKLK2G1Cb+Our8bo74m+5W10kuGWrmzIZND2tuLK4Pec6hJNF6a0VWMX7NA4BsP+ BBMCxbQLZnf3QEilwJYP+JueeOMQZuqU9/TbDIFfxGuc42xCCcFmyAoYKsJkGif+ F7KNWZl9UAcCKdieGluCY1qhKWMX3yVXiuX7TdKMEbGleRUAOxY0JgRrSrbFMBCS srdAlyEuw+/c2vfvIYSDHXJDIJeDCxbwaLB4yQBADR9lWMVwwTN5bAGrI/8Fe+t9 nYPqTGoIA8jlGFhlcwTMYEIbyBRsIilb7iamtASg4adpHZ0bJMRzODr66NskWKU9 +19wyT60LHbhTWSkh0goYOGc36n3WMKle+ua6J7iYDyeTIkjsMpBOqrHX/UzN3Mz QyKq5hoWo/rCcsWMgSw9LlR3Mmjd0Xvj3XVwzRFcS5g730wHhnHyUFKTe9TCMYbI ErAiNw4Kvf/TBMxtHY88sXDXF2vUkChqsIKozrdz+mB4aPhZzDTYCKZembS/Fg0a ouvcCEpnfWImYs5DFq2UFKqaBCzIMcR2A1HVpAjOswzOVRzIvFU= =SC8I -END PGP SIGNATURE- --- No new revisions were added by this update.
(nifi-minifi-cpp) branch MINIFICPP-2354-RC1 created (now 63e213efe)
This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a change to branch MINIFICPP-2354-RC1 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git at 63e213efe MINIFICPP-2356 Fix Kafka broker connection issue to Zookeeper No new revisions were added by this update.
(nifi-minifi-cpp) branch main updated: MINIFICPP-2356 Fix Kafka broker connection issue to Zookeeper
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 63e213efe MINIFICPP-2356 Fix Kafka broker connection issue to Zookeeper
63e213efe is described below
commit 63e213efeefae112e8094e2db7292bdb03fb6ce3
Author: Gabor Gyimesi
AuthorDate: Thu May 2 13:15:42 2024 +0200
MINIFICPP-2356 Fix Kafka broker connection issue to Zookeeper
- Additionally update Zookeeper image version
Closes #1777
Signed-off-by: Marton Szasz
---
.../test/integration/cluster/containers/KafkaBrokerContainer.py | 8 +++-
docker/test/integration/cluster/containers/ZookeeperContainer.py | 2 +-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/docker/test/integration/cluster/containers/KafkaBrokerContainer.py
b/docker/test/integration/cluster/containers/KafkaBrokerContainer.py
index 13062cd36..03cfe599a 100644
--- a/docker/test/integration/cluster/containers/KafkaBrokerContainer.py
+++ b/docker/test/integration/cluster/containers/KafkaBrokerContainer.py
@@ -43,6 +43,7 @@ class KafkaBrokerContainer(Container):
self.server_truststore_file.close()
self.server_properties_file = tempfile.NamedTemporaryFile(delete=False)
+self.feature_id = feature_context.id
with open(os.environ['TEST_DIRECTORY'] +
"/resources/kafka_broker/conf/server.properties") as server_properties_file:
server_properties_content = server_properties_file.read()
patched_server_properties_content =
server_properties_content.replace("kafka-broker",
f"kafka-broker-{feature_context.id}")
@@ -65,6 +66,10 @@ class KafkaBrokerContainer(Container):
name=self.name,
network=self.network.name,
ports={'9092/tcp': 9092, '29092/tcp': 29092, '9093/tcp': 9093,
'29093/tcp': 29093, '9094/tcp': 9094, '29094/tcp': 29094, '9094/tcp': 9094,
'29095/tcp': 29095},
+environment=[
+"ZOOKEEPER_HOST=zookeeper-" + self.feature_id,
+"ZOOKEEPER_PORT=2181"
+],
mounts=[
docker.types.Mount(
type='bind',
@@ -85,6 +90,7 @@ class KafkaBrokerContainer(Container):
type='bind',
source=self.server_truststore_file.name,
target='/usr/local/etc/kafka/certs/server_truststore.pem'
-)],
+)
+],
entrypoint=self.command)
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/ZookeeperContainer.py
b/docker/test/integration/cluster/containers/ZookeeperContainer.py
index 55829c78f..3fc383a2d 100644
--- a/docker/test/integration/cluster/containers/ZookeeperContainer.py
+++ b/docker/test/integration/cluster/containers/ZookeeperContainer.py
@@ -31,7 +31,7 @@ class ZookeeperContainer(Container):
logging.info('Creating and running zookeeper docker container...')
self.client.containers.run(
-image="zookeeper:3.8.1",
+image="zookeeper:3.9.2",
detach=True,
name=self.name,
network=self.network.name,
(nifi) branch support/nifi-1.x updated: NIFI-13121: Handle runtime exceptions in FetchHDFS
This is an automated email from the ASF dual-hosted git repository.
jgresock pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new d1579b4d21 NIFI-13121: Handle runtime exceptions in FetchHDFS
d1579b4d21 is described below
commit d1579b4d21335c1e200de079cde151f670764c01
Author: Matt Burgess
AuthorDate: Wed May 1 16:52:57 2024 -0400
NIFI-13121: Handle runtime exceptions in FetchHDFS
Signed-off-by: Joe Gresock
This closes #8727.
---
.../apache/nifi/processors/hadoop/FetchHDFS.java | 7 +++-
.../nifi/processors/hadoop/TestFetchHDFS.java | 19 +
.../processors/hadoop/util/MockFileSystem.java | 48 +-
3 files changed, 72 insertions(+), 2 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FetchHDFS.java
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FetchHDFS.java
index e026c01862..ce551acb75 100644
---
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FetchHDFS.java
+++
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FetchHDFS.java
@@ -43,6 +43,7 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.FlowFileAccessException;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import
org.apache.nifi.processors.hadoop.util.GSSExceptionRollbackYieldSessionHandler;
@@ -175,7 +176,7 @@ public class FetchHDFS extends AbstractHadoopProcessor {
outgoingFlowFile = session.putAttribute(outgoingFlowFile,
CoreAttributes.FILENAME.key(), outputFilename);
stopWatch.stop();
-getLogger().info("Successfully received content from {} for {}
in {}", new Object[]{qualifiedPath, outgoingFlowFile, stopWatch.getDuration()});
+getLogger().info("Successfully received content from {} for {}
in {}", qualifiedPath, outgoingFlowFile, stopWatch.getDuration());
outgoingFlowFile = session.putAttribute(outgoingFlowFile,
HADOOP_FILE_URL_ATTRIBUTE, qualifiedPath.toString());
session.getProvenanceReporter().fetch(outgoingFlowFile,
qualifiedPath.toString(), stopWatch.getDuration(TimeUnit.MILLISECONDS));
session.transfer(outgoingFlowFile, getSuccessRelationship());
@@ -190,6 +191,10 @@ public class FetchHDFS extends AbstractHadoopProcessor {
outgoingFlowFile = session.penalize(outgoingFlowFile);
session.transfer(outgoingFlowFile,
getCommsFailureRelationship());
}
+} catch (FlowFileAccessException ffae) {
+getLogger().error("Failed to retrieve S3 Object for {};
routing to failure", outgoingFlowFile, ffae);
+outgoingFlowFile = session.penalize(outgoingFlowFile);
+session.transfer(outgoingFlowFile,
getCommsFailureRelationship());
} finally {
IOUtils.closeQuietly(stream);
}
diff --git
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java
index 036904312c..9845c61e28 100644
---
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java
+++
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java
@@ -214,6 +214,25 @@ public class TestFetchHDFS {
fileSystem.setFailOnOpen(false);
}
+@Test
+public void testRuntimeException() {
+MockFileSystem fileSystem = new MockFileSystem();
+fileSystem.setRuntimeFailOnOpen(true);
+FetchHDFS proc = new TestableFetchHDFS(kerberosProperties, fileSystem);
+TestRunner runner = TestRunners.newTestRunner(proc);
+runner.setProperty(FetchHDFS.FILENAME,
"src/test/resources/testdata/randombytes-1.gz");
+runner.setProperty(FetchHDFS.COMPRESSION_CODEC, "NONE");
+runner.enqueue("trigger flow file");
+runner.run();
+
+runner.assertTransferCount(FetchHDFS.REL_SUCCESS, 0);
+runner.assertTransferCount(FetchHDFS.REL_FAILURE, 0);
+runner.assertTransferCount(FetchHDFS.REL_COMMS_FAILURE, 1);
+// assert that the file was penalized
+runner.assertPenalizeCount(1);
+
(nifi) branch main updated: NIFI-13121: Handle runtime exceptions in FetchHDFS
This is an automated email from the ASF dual-hosted git repository.
jgresock pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new b6a8699b57 NIFI-13121: Handle runtime exceptions in FetchHDFS
b6a8699b57 is described below
commit b6a8699b574f495cbc8ef6590929de6d91a0cd95
Author: Matt Burgess
AuthorDate: Wed May 1 16:52:57 2024 -0400
NIFI-13121: Handle runtime exceptions in FetchHDFS
Signed-off-by: Joe Gresock
This closes #8727.
---
.../apache/nifi/processors/hadoop/FetchHDFS.java | 7 +++-
.../nifi/processors/hadoop/TestFetchHDFS.java | 19 +
.../processors/hadoop/util/MockFileSystem.java | 48 +-
3 files changed, 72 insertions(+), 2 deletions(-)
diff --git
a/nifi-extension-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FetchHDFS.java
b/nifi-extension-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FetchHDFS.java
index e026c01862..ce551acb75 100644
---
a/nifi-extension-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FetchHDFS.java
+++
b/nifi-extension-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FetchHDFS.java
@@ -43,6 +43,7 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.FlowFileAccessException;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import
org.apache.nifi.processors.hadoop.util.GSSExceptionRollbackYieldSessionHandler;
@@ -175,7 +176,7 @@ public class FetchHDFS extends AbstractHadoopProcessor {
outgoingFlowFile = session.putAttribute(outgoingFlowFile,
CoreAttributes.FILENAME.key(), outputFilename);
stopWatch.stop();
-getLogger().info("Successfully received content from {} for {}
in {}", new Object[]{qualifiedPath, outgoingFlowFile, stopWatch.getDuration()});
+getLogger().info("Successfully received content from {} for {}
in {}", qualifiedPath, outgoingFlowFile, stopWatch.getDuration());
outgoingFlowFile = session.putAttribute(outgoingFlowFile,
HADOOP_FILE_URL_ATTRIBUTE, qualifiedPath.toString());
session.getProvenanceReporter().fetch(outgoingFlowFile,
qualifiedPath.toString(), stopWatch.getDuration(TimeUnit.MILLISECONDS));
session.transfer(outgoingFlowFile, getSuccessRelationship());
@@ -190,6 +191,10 @@ public class FetchHDFS extends AbstractHadoopProcessor {
outgoingFlowFile = session.penalize(outgoingFlowFile);
session.transfer(outgoingFlowFile,
getCommsFailureRelationship());
}
+} catch (FlowFileAccessException ffae) {
+getLogger().error("Failed to retrieve S3 Object for {};
routing to failure", outgoingFlowFile, ffae);
+outgoingFlowFile = session.penalize(outgoingFlowFile);
+session.transfer(outgoingFlowFile,
getCommsFailureRelationship());
} finally {
IOUtils.closeQuietly(stream);
}
diff --git
a/nifi-extension-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java
b/nifi-extension-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java
index 036904312c..9845c61e28 100644
---
a/nifi-extension-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java
+++
b/nifi-extension-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java
@@ -214,6 +214,25 @@ public class TestFetchHDFS {
fileSystem.setFailOnOpen(false);
}
+@Test
+public void testRuntimeException() {
+MockFileSystem fileSystem = new MockFileSystem();
+fileSystem.setRuntimeFailOnOpen(true);
+FetchHDFS proc = new TestableFetchHDFS(kerberosProperties, fileSystem);
+TestRunner runner = TestRunners.newTestRunner(proc);
+runner.setProperty(FetchHDFS.FILENAME,
"src/test/resources/testdata/randombytes-1.gz");
+runner.setProperty(FetchHDFS.COMPRESSION_CODEC, "NONE");
+runner.enqueue("trigger flow file");
+runner.run();
+
+runner.assertTransferCount(FetchHDFS.REL_SUCCESS, 0);
+runner.assertTransferCount(FetchHDFS.REL_FAILURE, 0);
+runner.assertTransferCount(FetchHDFS.REL_COMMS_FAILURE, 1);
+// assert that the file was penalized
+
