[nifi] branch master updated: NIFI-6736: Create thread on demand to handle incoming request from client for load balancing. This allows us to avoid situations where we don't have enough threads and we
This is an automated email from the ASF dual-hosted git repository.
bbende pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new 99cf87c NIFI-6736: Create thread on demand to handle incoming request
from client for load balancing. This allows us to avoid situations where we
don't have enough threads and we block on the server side, waiting for data,
when clients are trying to send data in another connection
99cf87c is described below
commit 99cf87c330a2b27757cb188a4e806a46c31ecd1b
Author: Mark Payne
AuthorDate: Wed Oct 2 09:35:46 2019 -0400
NIFI-6736: Create thread on demand to handle incoming request from client
for load balancing. This allows us to avoid situations where we don't have
enough threads and we block on the server side, waiting for data, when clients
are trying to send data in another connection
This closes #3784.
Signed-off-by: Bryan Bende
---
.../controller/queue/SwappablePriorityQueue.java | 27 +++
.../async/nio/NioAsyncLoadBalanceClient.java | 45 ++-
.../clustered/partition/RemoteQueuePartition.java | 9 ++-
.../server/ConnectionLoadBalanceServer.java| 90 --
.../clustered/server/LoadBalanceProtocol.java | 8 +-
.../server/StandardLoadBalanceProtocol.java| 7 +-
6 files changed, 99 insertions(+), 87 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java
index 8613130..b81bd3f 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java
@@ -410,6 +410,33 @@ public class SwappablePriorityQueue {
return getFlowFileQueueSize().isEmpty();
}
+public boolean isFlowFileAvailable() {
+if (isEmpty()) {
+return false;
+}
+
+readLock.lock();
+try {
+// If we have data in the active or swap queue that is penalized,
then we know that all FlowFiles
+// are penalized. As a result, we can say that no FlowFile is
available.
+FlowFileRecord firstRecord = activeQueue.peek();
+if (firstRecord == null && !swapQueue.isEmpty()) {
+firstRecord = swapQueue.get(0);
+}
+
+if (firstRecord == null) {
+// If the queue is not empty, then all data is swapped out. We
don't actually know whether or not the swapped out data is penalized, so we
assume
+// that it is not penalized and is therefore available.
+return !isEmpty();
+}
+
+// We do have a FlowFile that was retrieved from the active or
swap queue. It is available if it is not penalized.
+return !firstRecord.isPenalized();
+} finally {
+readLock.unlock("isFlowFileAvailable");
+}
+}
+
public boolean isActiveQueueEmpty() {
final FlowFileQueueSize queueSize = getFlowFileQueueSize();
return queueSize.getActiveCount() == 0 && queueSize.getSwappedCount()
== 0;
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/NioAsyncLoadBalanceClient.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/NioAsyncLoadBalanceClient.java
index 855db8e..1257a3c 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/NioAsyncLoadBalanceClient.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/NioAsyncLoadBalanceClient.java
@@ -385,38 +385,37 @@ public class NioAsyncLoadBalanceClient implements
AsyncLoadBalanceClient {
return selector != null && channel != null && channel.isConnected();
}
-private synchronized void establishConnection() throws IOException {
+private void establishConnection() throws IOException {
SocketChannel socketChannel = null;
try {
-selector = Selector.open();
-socketChannel = createChannel();
+final PeerChannel peerChannel;
+synchronized (this) {
+if (isConnectionEstablished()) {
+return;
+
[nifi-minifi] branch master updated: MINIFI-515 Use a repository location with long lived versions for the commons-daemon Windows binary files.
This is an automated email from the ASF dual-hosted git repository.
aldrin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi-minifi.git
The following commit(s) were added to refs/heads/master by this push:
new 8d26588 MINIFI-515 Use a repository location with long lived versions
for the commons-daemon Windows binary files.
8d26588 is described below
commit 8d26588caefacc072a1c86bb0a1ab09fbed56072
Author: Aldrin Piri
AuthorDate: Mon Sep 30 16:06:36 2019 -0400
MINIFI-515 Use a repository location with long lived versions for the
commons-daemon Windows binary files.
---
minifi-assembly/pom.xml | 80 -
1 file changed, 39 insertions(+), 41 deletions(-)
diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml
index cc90adc..6d5e826 100644
--- a/minifi-assembly/pom.xml
+++ b/minifi-assembly/pom.xml
@@ -281,53 +281,14 @@ limitations under the License.
1.2.1
62777d2424361162736b4a9b39b9fb4155760d4d7c4a3489fe1868e0912158ae126ef5f42abbd1bcca1dbf5aa6204d8d0029be7c49f93d6e1421a5173dd7
+
https://repo1.maven.org/maven2/commons-daemon/commons-daemon/${commons.daemon.version}
-default-assembly
-
-false
-
-
-
-
-maven-assembly-plugin
-
-minifi-${project.version}
-false
-
-
-
-make shared resource
-
-single
-
-package
-
-
-
0775
-0775
-0664
-
-
-
src/main/assembly/dependencies.xml
-
-posix
-
-
-
-
-
-
-
-
windows-service-assembly
true
-
-!default-assembly
-
@@ -359,7 +320,7 @@ limitations under the License.
-https://apache.claz.org/commons/daemon/binaries/windows/${commons.daemon.file};
dest="${java.io.tmpdir}/${commons.daemon.file}" skipexisting="true"/>
+
@@ -407,6 +368,43 @@ limitations under the License.
+
+minimal-assembly
+
+false
+
+
+
+
+maven-assembly-plugin
+
+minifi-${project.version}
+false
+
+
+
+make shared resource
+
+single
+
+package
+
+
+
0775
+0775
+0664
+
+
+
src/main/assembly/dependencies.xml
+
+posix
+
+
+
+
+
+
+
[nifi-minifi] branch master updated: MINIFI-508 Providing a profile, active by default, to download/use the Commons Daemon for running MiNiFi as a Windows Service. Those that wish to exclude these dep
This is an automated email from the ASF dual-hosted git repository.
aldrin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi-minifi.git
The following commit(s) were added to refs/heads/master by this push:
new fd99a6c MINIFI-508 Providing a profile, active by default, to
download/use the Commons Daemon for running MiNiFi as a Windows Service. Those
that wish to exclude these dependencies can use the default-assembly profile to
not require external resources.
fd99a6c is described below
commit fd99a6c72fe830b2aee5eb565e33d7c4b762d1c4
Author: Aldrin Piri
AuthorDate: Thu Sep 26 15:39:10 2019 -0400
MINIFI-508 Providing a profile, active by default, to download/use the
Commons Daemon for running MiNiFi as a Windows Service. Those that wish to
exclude these dependencies can use the default-assembly profile to not require
external resources.
---
minifi-assembly/pom.xml| 214 ++---
...encies.xml => dependencies-windows-service.xml} | 0
minifi-assembly/src/main/assembly/dependencies.xml | 10 -
pom.xml| 1 -
4 files changed, 141 insertions(+), 84 deletions(-)
diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml
index 778e350..cc90adc 100644
--- a/minifi-assembly/pom.xml
+++ b/minifi-assembly/pom.xml
@@ -25,7 +25,21 @@ limitations under the License.
minifi-assembly
pom
This is the assembly of Apache NiFi - MiNiFi
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-assembly-plugin
+3.1.1
+
+
+
org.codehaus.mojo
@@ -50,81 +64,9 @@ limitations under the License.
buildBranch
-
-org.apache.maven.plugins
-maven-antrun-plugin
-
-
-ant-contrib
-ant-contrib
-20020829
-
-
-
-
-download-commons-daemon
-prepare-package
-
-run
-
-
-
-
-
-
-
-
-
-
-
-
-https://apache.claz.org/commons/daemon/binaries/windows/${commons.daemon.file};
dest="${java.io.tmpdir}/${commons.daemon.file}" skipexisting="true"/>
-
-
-
-
-
-
-Checksum error
-
-
-
-
-
-
-
-
-
-
-
-maven-assembly-plugin
-
-minifi-${project.version}
-false
-
-
-
-make shared resource
-
-single
-
-package
-
-
-
0775
-0775
-0664
-
-
-
src/main/assembly/dependencies.xml
-
-posix
-
-
-
-
+
@@ -341,4 +283,130 @@ limitations under the License.
62777d2424361162736b4a9b39b9fb4155760d4d7c4a3489fe1868e0912158ae126ef5f42abbd1bcca1dbf5aa6204d8d0029be7c49f93d6e1421a5173dd7
+
+
+default-assembly
+
+false
+
+
+
+
+maven-assembly-plugin
+
+minifi-${project.version}
+false
+
[nifi-registry] branch master updated: NIFIREG-323 Clear secret key when auto restarting in order to obtain new secret key from the NiFi Registry process
This is an automated email from the ASF dual-hosted git repository.
thenatog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi-registry.git
The following commit(s) were added to refs/heads/master by this push:
new a20694a NIFIREG-323 Clear secret key when auto restarting in order to
obtain new secret key from the NiFi Registry process
new 6b9b6b3 Merge pull request #233 from bbende/NIFIREG-323
a20694a is described below
commit a20694a243fb6eae2573ee5af84bd069b77e1876
Author: Bryan Bende
AuthorDate: Mon Sep 30 15:13:19 2019 -0400
NIFIREG-323 Clear secret key when auto restarting in order to obtain new
secret key from the NiFi Registry process
---
.../main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java| 1 +
1 file changed, 1 insertion(+)
diff --git
a/nifi-registry-core/nifi-registry-bootstrap/src/main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java
b/nifi-registry-core/nifi-registry-bootstrap/src/main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java
index e7a46b7..af11fa4 100644
---
a/nifi-registry-core/nifi-registry-bootstrap/src/main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java
+++
b/nifi-registry-core/nifi-registry-bootstrap/src/main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java
@@ -1055,6 +1055,7 @@ public class RunNiFiRegistry {
}
defaultLogger.warn("Apache NiFi Registry appears to have
died. Restarting...");
+secretKey = null;
process = builder.start();
handleLogging(process);
[nifi-minifi-cpp] branch master updated (b95fc10 -> a3bd90f)
This is an automated email from the ASF dual-hosted git repository. aboda pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git. from b95fc10 MINIFICPP-1049 - Fix make docker add a3bd90f MINIFICPP-1025 Code review. No new revisions were added by this update. Summary of changes: .../windows-event-log/ConsumeWindowsEventLog.cpp | 255 +-- .../windows-event-log/wel/MetadataWalker.cpp | 261 +-- extensions/windows-event-log/wel/MetadataWalker.h | 28 +- .../windows-event-log/wel/WindowsEventLog.cpp | 281 ++--- extensions/windows-event-log/wel/WindowsEventLog.h | 223 extensions/windows-event-log/wel/XMLString.h | 20 +- 6 files changed, 518 insertions(+), 550 deletions(-)
[nifi] branch master updated (8d74822 -> 2f6f852)
This is an automated email from the ASF dual-hosted git repository. bbende pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/nifi.git. from 8d74822 NIFI-6275 ListHDFS now ignores scheme and authority when uses "Full Path" filter mode Updated description for "Full Path" filter mode to state that it will ignore scheme and authority Added tests to TestListHDFS for listing an empty and nonexistent dirs Updated TestListHDFS' mock file system to track state properly when FileStatus instances are added, and updated listStatus to work properly with the underlying Map that contains FileStatus instances Updated ListHDFS' addi [...] add 2f6f852 NIFI-6589: Addressed NPE No new revisions were added by this update. Summary of changes: .../nifi/controller/leader/election/CuratorLeaderElectionManager.java| 1 + 1 file changed, 1 insertion(+)
[jira] [Created] (MINIFI-516) Override processor SSLs with parent SSL
GCHQ NiFi created MINIFI-516: Summary: Override processor SSLs with parent SSL Key: MINIFI-516 URL: https://issues.apache.org/jira/browse/MINIFI-516 Project: Apache NiFi MiNiFi Issue Type: Improvement Reporter: GCHQ NiFi Assignee: GCHQ NiFi When using a ingestor to deploy flows to MiNiFi, processors using SSL contexts don't start up as the sensitive properties are not passed along in the NiFi templates. A user should be able to set a boolean in the bootstrap.conf to tell MiNiFi to ignore any custom SSL contexts in the config.yml, and repoint to the parent SSL called "SSL-Context-Service" instead. -- This message was sent by Atlassian Jira (v8.3.4#803005)
[nifi] branch master updated: NIFI-6275 ListHDFS now ignores scheme and authority when uses "Full Path" filter mode Updated description for "Full Path" filter mode to state that it will ignore scheme
This is an automated email from the ASF dual-hosted git repository.
ijokarumawak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new 8d74822 NIFI-6275 ListHDFS now ignores scheme and authority when uses
"Full Path" filter mode Updated description for "Full Path" filter mode to
state that it will ignore scheme and authority Added tests to TestListHDFS for
listing an empty and nonexistent dirs Updated TestListHDFS' mock file system to
track state properly when FileStatus instances are added, and updated
listStatus to work properly with the underlying Map that contains FileStatus
instances Updated ListHDFS' addi [...]
8d74822 is described below
commit 8d748223ff8f80c7a85fc38013ecf0b221adc2da
Author: Jeff Storck
AuthorDate: Mon May 13 16:26:36 2019 -0400
NIFI-6275 ListHDFS now ignores scheme and authority when uses "Full Path"
filter mode
Updated description for "Full Path" filter mode to state that it will
ignore scheme and authority
Added tests to TestListHDFS for listing an empty and nonexistent dirs
Updated TestListHDFS' mock file system to track state properly when
FileStatus instances are added, and updated listStatus to work properly with
the underlying Map that contains FileStatus instances
Updated ListHDFS' additional details to document "Full Path" filter mode
ignoring scheme and authority, with an example
Updated TestRunners, StandardProcessorTestRunner,
MockProcessorInitializationContext to support passing in a logger.
NIFI-6275 Updated the "Full Path" filter mode to check the full path of a
file with and without its scheme and authority against the filter regex
Added additional documentation for how ListHDFS handles scheme and
authority when "Full Path" filter mode is used
Added test case for "Full Path" filter mode with a regular expression that
includes scheme and authority
This closes #3483.
Signed-off-by: Koji Kawamura
---
.../util/MockProcessorInitializationContext.java | 6 +-
.../nifi/util/StandardProcessorTestRunner.java | 8 +-
.../java/org/apache/nifi/util/TestRunners.java | 50 +
.../apache/nifi/processors/hadoop/ListHDFS.java| 35 ++-
.../additionalDetails.html | 11 +-
.../nifi/processors/hadoop/TestListHDFS.java | 243 +++--
6 files changed, 316 insertions(+), 37 deletions(-)
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorInitializationContext.java
b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorInitializationContext.java
index e44e731..6a26371 100644
---
a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorInitializationContext.java
+++
b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorInitializationContext.java
@@ -33,8 +33,12 @@ public class MockProcessorInitializationContext implements
ProcessorInitializati
private final MockProcessContext context;
public MockProcessorInitializationContext(final Processor processor, final
MockProcessContext context) {
+this(processor, context, null);
+}
+
+public MockProcessorInitializationContext(final Processor processor, final
MockProcessContext context, final MockComponentLog logger) {
processorId = UUID.randomUUID().toString();
-logger = new MockComponentLog(processorId, processor);
+this.logger = logger == null ? new MockComponentLog(processorId,
processor) : logger;
this.context = context;
}
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
index 89af696..d995f8e 100644
---
a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
+++
b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
@@ -99,6 +99,10 @@ public class StandardProcessorTestRunner implements
TestRunner {
}
StandardProcessorTestRunner(final Processor processor, String
processorName) {
+this(processor, processorName, null);
+}
+
+StandardProcessorTestRunner(final Processor processor, String
processorName, MockComponentLog logger) {
this.processor = processor;
this.idGenerator = new AtomicLong(0L);
this.sharedState = new SharedSessionState(processor, idGenerator);
@@ -108,9 +112,9 @@ public class StandardProcessorTestRunner implements
TestRunner {
this.variableRegistry = new MockVariableRegistry();
this.context = new MockProcessContext(processor, processorName,
processorStateManager, variableRegistry);
-final MockProcessorInitializationContext mockInitContext = new
MockProcessorInitializationContext(processor, context);
+final MockProcessorInitializationContext mockInitContext = new
