This is an automated email from the ASF dual-hosted git repository. mattyb149 pushed a change to branch MINIFI-422 in repository https://gitbox.apache.org/repos/asf/nifi.git.
from 04711ab NIFI-7477 Optionally adding validation details as a new attribute of the flowfile add e31c323 NIFI-7447: When returning an object from a Controller Service, if that object is defined as an interface, proxy that interface. This way, any method call into the object will also change the classloader to the appropriate classloader. add 91dd59d NIFI-7312: Enable search in variable registry of root process group add 5c2bfcf NIFI-7369 Adding decimal support for record handling in order to avoid missing precision when reading in records add e0dd6d4 NIFI-7369: Consider DECIMAL type as a numeric type when using a CHOICE type in QueryRecord add 915617d NIFI-7299 Add basic OAuth2 token provider service that can fetch access tokens when supplied with appropriate credentials. add 359fd3f NIFI-7476: Implemented FlowFileGating / FlowFileConcurrency at the ProcessGroup level Added FlowFileOutboundPolicy to ProcessGroups and updated LocalPort to make use of it Persisted FlowFile Concurrency and FlowFile Output Policy to flow.xml.gz and included in flow fingerprint Added configuration for FlowFile concurrency and outbound policy to UI for configuration of Process Groups Added system tests. Fixed a couple of bugs that were found Fixed a couple of typos in the [...] add dfefeb7 NIFI-7508: Reset classloader after running TestStandardControllerServiceInvocationHandler and fix checkstyle violation on NiFiSystemIT add 845b66a NIFI-7467 Refactored S2S peer selection logic. Removed list structure for peer selection as it was unnecessary and often wasteful (most clusters are 3 - 7 nodes, the list was always 128 elements). Changed integer percentages to double to allow for better normalization. Removed 80% cap on remote peers as it was due to legacy requirements. Added unit tests for non-deterministic distribution calculations. Added unit tests for edge cases due to rounding errors, single valid [...] No new revisions were added by this update. Summary of changes: nifi-assembly/pom.xml | 6 + .../nifi/serialization/record/RecordFieldType.java | 19 +- .../serialization/record/ResultSetRecordSet.java | 29 +- .../serialization/record/type/DecimalDataType.java | 70 ++ .../serialization/record/util/DataTypeUtils.java | 94 +- .../record/ResultSetRecordSetTest.java | 165 +++ .../serialization/record/TestDataTypeUtils.java | 136 +++ nifi-commons/nifi-site-to-site-client/pom.xml | 9 + .../org/apache/nifi/remote/PeerDescription.java | 8 + .../java/org/apache/nifi/remote/PeerStatus.java | 20 + .../apache/nifi/remote/client/PeerSelector.java | 659 ++++++++---- .../nifi/remote/client/PeerStatusProvider.java | 6 +- .../apache/nifi/remote/client/http/HttpClient.java | 55 +- .../client/socket/EndpointConnectionPool.java | 67 +- .../remote/protocol/CommunicationsSession.java | 3 +- .../apache/nifi/remote/util/PeerStatusCache.java | 18 +- .../nifi/remote/util/SiteToSiteRestApiClient.java | 109 +- .../nifi/remote/client/PeerSelectorTest.groovy | 1133 ++++++++++++++++++++ .../nifi/remote/client/TestPeerSelector.java | 383 ------- .../src/test/resources/logback-test.xml | 11 +- .../controller/ControllerServiceProxyWrapper.java | 56 + .../elasticsearch/PutElasticsearchHttpRecord.java | 3 + .../TestPutElasticsearchHttpRecord.java | 4 +- .../java/org/apache/nifi/avro/AvroTypeUtil.java | 11 +- .../schema/access/InferenceSchemaStrategy.java | 4 + .../org/apache/nifi/avro/TestAvroTypeUtil.java | 42 +- .../schema/access/InferenceSchemaStrategyTest.java | 143 +++ .../schema/validation/StandardSchemaValidator.java | 10 +- .../validation/TestStandardSchemaValidator.java | 13 +- .../apache/nifi/web/api/dto/ProcessGroupDTO.java | 21 + .../apache/nifi/groups/FlowFileConcurrency.java | 39 + .../java/org/apache/nifi/groups/FlowFileGate.java | 18 +- .../apache/nifi/groups/FlowFileOutboundPolicy.java | 35 + .../java/org/apache/nifi/groups/ProcessGroup.java | 42 + .../org/apache/nifi/connectable/LocalPort.java | 127 ++- .../nifi/controller/StandardFlowSnippet.java | 13 + .../nifi/controller/StandardFlowSynchronizer.java | 31 + .../serialization/FlowFromDOMFactory.java | 2 + .../serialization/StandardFlowSerializer.java | 2 + ...StandardControllerServiceInvocationHandler.java | 187 +++- .../nifi/fingerprint/FingerprintFactory.java | 2 + .../nifi/groups/SingleConcurrencyFlowFileGate.java | 57 + .../apache/nifi/groups/StandardProcessGroup.java | 93 ++ .../apache/nifi/groups/UnboundedFlowFileGate.java | 25 +- .../nifi/remote/StandardRemoteProcessGroup.java | 65 +- .../src/main/resources/FlowConfiguration.xsd | 22 +- .../org/apache/nifi/connectable/TestLocalPort.java | 14 +- ...StandardControllerServiceInvocationHandler.java | 171 +++ .../controller/service/mock/MockProcessGroup.java | 45 + .../src/test/resources/logback-test.xml | 2 +- .../apache/nifi/web/api/SiteToSiteResource.java | 42 +- .../org/apache/nifi/web/api/dto/DtoFactory.java | 4 + .../web/controller/ControllerSearchService.java | 4 +- .../nifi/web/dao/impl/StandardProcessGroupDAO.java | 14 +- .../ProcessGroupSearchResultEnricher.java | 10 +- .../AbstractControllerSearchIntegrationTest.java | 8 +- .../ControllerSearchServiceFilterTest.java | 20 + .../ControllerSearchServiceIntegrationTest.java | 88 ++ .../ControllerSearchServiceRegressionTest.java | 12 +- .../controller/ControllerSearchServiceTest.java | 30 +- .../ComponentSearchResultEnricherTest.java | 61 +- .../canvas/process-group-configuration.jsp | 18 + .../webapp/css/process-group-configuration.css | 8 + .../js/nf/canvas/nf-process-group-configuration.js | 45 +- .../apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java | 6 + .../org/apache/nifi/util/orc/TestNiFiOrcUtils.java | 3 + .../apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java | 15 + .../org/apache/nifi/processors/orc/PutORCTest.java | 15 +- .../org/apache/nifi/util/orc/TestNiFiOrcUtils.java | 20 + .../nifi/controller/kudu/KuduLookupService.java | 6 +- .../processors/kudu/AbstractKuduProcessor.java | 38 +- .../org/apache/nifi/processors/kudu/PutKudu.java | 9 +- .../apache/nifi/processors/kudu/MockPutKudu.java | 15 +- .../apache/nifi/processors/kudu/TestPutKudu.java | 30 +- .../reporting/prometheus/PrometheusRecordSink.java | 1 + .../prometheus/TestPrometheusRecordSink.java | 16 +- .../nifi/rules/handlers/RecordSinkHandler.java | 5 + .../nifi/rules/handlers/TestRecordSinkHandler.java | 4 + .../reporting/AbstractSiteToSiteReportingTask.java | 1 + .../org/apache/nifi/processors/solr/SolrUtils.java | 5 +- .../apache/nifi/processors/solr/SolrUtilsTest.java | 62 ++ .../org/apache/nifi/queryrecord/FlowFileTable.java | 4 + .../nifi-oauth2-provider-api/pom.xml | 40 + .../java/org/apache/nifi/oauth2/AccessToken.java | 69 ++ .../oauth2/AccessTokenAcquisitionException.java | 24 +- .../apache/nifi/oauth2/OAuth2TokenProvider.java | 58 + .../nifi-oauth2-provider-nar/pom.xml | 47 + .../src/main/resources/META-INF/LICENSE | 0 .../src/main/resources/META-INF/NOTICE | 42 + .../nifi-oauth2-provider-service/pom.xml | 98 ++ .../nifi/oauth2/OAuth2TokenProviderImpl.java | 155 +++ .../src/main/java/org/apache/nifi/oauth2/Util.java | 139 +++ .../org.apache.nifi.controller.ControllerService | 15 + .../nifi/oauth2/OAuth2TokenProviderImplTest.java | 194 ++++ .../nifi-oauth2-provider-bundle/pom.xml | 48 + .../nifi-record-serialization-services/pom.xml | 1 + .../apache/nifi/csv/AbstractCSVRecordReader.java | 1 + .../org/apache/nifi/csv/CSVSchemaInference.java | 5 + .../java/org/apache/nifi/csv/WriteCSVResult.java | 1 + .../org/apache/nifi/json/JsonSchemaInference.java | 8 + .../apache/nifi/json/JsonTreeRowRecordReader.java | 1 + .../java/org/apache/nifi/json/WriteJsonResult.java | 3 + .../java/org/apache/nifi/xml/WriteXMLResult.java | 1 + .../java/org/apache/nifi/xml/XMLRecordReader.java | 2 + .../nifi/xml/inference/XmlSchemaInference.java | 5 + .../avro/TestAvroReaderWithEmbeddedSchema.java | 4 +- .../org/apache/nifi/avro/TestWriteAvroResult.java | 57 +- .../org/apache/nifi/csv/TestCSVRecordReader.java | 22 + .../org/apache/nifi/csv/TestWriteCSVResult.java | 4 +- .../apache/nifi/json/TestJsonSchemaInference.java | 2 +- .../nifi/json/TestJsonTreeRowRecordReader.java | 16 +- .../org/apache/nifi/json/TestWriteJsonResult.java | 2 + .../schema/inference/TestFieldTypeInference.java | 20 + .../org/apache/nifi/xml/TestWriteXMLResult.java | 5 +- .../org/apache/nifi/xml/TestXMLRecordReader.java | 17 + .../src/test/resources/avro/decimals.avsc | 36 + .../src/test/resources/json/output/dataTypes.json | 1 + .../nifi-standard-services-api-nar/pom.xml | 6 + nifi-nar-bundles/nifi-standard-services/pom.xml | 7 +- .../apache/nifi/tests/system/NiFiClientUtil.java | 72 +- .../org/apache/nifi/tests/system/NiFiSystemIT.java | 30 +- .../system/pg/SingleFlowFileConcurrencyIT.java | 152 +++ .../cli/impl/client/nifi/OutputPortClient.java | 6 + .../client/nifi/impl/JerseyOutputPortClient.java | 5 + 124 files changed, 5330 insertions(+), 1009 deletions(-) create mode 100644 nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/type/DecimalDataType.java create mode 100644 nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/ResultSetRecordSetTest.java create mode 100644 nifi-commons/nifi-site-to-site-client/src/test/groovy/org/apache/nifi/remote/client/PeerSelectorTest.groovy delete mode 100644 nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/TestPeerSelector.java copy {nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core => nifi-commons/nifi-site-to-site-client}/src/test/resources/logback-test.xml (73%) create mode 100644 nifi-framework-api/src/main/java/org/apache/nifi/controller/ControllerServiceProxyWrapper.java create mode 100644 nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/schema/access/InferenceSchemaStrategyTest.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/FlowFileConcurrency.java copy nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/OutputPortClient.java => nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/FlowFileGate.java (52%) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/FlowFileOutboundPolicy.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/SingleConcurrencyFlowFileGate.java copy nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/OutputPortClient.java => nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/UnboundedFlowFileGate.java (52%) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceInvocationHandler.java create mode 100644 nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/SolrUtilsTest.java create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-api/pom.xml create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-api/src/main/java/org/apache/nifi/oauth2/AccessToken.java copy nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/OutputPortClient.java => nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-api/src/main/java/org/apache/nifi/oauth2/AccessTokenAcquisitionException.java (52%) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-api/src/main/java/org/apache/nifi/oauth2/OAuth2TokenProvider.java create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-bundle/nifi-oauth2-provider-nar/pom.xml copy nifi-nar-bundles/{nifi-windows-event-log-bundle/nifi-windows-event-log-nar => nifi-standard-services/nifi-oauth2-provider-bundle/nifi-oauth2-provider-nar}/src/main/resources/META-INF/LICENSE (100%) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-bundle/nifi-oauth2-provider-nar/src/main/resources/META-INF/NOTICE create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-bundle/nifi-oauth2-provider-service/pom.xml create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-bundle/nifi-oauth2-provider-service/src/main/java/org/apache/nifi/oauth2/OAuth2TokenProviderImpl.java create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-bundle/nifi-oauth2-provider-service/src/main/java/org/apache/nifi/oauth2/Util.java create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-bundle/nifi-oauth2-provider-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-bundle/nifi-oauth2-provider-service/src/test/java/org/apache/nifi/oauth2/OAuth2TokenProviderImplTest.java create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-oauth2-provider-bundle/pom.xml create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/avro/decimals.avsc create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/pg/SingleFlowFileConcurrencyIT.java