NIFI-40: Merged in NIFI-250 and bug fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/79c60016 Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/79c60016 Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/79c60016 Branch: refs/heads/prov-query-language Commit: 79c60016384f481b1f6348f7f1c08f9ccb6d51ef Parents: df43c8b fe09680 Author: Mark Payne <marka...@hotmail.com> Authored: Thu Mar 12 15:38:49 2015 -0400 Committer: Mark Payne <marka...@hotmail.com> Committed: Thu Mar 12 15:38:49 2015 -0400 ---------------------------------------------------------------------- README.md | 6 +- nifi-site/Gruntfile.js | 5 +- nifi-site/src/includes/topbar.hbs | 7 +- nifi-site/src/js/app.js | 2 + nifi-site/src/pages/html/developer-guide.hbs | 7 + nifi-site/src/pages/html/download.hbs | 2 +- nifi-site/src/pages/html/overview.hbs | 7 + nifi-site/src/pages/html/screencasts.hbs | 50 + nifi-site/src/pages/html/user-guide.hbs | 7 + nifi-site/src/scss/app.scss | 22 + nifi/LICENSE | 342 ++-- nifi/NOTICE | 11 +- nifi/README.md | 26 +- nifi/nifi-api/pom.xml | 1 - .../nifi/annotation/lifecycle/OnDisabled.java | 29 +- .../nifi/annotation/lifecycle/OnEnabled.java | 34 +- .../annotation/lifecycle/OnUnscheduled.java | 2 - .../AbstractConfigurableComponent.java | 13 - .../nifi/components/PropertyDescriptor.java | 11 +- .../nifi/components/ValidationContext.java | 9 + .../controller/AbstractControllerService.java | 13 +- .../ControllerServiceInitializationContext.java | 10 + .../controller/ControllerServiceLookup.java | 7 + .../org/apache/nifi/logging/ComponentLog.java | 100 ++ .../org/apache/nifi/logging/ProcessorLog.java | 61 +- .../AbstractSessionFactoryProcessor.java | 2 - .../apache/nifi/remote/RemoteDestination.java | 53 + .../nifi/reporting/AbstractReportingTask.java | 29 + .../ReportingInitializationContext.java | 10 + .../main/java/org/apache/nifi/web/Revision.java | 32 +- nifi/nifi-assembly/LICENSE | 831 +++++++++ nifi/nifi-assembly/NOTICE | 537 ++++++ nifi/nifi-assembly/README.md | 124 ++ nifi/nifi-assembly/pom.xml | 18 +- .../src/main/assembly/dependencies.xml | 6 +- .../nifi-data-provenance-utils/pom.xml | 1 - .../nifi-expression-language/pom.xml | 1 - .../nifi-commons/nifi-flowfile-packager/pom.xml | 1 - nifi/nifi-commons/nifi-logging-utils/pom.xml | 1 - .../nifi-processor-utilities/pom.xml | 1 - nifi/nifi-commons/nifi-properties/pom.xml | 1 - .../org/apache/nifi/util/NiFiProperties.java | 5 + .../org/apache/nifi/pql/ProvenanceQuery.java | 111 +- nifi/nifi-commons/nifi-security-utils/pom.xml | 1 - .../nifi-site-to-site-client/pom.xml | 45 + .../remote/AbstractCommunicationsSession.java | 54 + .../org/apache/nifi/remote/Communicant.java | 47 + .../main/java/org/apache/nifi/remote/Peer.java | 151 ++ .../org/apache/nifi/remote/PeerDescription.java | 79 + .../java/org/apache/nifi/remote/PeerStatus.java | 60 + .../nifi/remote/RemoteResourceInitiator.java | 73 + .../org/apache/nifi/remote/Transaction.java | 214 +++ .../nifi/remote/TransactionCompletion.java | 63 + .../apache/nifi/remote/TransferDirection.java | 34 + .../nifi/remote/VersionedRemoteResource.java | 24 + .../nifi/remote/client/SiteToSiteClient.java | 519 ++++++ .../remote/client/SiteToSiteClientConfig.java | 124 ++ .../client/socket/EndpointConnection.java | 54 + .../client/socket/EndpointConnectionPool.java | 965 +++++++++++ .../nifi/remote/client/socket/SocketClient.java | 218 +++ .../remote/cluster/AdaptedNodeInformation.java | 66 + .../remote/cluster/ClusterNodeInformation.java | 67 + .../nifi/remote/cluster/NodeInformation.java | 98 ++ .../remote/cluster/NodeInformationAdapter.java | 41 + .../apache/nifi/remote/codec/FlowFileCodec.java | 71 + .../remote/codec/StandardFlowFileCodec.java | 129 ++ .../remote/exception/HandshakeException.java | 37 + .../exception/PortNotRunningException.java | 30 + .../remote/exception/ProtocolException.java | 40 + .../remote/exception/UnknownPortException.java | 29 + .../SocketChannelCommunicationsSession.java | 110 ++ .../remote/io/socket/SocketChannelInput.java | 71 + .../remote/io/socket/SocketChannelOutput.java | 58 + .../SSLSocketChannelCommunicationsSession.java | 113 ++ .../io/socket/ssl/SSLSocketChannelInput.java | 55 + .../io/socket/ssl/SSLSocketChannelOutput.java | 44 + .../nifi/remote/protocol/ClientProtocol.java | 86 + .../remote/protocol/CommunicationsInput.java | 33 + .../remote/protocol/CommunicationsOutput.java | 27 + .../remote/protocol/CommunicationsSession.java | 64 + .../apache/nifi/remote/protocol/DataPacket.java | 45 + .../nifi/remote/protocol/RequestType.java | 43 + .../protocol/socket/HandshakeProperty.java | 61 + .../nifi/remote/protocol/socket/Response.java | 51 + .../remote/protocol/socket/ResponseCode.java | 153 ++ .../protocol/socket/SocketClientProtocol.java | 438 +++++ .../socket/SocketClientTransaction.java | 399 +++++ .../SocketClientTransactionCompletion.java | 57 + .../nifi/remote/util/NiFiRestApiUtil.java | 98 ++ .../nifi/remote/util/PeerStatusCache.java | 43 + .../nifi/remote/util/StandardDataPacket.java | 50 + .../socket/TestEndpointConnectionStatePool.java | 95 ++ .../client/socket/TestSiteToSiteClient.java | 105 ++ nifi/nifi-commons/nifi-socket-utils/pom.xml | 1 - .../io/socket/SocketChannelInputStream.java | 12 + .../remote/io/socket/ssl/SSLSocketChannel.java | 12 + .../socket/ssl/SSLSocketChannelInputStream.java | 4 + .../java/org/apache/nifi/util/RingBuffer.java | 2 +- .../nifi/util/timebuffer/TestRingBuffer.java | 10 + nifi/nifi-commons/nifi-web-utils/pom.xml | 1 - nifi/nifi-commons/nifi-write-ahead-log/pom.xml | 1 - .../org/wali/MinimalLockingWriteAheadLog.java | 7 +- nifi/nifi-commons/pom.xml | 1 + nifi/nifi-docs/LICENSE | 235 +++ nifi/nifi-docs/NOTICE | 5 + nifi/nifi-docs/pom.xml | 30 +- .../src/main/asciidoc/administration-guide.adoc | 168 +- .../src/main/asciidoc/asciidoc-mod.css | 418 +++++ .../src/main/asciidoc/developer-guide.adoc | 32 +- nifi/nifi-docs/src/main/asciidoc/images/ncm.png | Bin 0 -> 339522 bytes nifi/nifi-docs/src/main/asciidoc/overview.adoc | 24 +- .../nifi-docs/src/main/asciidoc/user-guide.adoc | 11 + .../src/main/assembly/dependencies.xml | 16 + nifi/nifi-external/README.md | 19 + nifi/nifi-external/nifi-spark-receiver/pom.xml | 37 + .../org/apache/nifi/spark/NiFiDataPacket.java | 40 + .../org/apache/nifi/spark/NiFiReceiver.java | 198 +++ nifi/nifi-external/pom.xml | 29 + nifi/nifi-mock/pom.xml | 1 - .../MockProvenanceEventRepository.java | 11 + ...kControllerServiceInitializationContext.java | 17 + .../nifi/util/MockControllerServiceLookup.java | 9 + .../apache/nifi/util/MockProcessContext.java | 4 +- .../MockProcessorInitializationContext.java | 5 + .../org/apache/nifi/util/MockProcessorLog.java | 34 +- .../MockReportingInitializationContext.java | 10 +- .../apache/nifi/util/MockValidationContext.java | 11 + .../nifi/util/StandardProcessorTestRunner.java | 5 +- .../nifi-framework-nar/pom.xml | 1 - .../src/main/resources/META-INF/LICENSE | 264 +++ .../src/main/resources/META-INF/NOTICE | 131 ++ .../nifi-framework/nifi-administration/pom.xml | 1 - .../org/apache/nifi/admin/dao/ActionDAO.java | 6 +- .../nifi/admin/dao/impl/StandardActionDAO.java | 60 +- .../apache/nifi/admin/service/AuditService.java | 6 +- .../admin/service/action/GetPreviousValues.java | 8 +- .../service/impl/StandardAuditService.java | 4 +- .../nifi-framework/nifi-client-dto/.gitignore | 1 + .../nifi/web/api/dto/ComponentHistoryDTO.java | 56 + .../web/api/dto/ControllerConfigurationDTO.java | 18 + .../nifi/web/api/dto/ControllerServiceDTO.java | 175 ++ ...ontrollerServiceReferencingComponentDTO.java | 177 ++ .../nifi/web/api/dto/DocumentedTypeDTO.java | 20 +- .../nifi/web/api/dto/ProcessorConfigDTO.java | 215 +-- .../nifi/web/api/dto/ProcessorHistoryDTO.java | 56 - .../nifi/web/api/dto/PropertyDescriptorDTO.java | 243 +++ .../nifi/web/api/dto/ReportingTaskDTO.java | 183 ++ .../apache/nifi/web/api/dto/RevisionDTO.java | 15 + .../component/details/ComponentDetailsDTO.java | 2 +- .../component/details/ExtensionDetailsDTO.java | 41 + .../component/details/ProcessorDetailsDTO.java | 41 - .../web/api/entity/ComponentHistoryEntity.java | 45 + .../web/api/entity/ControllerServiceEntity.java | 45 + ...ollerServiceReferencingComponentsEntity.java | 46 + .../entity/ControllerServiceTypesEntity.java | 46 + .../api/entity/ControllerServicesEntity.java | 46 + .../web/api/entity/ProcessorHistoryEntity.java | 45 - .../web/api/entity/ReportingTaskEntity.java | 45 + .../api/entity/ReportingTaskTypesEntity.java | 46 + .../web/api/entity/ReportingTasksEntity.java | 46 + .../nifi-cluster-protocol/pom.xml | 1 - .../nifi-framework/nifi-cluster-web/pom.xml | 4 - .../context/ClusterContextThreadLocal.java | 7 +- .../ClusterAwareOptimisticLockingManager.java | 96 -- .../nifi-framework/nifi-cluster/pom.xml | 8 + .../nifi/cluster/flow/ClusterDataFlow.java | 15 +- .../cluster/flow/DataFlowManagementService.java | 17 + .../nifi/cluster/flow/impl/DataFlowDaoImpl.java | 43 +- .../impl/DataFlowManagementServiceImpl.java | 65 +- .../nifi/cluster/manager/ClusterManager.java | 18 +- .../manager/impl/ClusteredEventAccess.java | 11 + .../cluster/manager/impl/WebClusterManager.java | 502 ++++-- .../spring/WebClusterManagerFactoryBean.java | 16 +- .../resources/nifi-cluster-manager-context.xml | 4 + .../nifi-file-authorization-provider/pom.xml | 1 - .../nifi-framework-core-api/.gitignore | 2 + .../nifi-framework-core-api/pom.xml | 5 +- .../nifi/cluster/AdaptedNodeInformation.java | 66 - .../nifi/cluster/ClusterNodeInformation.java | 67 - .../org/apache/nifi/cluster/NodeInformant.java | 22 - .../apache/nifi/cluster/NodeInformation.java | 98 -- .../nifi/cluster/NodeInformationAdapter.java | 39 - .../controller/AbstractConfiguredComponent.java | 18 +- .../apache/nifi/controller/Availability.java | 24 - .../nifi/controller/ProcessScheduler.java | 27 +- .../apache/nifi/controller/ProcessorNode.java | 16 + .../nifi/controller/ReportingTaskNode.java | 26 +- .../apache/nifi/controller/StandardFunnel.java | 2 +- .../controller/ValidationContextFactory.java | 4 + .../reporting/ReportingTaskProvider.java | 103 ++ .../service/ControllerServiceNode.java | 40 +- .../service/ControllerServiceProvider.java | 54 +- .../service/ControllerServiceReference.java | 7 +- .../service/ControllerServiceState.java | 45 + .../apache/nifi/groups/RemoteProcessGroup.java | 39 +- .../main/java/org/apache/nifi/remote/Peer.java | 107 -- .../java/org/apache/nifi/remote/PeerStatus.java | 72 - .../org/apache/nifi/remote/RemoteGroupPort.java | 22 +- .../apache/nifi/remote/TransferDirection.java | 23 - .../nifi/remote/VersionedRemoteResource.java | 24 - .../nifi/remote/cluster/NodeInformant.java | 22 + .../apache/nifi/remote/codec/FlowFileCodec.java | 79 - .../remote/exception/HandshakeException.java | 30 - .../exception/PortNotRunningException.java | 26 - .../remote/exception/ProtocolException.java | 34 - .../remote/exception/UnknownPortException.java | 26 - .../nifi/remote/protocol/ClientProtocol.java | 78 - .../remote/protocol/CommunicationsInput.java | 27 - .../remote/protocol/CommunicationsOutput.java | 27 - .../remote/protocol/CommunicationsSession.java | 64 - .../nifi/remote/protocol/RequestType.java | 43 - .../nifi/remote/protocol/ServerProtocol.java | 2 +- .../nifi-framework/nifi-framework-core/pom.xml | 4 + .../apache/nifi/controller/FlowController.java | 115 +- .../nifi/controller/FlowFromDOMFactory.java | 60 +- .../nifi/controller/StandardFlowSerializer.java | 68 +- .../nifi/controller/StandardFlowService.java | 9 +- .../controller/StandardFlowSynchronizer.java | 163 +- .../nifi/controller/StandardProcessorNode.java | 47 +- .../apache/nifi/controller/TemplateManager.java | 2 +- .../reporting/AbstractReportingTaskNode.java | 68 +- .../reporting/StandardReportingContext.java | 6 + .../StandardReportingInitializationContext.java | 18 +- .../repository/StandardProcessSession.java | 46 +- .../scheduling/QuartzSchedulingAgent.java | 21 +- .../controller/scheduling/ScheduleState.java | 18 +- .../scheduling/StandardProcessScheduler.java | 184 +- .../scheduling/TimerDrivenSchedulingAgent.java | 124 +- .../service/ControllerServiceLoader.java | 156 +- ...dControllerServiceInitializationContext.java | 15 +- .../service/StandardControllerServiceNode.java | 122 +- .../StandardControllerServiceProvider.java | 281 +++- .../StandardControllerServiceReference.java | 19 +- .../tasks/ContinuallyRunConnectableTask.java | 32 +- .../tasks/ContinuallyRunProcessorTask.java | 82 +- .../nifi/fingerprint/FingerprintFactory.java | 79 + .../nifi/groups/StandardProcessGroup.java | 25 +- .../nifi/persistence/FlowConfigurationDAO.java | 25 - .../StandardXMLFlowConfigurationDAO.java | 191 +-- .../nifi/processor/SimpleProcessLogger.java | 36 +- .../nifi/processor/StandardProcessContext.java | 5 + .../processor/StandardSchedulingContext.java | 5 +- .../processor/StandardValidationContext.java | 14 + .../StandardValidationContextFactory.java | 5 + .../org/apache/nifi/remote/RemoteNiFiUtils.java | 216 +++ .../nifi/remote/StandardRemoteProcessGroup.java | 353 +--- .../ControllerServiceConfiguration.xsd | 61 - .../src/main/resources/FlowConfiguration.xsd | 49 +- .../resources/ReportingTaskConfiguration.xsd | 87 - .../TestStandardControllerServiceProvider.java | 235 +++ .../controller/service/mock/DummyProcessor.java | 49 + .../nifi/controller/service/mock/ServiceA.java | 49 + .../nifi/controller/service/mock/ServiceB.java | 23 + .../processor/TestStandardPropertyValue.java | 6 +- .../src/main/resources/conf/nifi.properties | 2 + .../nifi-framework/nifi-security/pom.xml | 1 - .../nifi-framework/nifi-site-to-site/.gitignore | 1 + .../nifi-framework/nifi-site-to-site/pom.xml | 5 +- .../remote/AbstractCommunicationsSession.java | 54 - .../nifi/remote/RemoteResourceFactory.java | 50 +- .../nifi/remote/SocketRemoteSiteListener.java | 16 +- .../nifi/remote/StandardRemoteGroupPort.java | 697 +++----- .../remote/codec/StandardFlowFileCodec.java | 169 -- .../SocketChannelCommunicationsSession.java | 90 - .../remote/io/socket/SocketChannelInput.java | 66 - .../remote/io/socket/SocketChannelOutput.java | 58 - .../SSLSocketChannelCommunicationsSession.java | 93 -- .../io/socket/ssl/SSLSocketChannelInput.java | 50 - .../io/socket/ssl/SSLSocketChannelOutput.java | 44 - .../socket/ClusterManagerServerProtocol.java | 7 +- .../protocol/socket/HandshakeProperty.java | 23 - .../nifi/remote/protocol/socket/Response.java | 51 - .../remote/protocol/socket/ResponseCode.java | 152 -- .../protocol/socket/SocketClientProtocol.java | 510 ------ .../socket/SocketFlowFileServerProtocol.java | 199 ++- .../remote/TestStandardRemoteGroupPort.java | 97 -- .../nifi-framework/nifi-user-actions/pom.xml | 1 - .../java/org/apache/nifi/action/Component.java | 4 +- .../component/details/ExtensionDetails.java | 34 + .../component/details/ProcessorDetails.java | 34 - .../nifi-web/nifi-web-api/pom.xml | 2 +- .../nifi/audit/ControllerServiceAuditor.java | 436 +++++ .../org/apache/nifi/audit/FunnelAuditor.java | 8 +- .../java/org/apache/nifi/audit/NiFiAuditor.java | 11 +- .../java/org/apache/nifi/audit/PortAuditor.java | 17 +- .../apache/nifi/audit/ProcessGroupAuditor.java | 18 +- .../org/apache/nifi/audit/ProcessorAuditor.java | 20 +- .../apache/nifi/audit/RelationshipAuditor.java | 18 +- .../nifi/audit/RemoteProcessGroupAuditor.java | 22 +- .../apache/nifi/audit/ReportingTaskAuditor.java | 353 ++++ .../org/apache/nifi/audit/SnippetAuditor.java | 13 +- .../org/apache/nifi/web/NiFiServiceFacade.java | 177 +- .../nifi/web/StandardNiFiServiceFacade.java | 1331 ++++++++------- .../apache/nifi/web/StandardNiFiWebContext.java | 22 +- .../nifi/web/api/ApplicationResource.java | 76 +- .../apache/nifi/web/api/ClusterResource.java | 2 +- .../apache/nifi/web/api/ConnectionResource.java | 6 +- .../apache/nifi/web/api/ControllerResource.java | 99 +- .../nifi/web/api/ControllerServiceResource.java | 728 ++++++++ .../org/apache/nifi/web/api/FunnelResource.java | 6 +- .../apache/nifi/web/api/HistoryResource.java | 70 +- .../apache/nifi/web/api/InputPortResource.java | 6 +- .../org/apache/nifi/web/api/LabelResource.java | 6 +- .../apache/nifi/web/api/OutputPortResource.java | 6 +- .../nifi/web/api/ProcessGroupResource.java | 16 +- .../apache/nifi/web/api/ProcessorResource.java | 6 +- .../web/api/RemoteProcessGroupResource.java | 18 +- .../nifi/web/api/ReportingTaskResource.java | 580 +++++++ .../apache/nifi/web/api/SnippetResource.java | 6 +- .../org/apache/nifi/web/api/dto/DtoFactory.java | 351 +++- .../nifi/web/controller/ControllerFacade.java | 85 +- .../nifi/web/dao/ControllerServiceDAO.java | 110 ++ .../apache/nifi/web/dao/ReportingTaskDAO.java | 88 + .../dao/impl/StandardControllerServiceDAO.java | 303 ++++ .../web/dao/impl/StandardReportingTaskDAO.java | 304 ++++ .../ControllerServiceProviderFactoryBean.java | 68 + .../OptimisticLockingManagerFactoryBean.java | 67 + .../ReportingTaskProviderFactoryBean.java | 69 + .../org/apache/nifi/web/util/Availability.java | 34 + .../src/main/resources/META-INF/NOTICE | 27 + .../src/main/resources/nifi-web-api-context.xml | 49 +- .../src/main/resources/META-INF/NOTICE | 19 + .../nifi-web-optimistic-locking/pom.xml | 12 + .../apache/nifi/web/ConfigurationRequest.java | 34 + .../apache/nifi/web/ConfigurationSnapshot.java | 22 +- .../org/apache/nifi/web/FlowModification.java | 57 + .../nifi/web/OptimisticLockingManager.java | 76 +- .../web/StandardOptimisticLockingManager.java | 150 +- .../org/apache/nifi/web/UpdateRevision.java | 31 + .../nifi/web/security/user/NiFiUserUtils.java | 10 + .../nifi-framework/nifi-web/nifi-web-ui/pom.xml | 10 +- .../src/main/resources/META-INF/LICENSE | 434 +++++ .../main/resources/filters/canvas.properties | 6 +- .../src/main/webapp/WEB-INF/pages/canvas.jsp | 11 +- .../src/main/webapp/WEB-INF/pages/summary.jsp | 2 + .../WEB-INF/partials/canvas/canvas-header.jsp | 2 +- .../canvas/controller-service-configuration.jsp | 80 + .../disable-controller-service-dialog.jsp | 68 + .../canvas/enable-controller-service-dialog.jsp | 67 + .../partials/canvas/fill-color-dialog.jsp | 6 +- .../canvas/new-controller-service-dialog.jsp | 53 + .../partials/canvas/new-processor-dialog.jsp | 6 +- .../canvas/new-processor-property-dialog.jsp | 34 - .../canvas/new-remote-process-group-dialog.jsp | 2 +- .../canvas/new-reporting-task-dialog.jsp | 53 + .../partials/canvas/processor-configuration.jsp | 18 +- .../WEB-INF/partials/canvas/registration.jsp | 2 +- .../canvas/reporting-task-configuration.jsp | 63 + .../partials/canvas/settings-content.jsp | 98 +- .../WEB-INF/partials/processor-details.jsp | 4 - .../nifi-web-ui/src/main/webapp/css/about.css | 1 - .../nifi-web-ui/src/main/webapp/css/canvas.css | 4 + .../webapp/css/connection-configuration.css | 1 - .../src/main/webapp/css/connection-details.css | 1 - .../src/main/webapp/css/controller-service.css | 268 +++ .../nifi-web-ui/src/main/webapp/css/dialog.css | 31 +- .../src/main/webapp/css/label-configuration.css | 1 - .../nifi-web-ui/src/main/webapp/css/main.css | 42 +- .../css/new-controller-service-dialog.css | 152 ++ .../main/webapp/css/new-processor-dialog.css | 53 +- .../webapp/css/new-reporting-task-dialog.css | 152 ++ .../src/main/webapp/css/port-configuration.css | 2 - .../src/main/webapp/css/port-details.css | 2 - .../webapp/css/process-group-configuration.css | 1 - .../main/webapp/css/process-group-details.css | 1 - .../main/webapp/css/processor-configuration.css | 153 -- .../src/main/webapp/css/processor-details.css | 27 - .../src/main/webapp/css/registration.css | 8 - .../css/remote-process-group-configuration.css | 3 - .../src/main/webapp/css/reporting-task.css | 80 + .../src/main/webapp/css/settings.css | 138 +- .../nifi-web-ui/src/main/webapp/css/shell.css | 2 +- .../src/main/webapp/css/status-history.css | 1 - .../main/webapp/images/buttonNewProperty.png | Bin 590 -> 0 bytes .../src/main/webapp/images/iconEnable.png | Bin 0 -> 472 bytes .../src/main/webapp/images/iconUndo.png | Bin 642 -> 0 bytes .../src/main/webapp/js/jquery/jquery.each.js | 2 +- .../webapp/js/jquery/modal/jquery.modal.css | 12 +- .../main/webapp/js/jquery/modal/jquery.modal.js | 108 +- .../js/jquery/nfeditor/jquery.nfeditor.js | 5 +- .../jquery/propertytable/buttonNewProperty.png | Bin 0 -> 590 bytes .../propertytable/jquery.propertytable.css | 176 ++ .../propertytable/jquery.propertytable.js | 1294 ++++++++++++++ .../main/webapp/js/jquery/tabbs/jquery.tabbs.js | 2 + .../js/jquery/tagcloud/jquery.tagcloud.css | 62 + .../js/jquery/tagcloud/jquery.tagcloud.js | 226 +++ .../js/nf/bulletin-board/nf-bulletin-board.js | 3 + .../src/main/webapp/js/nf/canvas/nf-actions.js | 77 +- .../src/main/webapp/js/nf/canvas/nf-birdseye.js | 3 + .../webapp/js/nf/canvas/nf-canvas-header.js | 180 +- .../webapp/js/nf/canvas/nf-canvas-toolbar.js | 21 +- .../webapp/js/nf/canvas/nf-canvas-toolbox.js | 227 +-- .../main/webapp/js/nf/canvas/nf-canvas-utils.js | 30 +- .../src/main/webapp/js/nf/canvas/nf-canvas.js | 55 +- .../main/webapp/js/nf/canvas/nf-clipboard.js | 3 + .../main/webapp/js/nf/canvas/nf-connectable.js | 114 +- .../js/nf/canvas/nf-connection-configuration.js | 3 + .../main/webapp/js/nf/canvas/nf-connection.js | 3 + .../main/webapp/js/nf/canvas/nf-context-menu.js | 12 +- .../js/nf/canvas/nf-controller-service.js | 1435 ++++++++++++++++ .../js/nf/canvas/nf-custom-processor-ui.js | 3 + .../main/webapp/js/nf/canvas/nf-draggable.js | 3 + .../src/main/webapp/js/nf/canvas/nf-funnel.js | 3 + .../src/main/webapp/js/nf/canvas/nf-go-to.js | 3 + .../webapp/js/nf/canvas/nf-graph-control.js | 3 + .../src/main/webapp/js/nf/canvas/nf-graph.js | 9 +- .../js/nf/canvas/nf-label-configuration.js | 3 + .../src/main/webapp/js/nf/canvas/nf-label.js | 3 + .../js/nf/canvas/nf-port-configuration.js | 3 + .../main/webapp/js/nf/canvas/nf-port-details.js | 3 + .../src/main/webapp/js/nf/canvas/nf-port.js | 3 + .../nf/canvas/nf-process-group-configuration.js | 3 + .../js/nf/canvas/nf-process-group-details.js | 3 + .../webapp/js/nf/canvas/nf-process-group.js | 3 + .../js/nf/canvas/nf-processor-configuration.js | 460 ++--- .../nf-processor-property-combo-editor.js | 177 -- .../canvas/nf-processor-property-nfel-editor.js | 207 --- .../js/nf/canvas/nf-processor-property-table.js | 567 ------- .../canvas/nf-processor-property-text-editor.js | 212 --- .../main/webapp/js/nf/canvas/nf-processor.js | 3 + .../main/webapp/js/nf/canvas/nf-registration.js | 7 +- .../nf-remote-process-group-configuration.js | 6 + .../canvas/nf-remote-process-group-details.js | 3 + .../nf/canvas/nf-remote-process-group-ports.js | 3 + .../js/nf/canvas/nf-remote-process-group.js | 3 + .../webapp/js/nf/canvas/nf-reporting-task.js | 429 +++++ .../js/nf/canvas/nf-secure-port-details.js | 3 + .../main/webapp/js/nf/canvas/nf-selectable.js | 3 + .../src/main/webapp/js/nf/canvas/nf-settings.js | 1576 +++++++++++++++++- .../src/main/webapp/js/nf/canvas/nf-snippet.js | 3 + .../src/main/webapp/js/nf/canvas/nf-storage.js | 3 + .../webapp/js/nf/canvas/nf-toolbar-action.js | 3 + .../webapp/js/nf/cluster/nf-cluster-table.js | 11 +- .../src/main/webapp/js/nf/cluster/nf-cluster.js | 3 + .../webapp/js/nf/counters/nf-counters-table.js | 5 +- .../main/webapp/js/nf/counters/nf-counters.js | 3 + .../webapp/js/nf/history/nf-history-model.js | 3 + .../webapp/js/nf/history/nf-history-table.js | 7 +- .../src/main/webapp/js/nf/history/nf-history.js | 3 + .../src/main/webapp/js/nf/nf-client.js | 3 + .../src/main/webapp/js/nf/nf-common.js | 74 +- .../main/webapp/js/nf/nf-connection-details.js | 3 + .../src/main/webapp/js/nf/nf-dialog.js | 18 +- .../main/webapp/js/nf/nf-processor-details.js | 349 +--- .../src/main/webapp/js/nf/nf-shell.js | 3 + .../src/main/webapp/js/nf/nf-status-history.js | 3 + .../js/nf/provenance/nf-provenance-lineage.js | 3 + .../js/nf/provenance/nf-provenance-table.js | 7 +- .../webapp/js/nf/provenance/nf-provenance.js | 3 + .../webapp/js/nf/summary/nf-summary-table.js | 16 +- .../src/main/webapp/js/nf/summary/nf-summary.js | 3 + .../js/nf/templates/nf-templates-table.js | 9 +- .../main/webapp/js/nf/templates/nf-templates.js | 3 + .../main/webapp/js/nf/users/nf-users-table.js | 5 +- .../nifi-framework/pom.xml | 1 - .../nifi-framework-bundle/pom.xml | 1 - .../nifi-hadoop-bundle/nifi-hadoop-nar/pom.xml | 1 - .../src/main/resources/META-INF/NOTICE | 35 + .../hadoop/CreateHadoopSequenceFile.java | 4 +- .../nifi-nar-bundles/nifi-hadoop-bundle/pom.xml | 1 - .../nifi-hadoop-libraries-nar/pom.xml | 86 +- .../src/main/resources/META-INF/LICENSE | 359 ++++ .../src/main/resources/META-INF/NOTICE | 237 +++ .../nifi-hadoop-libraries-bundle/pom.xml | 1 - nifi/nifi-nar-bundles/nifi-jetty-bundle/pom.xml | 2 - .../src/main/resources/META-INF/NOTICE | 37 + .../nifi-kafka-bundle/nifi-kafka-nar/pom.xml | 1 - .../src/main/resources/META-INF/LICENSE | 299 ++++ .../src/main/resources/META-INF/NOTICE | 72 + nifi/nifi-nar-bundles/nifi-kafka-bundle/pom.xml | 2 +- .../nifi-kite-bundle/nifi-kite-nar/pom.xml | 47 + .../nifi-kite-processors/pom.xml | 148 ++ .../processors/kite/AbstractKiteProcessor.java | 217 +++ .../apache/nifi/processors/kite/AvroUtil.java | 43 + .../nifi/processors/kite/ConvertCSVToAvro.java | 258 +++ .../nifi/processors/kite/ConvertJSONToAvro.java | 157 ++ .../nifi/processors/kite/JSONFileReader.java | 114 ++ .../processors/kite/StoreInKiteDataset.java | 168 ++ .../data/spi/filesystem/CSVFileReaderFixed.java | 172 ++ .../org.apache.nifi.processor.Processor | 17 + .../processors/kite/TestCSVToAvroProcessor.java | 126 ++ .../kite/TestConfigurationProperty.java | 77 + .../nifi/processors/kite/TestGetSchema.java | 98 ++ .../kite/TestJSONToAvroProcessor.java | 61 + .../kite/TestKiteProcessorsCluster.java | 132 ++ .../kite/TestKiteStorageProcessor.java | 171 ++ .../apache/nifi/processors/kite/TestUtil.java | 103 ++ nifi/nifi-nar-bundles/nifi-kite-bundle/pom.xml | 59 + .../JournalingProvenanceRepository.java | 12 +- .../LazyInitializedProvenanceEvent.java | 12 +- .../journaling/index/EventIndexSearcher.java | 4 +- .../journaling/index/LuceneIndexSearcher.java | 54 +- .../journaling/index/MultiIndexSearcher.java | 5 +- .../provenance/journaling/index/QueryUtils.java | 4 + .../TestJournalingProvenanceRepository.java | 59 +- .../nifi-provenance-repository-nar/pom.xml | 1 - .../src/main/resources/META-INF/NOTICE | 202 +++ .../nifi-provenance-repository-bundle/pom.xml | 1 - .../nifi-standard-nar/pom.xml | 1 - .../src/main/resources/META-INF/LICENSE | 321 ++++ .../src/main/resources/META-INF/NOTICE | 127 ++ .../nifi-standard-processors/pom.xml | 13 + .../standard/AbstractJsonPathProcessor.java | 117 ++ .../standard/Base64EncodeContent.java | 13 +- .../processors/standard/CompressContent.java | 27 +- .../nifi/processors/standard/EncodeContent.java | 248 +++ .../processors/standard/EvaluateJsonPath.java | 278 +++ .../nifi/processors/standard/EvaluateXPath.java | 2 +- .../processors/standard/ExecuteProcess.java | 513 ++++++ .../apache/nifi/processors/standard/GetFTP.java | 5 + .../processors/standard/HandleHttpRequest.java | 604 +++++++ .../processors/standard/HandleHttpResponse.java | 178 ++ .../nifi/processors/standard/HashContent.java | 14 +- .../processors/standard/IdentifyMimeType.java | 405 +---- .../nifi/processors/standard/JmsConsumer.java | 143 +- .../nifi/processors/standard/PutEmail.java | 5 +- .../apache/nifi/processors/standard/PutFTP.java | 7 +- .../apache/nifi/processors/standard/PutJMS.java | 11 +- .../processors/standard/SegmentContent.java | 114 +- .../nifi/processors/standard/SplitJson.java | 184 ++ .../nifi/processors/standard/TransformXml.java | 3 +- .../processors/standard/util/FTPTransfer.java | 48 +- .../standard/util/JmsProcessingSummary.java | 83 + .../processors/standard/util/JmsProperties.java | 3 +- .../util/JsonPathExpressionValidator.java | 487 ++++++ .../standard/util/SocksProxySocketFactory.java | 69 + .../util/ValidatingBase32InputStream.java | 78 + .../org.apache.nifi.processor.Processor | 5 + .../index.html | 155 ++ .../index.html | 100 ++ .../index.html | 255 +++ .../index.html | 112 ++ .../index.html | 85 + .../org/apache/tika/mime/custom-mimetypes.xml | 83 + .../standard/TestCompressContent.java | 18 + .../standard/TestDetectDuplicate.java | 5 +- .../processors/standard/TestEncodeContent.java | 164 ++ .../standard/TestEvaluateJsonPath.java | 264 +++ .../processors/standard/TestExecuteProcess.java | 78 + .../standard/TestHandleHttpRequest.java | 128 ++ .../standard/TestHandleHttpResponse.java | 172 ++ .../standard/TestIdentifyMimeType.java | 80 +- .../processors/standard/TestJmsConsumer.java | 173 ++ .../nifi/processors/standard/TestPutEmail.java | 45 + .../nifi/processors/standard/TestSplitJson.java | 126 ++ .../test/resources/TestIdentifyMimeType/1.tar | Bin 2048 -> 10240 bytes .../resources/TestIdentifyMimeType/1.tar.gz | Bin 0 -> 154 bytes .../test/resources/TestIdentifyMimeType/1.txt | 1 + .../test/resources/TestJson/json-sample.json | 415 +++++ .../provenance/GenerateProvenanceReport.java | 78 +- .../org.apache.nifi.reporting.ReportingTask | 2 + .../nifi-standard-bundle/pom.xml | 1 - .../DistributedMapCacheClientService.java | 18 +- .../DistributedSetCacheClientService.java | 11 +- .../cache/server/DistributedCacheServer.java | 9 +- .../nifi-distributed-cache-services-nar/pom.xml | 1 - .../src/main/resources/META-INF/NOTICE | 19 + .../pom.xml | 1 - .../nifi-http-context-map-api/pom.xml | 23 + .../org/apache/nifi/http/HttpContextMap.java | 72 + .../nifi-http-context-map-nar/pom.xml | 37 + .../nifi-http-context-map/pom.xml | 44 + .../nifi/http/StandardHttpContextMap.java | 176 ++ ...org.apache.nifi.controller.ControllerService | 1 + .../index.html | 67 + .../nifi-http-context-map-bundle/pom.xml | 31 + .../nifi-load-distribution-service-api/pom.xml | 1 - .../nifi-ssl-context-nar/pom.xml | 1 - .../src/main/resources/META-INF/NOTICE | 19 + .../nifi/ssl/StandardSSLContextService.java | 39 +- .../nifi-ssl-context-bundle/pom.xml | 1 - .../nifi-standard-services-api-nar/pom.xml | 6 +- .../nifi-standard-services/pom.xml | 3 +- .../nifi-update-attribute-nar/pom.xml | 2 - .../src/main/resources/META-INF/NOTICE | 19 + .../src/main/resources/META-INF/LICENSE | 238 +++ .../src/main/resources/META-INF/NOTICE | 49 + .../nifi-update-attribute-bundle/pom.xml | 3 - nifi/nifi-nar-bundles/pom.xml | 15 +- nifi/pom.xml | 60 +- 580 files changed, 37719 insertions(+), 9681 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-api/src/main/java/org/apache/nifi/components/AbstractConfigurableComponent.java ---------------------------------------------------------------------- diff --cc nifi/nifi-api/src/main/java/org/apache/nifi/components/AbstractConfigurableComponent.java index e093785,f4bea5e..e2183f4 --- a/nifi/nifi-api/src/main/java/org/apache/nifi/components/AbstractConfigurableComponent.java +++ b/nifi/nifi-api/src/main/java/org/apache/nifi/components/AbstractConfigurableComponent.java @@@ -22,17 -22,8 +22,8 @@@ import java.util.Collections import java.util.List; import java.util.Map; - import org.apache.nifi.logging.ProcessorLog; - import org.apache.nifi.processor.ProcessorInitializationContext; - public abstract class AbstractConfigurableComponent implements ConfigurableComponent { - private ComponentLogger loger; - - @Override - public void initialize(final ProcessorInitializationContext context) { - logger = context.getLogger(); - } - + /** * Allows subclasses to perform their own validation on the already set * properties. Since each property is validated as it is set this allows http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-api/src/main/java/org/apache/nifi/reporting/AbstractReportingTask.java ---------------------------------------------------------------------- diff --cc nifi/nifi-api/src/main/java/org/apache/nifi/reporting/AbstractReportingTask.java index 5ed8f24,efcf2a3..7d41813 --- a/nifi/nifi-api/src/main/java/org/apache/nifi/reporting/AbstractReportingTask.java +++ b/nifi/nifi-api/src/main/java/org/apache/nifi/reporting/AbstractReportingTask.java @@@ -17,13 -17,14 +17,19 @@@ package org.apache.nifi.reporting; import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicBoolean; ++import org.apache.nifi.annotation.lifecycle.OnScheduled; ++import org.apache.nifi.annotation.lifecycle.OnUnscheduled; import org.apache.nifi.components.AbstractConfigurableComponent; import org.apache.nifi.controller.ControllerServiceLookup; + import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.processor.ProcessorInitializationContext; public abstract class AbstractReportingTask extends AbstractConfigurableComponent implements ReportingTask { ++ private final AtomicBoolean scheduled = new AtomicBoolean(false); ++ private String identifier; private String name; private long schedulingNanos; @@@ -39,6 -42,6 +47,20 @@@ init(config); } ++ @OnUnscheduled ++ public final void setUnscheduledFlag() { ++ scheduled.set(false); ++ } ++ ++ @OnScheduled ++ public final void setScheduledFlag() { ++ scheduled.set(true); ++ } ++ ++ protected final boolean isScheduled() { ++ return scheduled.get(); ++ } ++ /** * Returns the {@link ControllerServiceLookup} that was passed to the * {@link #init(ProcessorInitializationContext)} method http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java ---------------------------------------------------------------------- diff --cc nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java index 40ccbf5,0000000..4dda39b mode 100644,000000..100644 --- a/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java +++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java @@@ -1,623 -1,0 +1,670 @@@ +package org.apache.nifi.pql; + +import static org.apache.nifi.pql.ProvenanceQueryParser.AND; +import static org.apache.nifi.pql.ProvenanceQueryParser.ASC; +import static org.apache.nifi.pql.ProvenanceQueryParser.ATTRIBUTE; +import static org.apache.nifi.pql.ProvenanceQueryParser.AVG; +import static org.apache.nifi.pql.ProvenanceQueryParser.COMPONENT_ID; +import static org.apache.nifi.pql.ProvenanceQueryParser.COUNT; +import static org.apache.nifi.pql.ProvenanceQueryParser.DAY; +import static org.apache.nifi.pql.ProvenanceQueryParser.EQUALS; +import static org.apache.nifi.pql.ProvenanceQueryParser.EVENT; +import static org.apache.nifi.pql.ProvenanceQueryParser.EVENT_PROPERTY; +import static org.apache.nifi.pql.ProvenanceQueryParser.FILESIZE; +import static org.apache.nifi.pql.ProvenanceQueryParser.FROM; +import static org.apache.nifi.pql.ProvenanceQueryParser.GROUP_BY; +import static org.apache.nifi.pql.ProvenanceQueryParser.GT; +import static org.apache.nifi.pql.ProvenanceQueryParser.HOUR; +import static org.apache.nifi.pql.ProvenanceQueryParser.IDENTIFIER; +import static org.apache.nifi.pql.ProvenanceQueryParser.LIMIT; +import static org.apache.nifi.pql.ProvenanceQueryParser.LT; +import static org.apache.nifi.pql.ProvenanceQueryParser.MATCHES; +import static org.apache.nifi.pql.ProvenanceQueryParser.MINUTE; +import static org.apache.nifi.pql.ProvenanceQueryParser.MONTH; +import static org.apache.nifi.pql.ProvenanceQueryParser.NOT; +import static org.apache.nifi.pql.ProvenanceQueryParser.NOT_EQUALS; +import static org.apache.nifi.pql.ProvenanceQueryParser.NUMBER; +import static org.apache.nifi.pql.ProvenanceQueryParser.OR; +import static org.apache.nifi.pql.ProvenanceQueryParser.ORDER_BY; +import static org.apache.nifi.pql.ProvenanceQueryParser.RELATIONSHIP; +import static org.apache.nifi.pql.ProvenanceQueryParser.SECOND; +import static org.apache.nifi.pql.ProvenanceQueryParser.STARTS_WITH; +import static org.apache.nifi.pql.ProvenanceQueryParser.STRING_LITERAL; +import static org.apache.nifi.pql.ProvenanceQueryParser.SUM; +import static org.apache.nifi.pql.ProvenanceQueryParser.TIMESTAMP; +import static org.apache.nifi.pql.ProvenanceQueryParser.TRANSIT_URI; +import static org.apache.nifi.pql.ProvenanceQueryParser.TYPE; +import static org.apache.nifi.pql.ProvenanceQueryParser.UUID; +import static org.apache.nifi.pql.ProvenanceQueryParser.WHERE; +import static org.apache.nifi.pql.ProvenanceQueryParser.YEAR; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.tree.Tree; +import org.apache.nifi.pql.evaluation.Accumulator; +import org.apache.nifi.pql.evaluation.BooleanEvaluator; +import org.apache.nifi.pql.evaluation.OperandEvaluator; +import org.apache.nifi.pql.evaluation.RecordEvaluator; +import org.apache.nifi.pql.evaluation.RepositoryEvaluator; +import org.apache.nifi.pql.evaluation.accumulation.AverageAccumulator; +import org.apache.nifi.pql.evaluation.accumulation.CountAccumulator; +import org.apache.nifi.pql.evaluation.accumulation.EventAccumulator; +import org.apache.nifi.pql.evaluation.accumulation.SumAccumulator; +import org.apache.nifi.pql.evaluation.comparison.EqualsEvaluator; +import org.apache.nifi.pql.evaluation.comparison.GreaterThanEvaluator; +import org.apache.nifi.pql.evaluation.comparison.LessThanEvaluator; +import org.apache.nifi.pql.evaluation.comparison.MatchesEvaluator; +import org.apache.nifi.pql.evaluation.comparison.RecordTypeEvaluator; +import org.apache.nifi.pql.evaluation.comparison.StartsWithEvaluator; +import org.apache.nifi.pql.evaluation.conversion.StringToLongEvaluator; +import org.apache.nifi.pql.evaluation.extraction.AttributeEvaluator; +import org.apache.nifi.pql.evaluation.extraction.ComponentIdEvaluator; +import org.apache.nifi.pql.evaluation.extraction.RelationshipEvaluator; +import org.apache.nifi.pql.evaluation.extraction.SizeEvaluator; +import org.apache.nifi.pql.evaluation.extraction.TimestampEvaluator; +import org.apache.nifi.pql.evaluation.extraction.TransitUriEvaluator; +import org.apache.nifi.pql.evaluation.extraction.TypeEvaluator; +import org.apache.nifi.pql.evaluation.extraction.UuidEvaluator; +import org.apache.nifi.pql.evaluation.function.TimeFieldEvaluator; +import org.apache.nifi.pql.evaluation.literals.LongLiteralEvaluator; +import org.apache.nifi.pql.evaluation.literals.StringLiteralEvaluator; +import org.apache.nifi.pql.evaluation.logic.AndEvaluator; +import org.apache.nifi.pql.evaluation.logic.OrEvaluator; +import org.apache.nifi.pql.evaluation.order.FieldSorter; +import org.apache.nifi.pql.evaluation.order.GroupedSorter; +import org.apache.nifi.pql.evaluation.order.RowSorter; +import org.apache.nifi.pql.evaluation.order.SortDirection; +import org.apache.nifi.pql.evaluation.repository.SelectAllRecords; +import org.apache.nifi.pql.exception.ProvenanceQueryLanguageException; +import org.apache.nifi.pql.exception.ProvenanceQueryLanguageParsingException; +import org.apache.nifi.pql.results.GroupingResultSet; +import org.apache.nifi.pql.results.StandardOrderedResultSet; +import org.apache.nifi.pql.results.StandardUnorderedResultSet; +import org.apache.nifi.provenance.ProvenanceEventRepository; +import org.apache.nifi.provenance.ProvenanceEventType; +import org.apache.nifi.provenance.SearchableFields; +import org.apache.nifi.provenance.StoredProvenanceEvent; +import org.apache.nifi.provenance.query.ProvenanceResultSet; +import org.apache.nifi.provenance.search.SearchableField; + +public class ProvenanceQuery { + private final Tree tree; + private final String pql; + private final List<Accumulator<?>> selectAccumulators; + private final List<RecordEvaluator<?>> groupEvaluators; + private final RecordEvaluator<Boolean> sourceEvaluator; + private final RecordEvaluator<Boolean> conditionEvaluator; + private final RowSorter sorter; + private final Long limit; + + private final Set<SearchableField> searchableFields; + private final Set<String> searchableAttributes; + private long accumulatorIdGenerator = 0L; + ++ private final Set<String> referencedFields = new HashSet<>(); + + public static ProvenanceQuery compile(final String pql, final Collection<SearchableField> searchableFields, final Collection<SearchableField> searchableAttributes) { + try { + final CommonTokenStream lexerTokenStream = createTokenStream(pql); + final ProvenanceQueryParser parser = new ProvenanceQueryParser(lexerTokenStream); + final Tree ast = (Tree) parser.pql().getTree(); + final Tree tree = ast.getChild(0); + + return new ProvenanceQuery(tree, pql, searchableFields, searchableAttributes); + } catch (final ProvenanceQueryLanguageParsingException e) { + throw e; + } catch (final Exception e) { + throw new ProvenanceQueryLanguageParsingException(e); + } + } + + private static CommonTokenStream createTokenStream(final String expression) throws ProvenanceQueryLanguageParsingException { + final CharStream input = new ANTLRStringStream(expression); + final ProvenanceQueryLexer lexer = new ProvenanceQueryLexer(input); + return new CommonTokenStream(lexer); + } + + private ProvenanceQuery(final Tree tree, final String pql, final Collection<SearchableField> searchableFields, final Collection<SearchableField> searchableAttributes) { + this.tree = tree; + this.pql = pql; + this.searchableFields = searchableFields == null ? null : Collections.unmodifiableSet(new HashSet<>(searchableFields)); + if (searchableAttributes == null) { + this.searchableAttributes = null; + } else { + final Set<String> attributes = new HashSet<>(); + for ( final SearchableField attr : searchableAttributes ) { + attributes.add(attr.getSearchableFieldName()); + } + this.searchableAttributes = Collections.unmodifiableSet(attributes); + } + + Tree fromTree = null; + Tree whereTree = null; + Tree groupByTree = null; + Tree limitTree = null; + Tree orderByTree = null; + + for (int i=1; i < tree.getChildCount(); i++) { + final Tree subTree = tree.getChild(i); + switch (subTree.getType()) { + case FROM: + fromTree = subTree; + break; + case WHERE: + whereTree = subTree; + break; + case GROUP_BY: + groupByTree = subTree; + break; - case LIMIT: ++ case LIMIT: + limitTree = subTree; + break; + case ORDER_BY: + orderByTree = subTree; + break; + default: + // TODO: Handle other types! + continue; + } + } + + sourceEvaluator = (fromTree == null) ? null : buildSourceEvaluator(fromTree); + - final BooleanEvaluator where = (whereTree == null) ? null : buildConditionEvaluator(whereTree.getChild(0)); ++ final BooleanEvaluator where = (whereTree == null) ? null : buildConditionEvaluator(whereTree.getChild(0), Clause.WHERE); + conditionEvaluator = where; + + groupEvaluators = (groupByTree == null) ? null : buildGroupEvaluators(groupByTree); + limit = (limitTree == null) ? null : Long.parseLong(limitTree.getChild(0).getText()); + sorter = (orderByTree == null) ? null : buildSorter(orderByTree, groupByTree != null); + + boolean requiresAggregate = false; + if ( groupEvaluators != null && !groupEvaluators.isEmpty() ) { + requiresAggregate = true; + } + if ( requiresAggregate ) { + selectAccumulators = buildAccumulators(tree.getChild(0), true); + } else { + final List<Accumulator<?>> accumulators = buildAccumulators(tree.getChild(0), false); + + for ( final Accumulator<?> accumulator : accumulators ) { + if ( accumulator.isAggregateFunction() ) { + requiresAggregate = true; + break; + } + } + + if ( requiresAggregate ) { + selectAccumulators = buildAccumulators(tree.getChild(0), true); + } else { + selectAccumulators = accumulators; + } + } + } + ++ public Set<String> getReferencedFields() { ++ return Collections.unmodifiableSet(referencedFields); ++ } ++ + @Override + public String toString() { + return printTree(tree); + } + + public String getQuery() { + return pql; + + } + + private String printTree(final Tree tree) { + final StringBuilder sb = new StringBuilder(); + printTree(tree, 0, sb); + + return sb.toString(); + } + + private void printTree(final Tree tree, final int spaces, final StringBuilder sb) { + for (int i=0; i < spaces; i++) { + sb.append(" "); + } + + if ( tree.getText().trim().isEmpty() ) { + sb.append(tree.toString()).append("\n"); + } else { + sb.append(tree.getText()).append("\n"); + } + + for (int i=0; i < tree.getChildCount(); i++) { + printTree(tree.getChild(i), spaces + 2, sb); + } + } + + private List<Accumulator<?>> buildAccumulators(final Tree selectTree, final boolean distinct) { + final List<Accumulator<?>> accumulators = new ArrayList<>(); + + if ( selectTree.getType() != ProvenanceQueryParser.SELECT ) { + throw new IllegalArgumentException("Cannot build accumulators for a non-SELECT tree"); + } + + for (int i=0; i < selectTree.getChildCount(); i++) { + final Tree childTree = selectTree.getChild(i); + accumulators.add(buildAccumulator(childTree, distinct)); + } + + return accumulators; + } + + private Accumulator<?> buildAccumulator(final Tree tree, final boolean distinct) { + switch (tree.getType()) { + case SUM: - return new SumAccumulator(accumulatorIdGenerator++, toLongEvaluator(buildOperandEvaluator(tree.getChild(0)), tree), "SUM(" + getLabel(tree.getChild(0)) + ")"); ++ return new SumAccumulator(accumulatorIdGenerator++, toLongEvaluator(buildOperandEvaluator(tree.getChild(0), Clause.SELECT), tree), "SUM(" + getLabel(tree.getChild(0)) + ")"); + case AVG: - return new AverageAccumulator(accumulatorIdGenerator++, toLongEvaluator(buildOperandEvaluator(tree.getChild(0)), tree), "AVG(" + getLabel(tree.getChild(0)) + ")"); ++ return new AverageAccumulator(accumulatorIdGenerator++, toLongEvaluator(buildOperandEvaluator(tree.getChild(0), Clause.SELECT), tree), "AVG(" + getLabel(tree.getChild(0)) + ")"); + case EVENT: + return new EventAccumulator(accumulatorIdGenerator++, getLabel(tree), distinct); + case IDENTIFIER: + return new EventAccumulator(accumulatorIdGenerator++, getLabel(tree.getChild(0)), distinct); + case EVENT_PROPERTY: + case ATTRIBUTE: - return new EventAccumulator(accumulatorIdGenerator++, getLabel(tree.getChild(0)), buildOperandEvaluator(tree), distinct); ++ return new EventAccumulator(accumulatorIdGenerator++, getLabel(tree.getChild(0)), buildOperandEvaluator(tree, Clause.SELECT), distinct); + case YEAR: + case DAY: + case HOUR: + case MINUTE: + case SECOND: - return new EventAccumulator(accumulatorIdGenerator++, getLabel(tree), buildOperandEvaluator(tree), distinct); ++ return new EventAccumulator(accumulatorIdGenerator++, getLabel(tree), buildOperandEvaluator(tree, Clause.SELECT), distinct); + case COUNT: + if ( "Event".equalsIgnoreCase(tree.getChild(0).getText() ) ) { + return new CountAccumulator(accumulatorIdGenerator++, null, "COUNT(" + getLabel(tree.getChild(0)) + ")"); + } - return new CountAccumulator(accumulatorIdGenerator++, buildOperandEvaluator(tree.getChild(0)), "COUNT(" + getLabel(tree.getChild(0)) + ")"); ++ return new CountAccumulator(accumulatorIdGenerator++, buildOperandEvaluator(tree.getChild(0), Clause.SELECT), "COUNT(" + getLabel(tree.getChild(0)) + ")"); + default: + throw new UnsupportedOperationException("Haven't implemented accumulators yet for " + tree); + } + } + + private String getLabel(final Tree tree) { + final int type = tree.getType(); + + switch (type) { + case EVENT_PROPERTY: + case ATTRIBUTE: + return tree.getChild(0).getText(); + case YEAR: + case DAY: + case HOUR: + case MINUTE: + case SECOND: + return tree.getText() + "(" + getLabel(tree.getChild(0)) + ")"; + } + + return tree.getText(); + } + - private OperandEvaluator<?> buildOperandEvaluator(final Tree tree) { ++ private OperandEvaluator<?> buildOperandEvaluator(final Tree tree, final Clause clause) { ++ // When events are pulled back from an index, for efficiency purposes, we may want to know which ++ // fields to pull back. The fields in the WHERE clause are irrelevant because they are not shown ++ // to the user, so no need to pull those back. ++ final boolean isReferenceInteresting = clause != Clause.WHERE; ++ + switch (tree.getType()) { + case EVENT_PROPERTY: + switch (tree.getChild(0).getType()) { + case FILESIZE: + if ( searchableFields != null && !searchableFields.contains(SearchableFields.FileSize) ) { + throw new ProvenanceQueryLanguageException("Query cannot reference FileSize because this field is not searchable by the repository"); + } ++ if ( isReferenceInteresting ) { ++ referencedFields.add(SearchableFields.FileSize.getSearchableFieldName()); ++ } + return new SizeEvaluator(); + case TRANSIT_URI: + if ( searchableFields != null && !searchableFields.contains(SearchableFields.TransitURI) ) { + throw new ProvenanceQueryLanguageException("Query cannot reference TransitURI because this field is not searchable by the repository"); + } ++ if ( isReferenceInteresting ) { ++ referencedFields.add(SearchableFields.TransitURI.getSearchableFieldName()); ++ } + return new TransitUriEvaluator(); + case TIMESTAMP: + // time is always indexed ++ if ( isReferenceInteresting ) { ++ referencedFields.add(SearchableFields.EventTime.getSearchableFieldName()); ++ } + return new TimestampEvaluator(); + case TYPE: + // type is always indexed so no need to check it ++ if ( isReferenceInteresting ) { ++ referencedFields.add(SearchableFields.EventType.getSearchableFieldName()); ++ } + return new TypeEvaluator(); + case COMPONENT_ID: + if ( searchableFields != null && !searchableFields.contains(SearchableFields.ComponentID) ) { + throw new ProvenanceQueryLanguageException("Query cannot reference Component ID because this field is not searchable by the repository"); + } ++ if ( isReferenceInteresting ) { ++ referencedFields.add(SearchableFields.ComponentID.getSearchableFieldName()); ++ } ++ + return new ComponentIdEvaluator(); + // TODO: Allow Component Type to be indexed and searched + case RELATIONSHIP: + if ( searchableFields != null && !searchableFields.contains(SearchableFields.Relationship) ) { + throw new ProvenanceQueryLanguageException("Query cannot reference Relationship because this field is not searchable by the repository"); + } - return new RelationshipEvaluator(); ++ if ( isReferenceInteresting ) { ++ referencedFields.add(SearchableFields.Relationship.getSearchableFieldName()); ++ } ++ ++ return new RelationshipEvaluator(); + case UUID: + if ( searchableFields != null && !searchableFields.contains(SearchableFields.FlowFileUUID) ) { + throw new ProvenanceQueryLanguageException("Query cannot reference FlowFile UUID because this field is not searchable by the repository"); + } - return new UuidEvaluator(); ++ if ( isReferenceInteresting ) { ++ referencedFields.add(SearchableFields.FlowFileUUID.getSearchableFieldName()); ++ } ++ ++ return new UuidEvaluator(); + default: + // TODO: IMPLEMENT + throw new UnsupportedOperationException("Haven't implemented extraction of property " + tree.getChild(0).getText()); + } + case ATTRIBUTE: + final String attributeName = tree.getChild(0).getText(); + if ( searchableAttributes != null && !searchableAttributes.contains(attributeName) ) { + throw new ProvenanceQueryLanguageException("Query cannot attribute '" + attributeName + "' because this attribute is not searchable by the repository"); + } - return new AttributeEvaluator(toStringEvaluator(buildOperandEvaluator(tree.getChild(0)), tree)); ++ ++ if ( isReferenceInteresting ) { ++ referencedFields.add(attributeName); ++ } ++ ++ return new AttributeEvaluator(toStringEvaluator(buildOperandEvaluator(tree.getChild(0), clause), tree)); + case STRING_LITERAL: + return new StringLiteralEvaluator(tree.getText()); + case NUMBER: + return new LongLiteralEvaluator(Long.valueOf(tree.getText())); + case YEAR: - return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0)), tree), Calendar.YEAR, YEAR); ++ return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0), clause), tree), Calendar.YEAR, YEAR); + case MONTH: - return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0)), tree), Calendar.MONTH, MONTH); ++ return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0), clause), tree), Calendar.MONTH, MONTH); + case DAY: - return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0)), tree), Calendar.DAY_OF_YEAR, DAY); ++ return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0), clause), tree), Calendar.DAY_OF_YEAR, DAY); + case HOUR: - return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0)), tree), Calendar.HOUR_OF_DAY, HOUR); ++ return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0), clause), tree), Calendar.HOUR_OF_DAY, HOUR); + case MINUTE: - return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0)), tree), Calendar.MINUTE, MINUTE); ++ return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0), clause), tree), Calendar.MINUTE, MINUTE); + case SECOND: - return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0)), tree), Calendar.SECOND, SECOND); ++ return new TimeFieldEvaluator(toLongEvaluator(buildOperandEvaluator(tree.getChild(0), clause), tree), Calendar.SECOND, SECOND); + default: + throw new ProvenanceQueryLanguageParsingException("Unable to extract value '" + tree.toString() + "' from event because it is not a valid "); + } + } + + + private RecordEvaluator<Boolean> buildSourceEvaluator(final Tree fromTree) { + if ( fromTree == null ) { + throw new NullPointerException(); + } + if ( fromTree.getType() != FROM ) { + throw new IllegalArgumentException("Cannot build Soruce Evaluator from a Tree that is not a FROM-tree"); + } + + final Set<ProvenanceEventType> types = new HashSet<>(); + for ( int i=0; i < fromTree.getChildCount(); i++ ) { + final Tree typeTree = fromTree.getChild(i); + if ( "*".equals(typeTree.getText()) ) { + return null; + } else { + types.add(ProvenanceEventType.valueOf(typeTree.getText().toUpperCase())); + } + } + + return new RecordTypeEvaluator(types); + } + + - private BooleanEvaluator buildConditionEvaluator(final Tree tree) { ++ private BooleanEvaluator buildConditionEvaluator(final Tree tree, final Clause clause) { + switch (tree.getType()) { + case AND: - return new AndEvaluator(buildConditionEvaluator(tree.getChild(0)), buildConditionEvaluator(tree.getChild(1))); ++ return new AndEvaluator(buildConditionEvaluator(tree.getChild(0), clause), buildConditionEvaluator(tree.getChild(1), clause)); + case OR: - return new OrEvaluator(buildConditionEvaluator(tree.getChild(0)), buildConditionEvaluator(tree.getChild(1))); ++ return new OrEvaluator(buildConditionEvaluator(tree.getChild(0), clause), buildConditionEvaluator(tree.getChild(1), clause)); + case EQUALS: - return new EqualsEvaluator(buildOperandEvaluator(tree.getChild(0)), buildOperandEvaluator(tree.getChild(1))); ++ return new EqualsEvaluator(buildOperandEvaluator(tree.getChild(0), clause), buildOperandEvaluator(tree.getChild(1), clause)); + case NOT_EQUALS: - return new EqualsEvaluator(buildOperandEvaluator(tree.getChild(0)), buildOperandEvaluator(tree.getChild(1)), true); ++ return new EqualsEvaluator(buildOperandEvaluator(tree.getChild(0), clause), buildOperandEvaluator(tree.getChild(1), clause), true); + case GT: - return new GreaterThanEvaluator(buildOperandEvaluator(tree.getChild(0)), buildOperandEvaluator(tree.getChild(1))); ++ return new GreaterThanEvaluator(buildOperandEvaluator(tree.getChild(0), clause), buildOperandEvaluator(tree.getChild(1), clause)); + case LT: - return new LessThanEvaluator(buildOperandEvaluator(tree.getChild(0)), buildOperandEvaluator(tree.getChild(1))); ++ return new LessThanEvaluator(buildOperandEvaluator(tree.getChild(0), clause), buildOperandEvaluator(tree.getChild(1), clause)); + case NOT: - return buildConditionEvaluator(tree.getChild(0)).negate(); ++ return buildConditionEvaluator(tree.getChild(0), clause).negate(); + case MATCHES: { - final OperandEvaluator<?> rhs = buildOperandEvaluator(tree.getChild(1)); ++ final OperandEvaluator<?> rhs = buildOperandEvaluator(tree.getChild(1), clause); + if ( !String.class.equals( rhs.getType() ) ) { + throw new ProvenanceQueryLanguageParsingException("Right-hand side of MATCHES operator must be a Regular Expression but found " + rhs); + } - return new MatchesEvaluator(buildOperandEvaluator(tree.getChild(0)), rhs); ++ return new MatchesEvaluator(buildOperandEvaluator(tree.getChild(0), clause), rhs); + } + case STARTS_WITH: { - final OperandEvaluator<?> rhs = buildOperandEvaluator(tree.getChild(1)); ++ final OperandEvaluator<?> rhs = buildOperandEvaluator(tree.getChild(1), clause); + if ( !String.class.equals( rhs.getType() ) ) { + throw new ProvenanceQueryLanguageParsingException("Right-hand side of STARTS WITH operator must be a String but found " + rhs); + } - return new StartsWithEvaluator(buildOperandEvaluator(tree.getChild(0)), rhs); ++ return new StartsWithEvaluator(buildOperandEvaluator(tree.getChild(0), clause), rhs); + } + default: + // TODO: Implement + throw new UnsupportedOperationException("Have not yet implemented condition evaluator for " + tree); + } + } + + + private <T> OperandEvaluator<T> castEvaluator(final OperandEvaluator<?> eval, final Tree tree, final Class<T> expectedType) { + if ( eval.getType() != expectedType ) { + throw new ProvenanceQueryLanguageParsingException("Expected type " + expectedType.getSimpleName() + " but found type " + eval.getType() + " for term: " + tree); + } + + @SuppressWarnings("unchecked") + final OperandEvaluator<T> retEvaluator = ((OperandEvaluator<T>) eval); + return retEvaluator; + + } + + private OperandEvaluator<String> toStringEvaluator(final OperandEvaluator<?> eval, final Tree tree) { + return castEvaluator(eval, tree, String.class); + } + + private OperandEvaluator<Long> toLongEvaluator(final OperandEvaluator<?> eval, final Tree tree) { + if ( eval.getType() == Long.class ) { + @SuppressWarnings("unchecked") + final OperandEvaluator<Long> retEvaluator = ((OperandEvaluator<Long>) eval); + return retEvaluator; + } else if ( eval.getType() == String.class ) { + @SuppressWarnings("unchecked") + final OperandEvaluator<String> stringEval = ((OperandEvaluator<String>) eval); + return new StringToLongEvaluator(stringEval); + } + + return castEvaluator(eval, tree, Long.class); + } + + + private List<RecordEvaluator<?>> buildGroupEvaluators(final Tree groupByTree) { + if ( groupByTree == null ) { + return null; + } + + if ( groupByTree.getType() != GROUP_BY ) { + throw new IllegalArgumentException("Expected GroupBy tree but got " + groupByTree); + } + + final List<RecordEvaluator<?>> evaluators = new ArrayList<>(groupByTree.getChildCount()); + for (int i=0; i < groupByTree.getChildCount(); i++) { + final Tree tree = groupByTree.getChild(i); + final RecordEvaluator<?> evaluator; + + switch (tree.getType()) { + case EVENT_PROPERTY: + case STRING_LITERAL: + case ATTRIBUTE: + case YEAR: + case DAY: + case HOUR: + case MINUTE: + case SECOND: - evaluator = buildOperandEvaluator(tree); ++ evaluator = buildOperandEvaluator(tree, Clause.GROUP); + break; + default: - evaluator = buildConditionEvaluator(tree); ++ evaluator = buildConditionEvaluator(tree, Clause.GROUP); + break; + } + + evaluators.add(evaluator); + } + + return evaluators; + } + + private RowSorter buildSorter(final Tree orderByTree, final boolean grouped) { + if ( orderByTree.getType() != ORDER_BY ) { + throw new IllegalArgumentException(); + } + + if ( grouped ) { + final Map<Accumulator<?>, SortDirection> accumulators = new LinkedHashMap<>(orderByTree.getChildCount()); + for (int i=0; i < orderByTree.getChildCount(); i++) { + final Tree orderTree = orderByTree.getChild(i); + final Accumulator<?> accumulator = buildAccumulator(orderTree.getChild(0), true); + + final SortDirection sortDir; + if ( orderTree.getChildCount() > 1 ) { + final int sortDirType = orderTree.getChild(1).getType(); + sortDir = (sortDirType == ASC) ? SortDirection.ASC : SortDirection.DESC; + } else { + sortDir = SortDirection.ASC; + } + + accumulators.put(accumulator, sortDir); + } + + return new GroupedSorter(accumulators); + } else { + // TODO: Allow ORDER BY of aggregate values + final Map<OperandEvaluator<?>, SortDirection> evaluators = new LinkedHashMap<>(orderByTree.getChildCount()); + for (int i=0; i < orderByTree.getChildCount(); i++) { + final Tree orderTree = orderByTree.getChild(i); - final OperandEvaluator<?> evaluator = buildOperandEvaluator(orderTree.getChild(0)); ++ final OperandEvaluator<?> evaluator = buildOperandEvaluator(orderTree.getChild(0), Clause.ORDER); + + final SortDirection sortDir; + if ( orderTree.getChildCount() > 1 ) { + final int sortDirType = orderTree.getChild(1).getType(); + sortDir = (sortDirType == ASC) ? SortDirection.ASC : SortDirection.DESC; + } else { + sortDir = SortDirection.ASC; + } + + evaluators.put(evaluator, sortDir); + } + + return new FieldSorter(evaluators); + } + } + + + public static ProvenanceResultSet execute(final String query, final ProvenanceEventRepository repo) throws IOException { + return ProvenanceQuery.compile(query, null, null).execute(repo); + } + + + public ProvenanceResultSet evaluate(final Iterator<? extends StoredProvenanceEvent> matchedEvents) { + final List<String> labels = new ArrayList<>(); + final List<Class<?>> returnTypes = new ArrayList<>(selectAccumulators.size()); + + for ( final Accumulator<?> accumulator : selectAccumulators ) { + labels.add(accumulator.getLabel()); + returnTypes.add(accumulator.getReturnType()); + } + + ProvenanceResultSet rs; + if ( isAggregateRequired() ) { + rs = new GroupingResultSet(matchedEvents, + selectAccumulators, sourceEvaluator, conditionEvaluator, + labels, returnTypes, groupEvaluators, sorter, limit); + } else if (sorter == null) { + rs = new StandardUnorderedResultSet(matchedEvents, selectAccumulators, sourceEvaluator, conditionEvaluator, labels, returnTypes, limit); + } else { + rs = new StandardOrderedResultSet(matchedEvents, selectAccumulators, sourceEvaluator, conditionEvaluator, labels, returnTypes, sorter, limit); + } + + return rs; + } + + public ProvenanceResultSet execute(final ProvenanceEventRepository repo) throws IOException { + final RepositoryEvaluator repoEvaluator = new SelectAllRecords(); + + final Iterator<StoredProvenanceEvent> potentialMatches = repoEvaluator.evaluate(repo); + final List<String> labels = new ArrayList<>(); + final List<Class<?>> returnTypes = new ArrayList<>(selectAccumulators.size()); + + for ( final Accumulator<?> accumulator : selectAccumulators ) { + labels.add(accumulator.getLabel()); + returnTypes.add(accumulator.getReturnType()); + } + + ProvenanceResultSet rs; + if ( isAggregateRequired() ) { + rs = new GroupingResultSet(potentialMatches, + selectAccumulators, sourceEvaluator, conditionEvaluator, + labels, returnTypes, groupEvaluators, sorter, limit); + } else if (sorter == null) { + rs = new StandardUnorderedResultSet(potentialMatches, selectAccumulators, sourceEvaluator, conditionEvaluator, labels, returnTypes, limit); + } else { + rs = new StandardOrderedResultSet(potentialMatches, selectAccumulators, sourceEvaluator, conditionEvaluator, labels, returnTypes, sorter, limit); + } + + return rs; + } + + private boolean isAggregateRequired() { + if ( groupEvaluators != null && !groupEvaluators.isEmpty() ) { + return true; + } + + for ( final Accumulator<?> accumulator : selectAccumulators ) { + if ( accumulator.isAggregateFunction() ) { + return true; + } + } + + + return false; + } + + + public RecordEvaluator<Boolean> getWhereClause() { + return conditionEvaluator; + } ++ ++ private static enum Clause { ++ SELECT, ++ FROM, ++ WHERE, ++ GROUP, ++ ORDER; ++ } +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-commons/pom.xml ---------------------------------------------------------------------- diff --cc nifi/nifi-commons/pom.xml index 35d64fe,768dfd9..2549649 --- a/nifi/nifi-commons/pom.xml +++ b/nifi/nifi-commons/pom.xml @@@ -35,6 -35,6 +35,7 @@@ <module>nifi-web-utils</module> <module>nifi-processor-utilities</module> <module>nifi-write-ahead-log</module> + <module>nifi-provenance-query-language</module> + <module>nifi-site-to-site-client</module> </modules> </project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceEventRepository.java ---------------------------------------------------------------------- diff --cc nifi/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceEventRepository.java index c4caa71,241041a..cdff42d --- a/nifi/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceEventRepository.java +++ b/nifi/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceEventRepository.java @@@ -25,6 -24,6 +25,7 @@@ import java.util.concurrent.atomic.Atom import org.apache.nifi.events.EventReporter; import org.apache.nifi.provenance.lineage.ComputeLineageSubmission; ++import org.apache.nifi.provenance.query.ProvenanceQuerySubmission; import org.apache.nifi.provenance.search.Query; import org.apache.nifi.provenance.search.QuerySubmission; import org.apache.nifi.provenance.search.SearchableField; @@@ -129,34 -128,4 +130,44 @@@ public class MockProvenanceEventReposit public ProvenanceEventBuilder eventBuilder() { return new StandardProvenanceEventRecord.Builder(); } + + @Override + public Long getEarliestEventTime() throws IOException { + final StoredProvenanceEvent event = getEvent(0); + if ( event == null ) { + return null; + } + + return event.getEventTime(); + } + + @Override + public StoredProvenanceEvent getEvent(final StorageLocation location) throws IOException { + if ( location instanceof EventIdLocation ) { + return getEvent( ((EventIdLocation) location).getId() ); + } + throw new IllegalArgumentException("Invalid StorageLocation"); + } + + @Override + public List<StoredProvenanceEvent> getEvents(final List<StorageLocation> storageLocations) throws IOException { + final List<StoredProvenanceEvent> events = new ArrayList<>(storageLocations.size()); + for ( final StorageLocation location : storageLocations ) { + final StoredProvenanceEvent event = getEvent(location); + if ( event != null ) { + events.add(event); + } + } + return events; + } ++ ++ @Override ++ public ProvenanceQuerySubmission submitQuery(final String query) { ++ return null; ++ } ++ ++ @Override ++ public ProvenanceQuerySubmission retrieveProvenanceQuerySubmission(final String queryIdentifier) { ++ return null; ++ } } http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/.gitignore ---------------------------------------------------------------------- diff --cc nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/.gitignore index cd1a4e7,cd1a4e7..9afaccf --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/.gitignore +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/.gitignore @@@ -4,3 -4,3 +4,4 @@@ /target /target /target ++/target/ http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/ClusteredEventAccess.java ---------------------------------------------------------------------- diff --cc nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/ClusteredEventAccess.java index 7780d04,2015530..398ba29 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/ClusteredEventAccess.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/ClusteredEventAccess.java @@@ -27,9 -25,7 +27,10 @@@ import org.apache.nifi.events.EventRepo import org.apache.nifi.provenance.ProvenanceEventBuilder; import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.provenance.ProvenanceEventRepository; +import org.apache.nifi.provenance.StorageLocation; +import org.apache.nifi.provenance.StoredProvenanceEvent; import org.apache.nifi.provenance.lineage.ComputeLineageSubmission; ++import org.apache.nifi.provenance.query.ProvenanceQuerySubmission; import org.apache.nifi.provenance.search.Query; import org.apache.nifi.provenance.search.QuerySubmission; import org.apache.nifi.provenance.search.SearchableField; @@@ -130,26 -130,6 +131,36 @@@ public class ClusteredEventAccess imple public void initialize(EventReporter eventReporter) throws IOException { } + + @Override + public Long getEarliestEventTime() throws IOException { + return null; + } + + @Override + public StoredProvenanceEvent getEvent(final StorageLocation location) throws IOException { + return null; + } + + @Override + public List<StoredProvenanceEvent> getEvents(final List<StorageLocation> storageLocations) throws IOException { + return Collections.emptyList(); + } + + @Override + public void registerEvents(final Collection<ProvenanceEventRecord> events) throws IOException { + + } ++ ++ @Override ++ public ProvenanceQuerySubmission submitQuery(String query) { ++ return null; ++ } ++ ++ @Override ++ public ProvenanceQuerySubmission retrieveProvenanceQuerySubmission(String queryIdentifier) { ++ return null; ++ } }; } } http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/.gitignore ---------------------------------------------------------------------- diff --cc nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/.gitignore index ea8c4bf,29546b5..d2d9c31 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/.gitignore +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/.gitignore @@@ -1,1 -1,2 +1,3 @@@ /target + /target/ ++/target/ http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/79c60016/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java ---------------------------------------------------------------------- diff --cc nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 54f0807,112e171..d1e24c0 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@@ -77,6 -78,7 +78,8 @@@ import org.apache.nifi.controller.excep import org.apache.nifi.controller.label.Label; import org.apache.nifi.controller.label.StandardLabel; import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException; + import org.apache.nifi.controller.reporting.ReportingTaskProvider; ++import org.apache.nifi.controller.reporting.StandardReportingInitializationContext; import org.apache.nifi.controller.reporting.StandardReportingTaskNode; import org.apache.nifi.controller.repository.ContentRepository; import org.apache.nifi.controller.repository.CounterRepository; @@@ -129,6 -131,6 +132,7 @@@ import org.apache.nifi.groups.ProcessGr import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.groups.RemoteProcessGroupPortDescriptor; import org.apache.nifi.groups.StandardProcessGroup; ++import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.logging.LogLevel; import org.apache.nifi.logging.LogRepository; import org.apache.nifi.logging.LogRepositoryFactory; @@@ -161,6 -163,6 +165,8 @@@ import org.apache.nifi.remote.protocol. import org.apache.nifi.reporting.Bulletin; import org.apache.nifi.reporting.BulletinRepository; import org.apache.nifi.reporting.EventAccess; ++import org.apache.nifi.reporting.InitializationException; ++import org.apache.nifi.reporting.ReportingInitializationContext; import org.apache.nifi.reporting.ReportingTask; import org.apache.nifi.reporting.Severity; import org.apache.nifi.scheduling.SchedulingStrategy; @@@ -2499,8 -2504,9 +2508,19 @@@ public class FlowController implements final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(controllerServiceProvider); final ReportingTaskNode taskNode = new StandardReportingTaskNode(task, id, this, processScheduler, validationContextFactory); + taskNode.setName(task.getClass().getSimpleName()); if ( firstTimeAdded ) { ++ final ComponentLog componentLog = new SimpleProcessLogger(id, taskNode.getReportingTask()); ++ final ReportingInitializationContext config = new StandardReportingInitializationContext(id, taskNode.getName(), ++ SchedulingStrategy.TIMER_DRIVEN, "1 min", componentLog, this); ++ ++ try { ++ task.initialize(config); ++ } catch (final InitializationException ie) { ++ throw new ReportingTaskInstantiationException("Failed to initialize reporting task of type " + type, ie); ++ } ++ try (final NarCloseable x = NarCloseable.withNarLoader()) { ReflectionUtils.invokeMethodsWithAnnotation(OnAdded.class, task); } catch (final Exception e) {
