MINIFI-23 Providing base implementation of a non-indexed, persistent provenance repository using NiFi interfaces and package structure.
Exposing rollover time as configurable property for provenance repository. This closes #18. Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi/commit/fb554819 Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi/tree/fb554819 Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi/diff/fb554819 Branch: refs/heads/master Commit: fb55481982f36bd74bc84e189a535d95b7b024aa Parents: 66dbda9 Author: Aldrin Piri <ald...@apache.org> Authored: Thu Apr 21 11:27:06 2016 -0400 Committer: Aldrin Piri <ald...@apache.org> Committed: Tue May 3 00:08:43 2016 -0400 ---------------------------------------------------------------------- minifi-api/pom.xml | 6 + .../provenance/ProvenanceEventRepository.java | 104 ++ minifi-assembly/pom.xml | 11 +- .../bootstrap/util/ConfigTransformer.java | 7 +- minifi-bootstrap/src/test/resources/config.yml | 3 + minifi-bootstrap/src/test/resources/default.yml | 3 + minifi-commons/minifi-utils/pom.xml | 28 + minifi-commons/pom.xml | 6 +- minifi-docs/Properties_Guide.md | 6 + .../minifi-framework-nar/pom.xml | 1 - .../src/main/resources/conf/config.yml | 3 + .../minifi-provenance-reporting-nar/pom.xml | 1 - .../minifi-provenance-reporting-bundle/pom.xml | 7 +- .../pom.xml | 57 + .../MiNiFiPersistentProvenanceRepository.java | 1640 ++++++++++++++++++ ...he.nifi.provenance.ProvenanceEventRepository | 15 + ...estMiNiFiPersistentProvenanceRepository.java | 691 ++++++++ .../org/apache/nifi/provenance/TestUtil.java | 82 + .../minifi-provenance-repository-nar/pom.xml | 42 + .../src/main/resources/META-INF/NOTICE | 202 +++ .../minifi-provenance-repository-bundle/pom.xml | 37 + minifi-nar-bundles/pom.xml | 1 + pom.xml | 24 + 23 files changed, 2967 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-api/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-api/pom.xml b/minifi-api/pom.xml index 660f357..01f4ad8 100644 --- a/minifi-api/pom.xml +++ b/minifi-api/pom.xml @@ -27,4 +27,10 @@ limitations under the License. <artifactId>minifi-api</artifactId> <packaging>jar</packaging> + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + </dependencies> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-api/src/main/java/org/apache/nifi/minifi/provenance/ProvenanceEventRepository.java ---------------------------------------------------------------------- diff --git a/minifi-api/src/main/java/org/apache/nifi/minifi/provenance/ProvenanceEventRepository.java b/minifi-api/src/main/java/org/apache/nifi/minifi/provenance/ProvenanceEventRepository.java new file mode 100644 index 0000000..fc955b4 --- /dev/null +++ b/minifi-api/src/main/java/org/apache/nifi/minifi/provenance/ProvenanceEventRepository.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.provenance; + +import java.io.IOException; +import java.util.List; + +import org.apache.nifi.events.EventReporter; +import org.apache.nifi.provenance.ProvenanceEventBuilder; +import org.apache.nifi.provenance.ProvenanceEventRecord; + +/** + * This Repository houses Provenance Events. The repository is responsible for + * managing the life-cycle of the events, providing access to the events that it + * has stored, and providing query capabilities against the events. + * + */ +public interface ProvenanceEventRepository { + + /** + * Performs any initialization needed. This should be called only by the + * framework. + * + * @param eventReporter to report to + * @throws IOException if unable to initialize + */ + void initialize(EventReporter eventReporter) throws IOException; + + /** + * Returns a {@link org.apache.nifi.provenance.ProvenanceEventBuilder} that is capable of building + * {@link ProvenanceEventRecord}s + * + * @return builder + */ + ProvenanceEventBuilder eventBuilder(); + + /** + * Adds the given event to the repository and returns a new event for which + * the event id has been populated. Depending on the implementation, the + * returned event may or may not be the same event given + * + * @param event to register + */ + void registerEvent(ProvenanceEventRecord event); + + /** + * Adds the given events to the repository. + * + * <p> + * This interface makes no assumptions about whether or not the registration + * of the Collection are atomic. This detail is implementation-specific. + * </p> + * + * @param events to register + */ + void registerEvents(Iterable<ProvenanceEventRecord> events); + + /** + * Returns a List of all <code>ProvenanceEventRecord</code>s in the + * repository starting with the given ID. The first ID in the repository + * will always be 0 or higher. + * + * @param firstRecordId id of the first record to retrieve + * @param maxRecords maximum number of records to retrieve + * @return records + * @throws IOException if error reading from repository + */ + List<ProvenanceEventRecord> getEvents(long firstRecordId, final int maxRecords) throws IOException; + + /** + * @return the largest ID of any event that is queryable in the repository. + * If no queryable events exists, returns null + */ + Long getMaxEventId(); + + /** + * @param id to lookup + * @return the Provenance Event Record with the given ID, if it exists, or + * {@code null} otherwise + * @throws IOException if failure while retrieving event + */ + ProvenanceEventRecord getEvent(long id) throws IOException; + + /** + * Closes the repository, freeing any resources + * + * @throws IOException if failure closing repository + */ + void close() throws IOException; +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 7c5266d..fa2dab3 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -93,22 +93,23 @@ limitations under the License. <classifier>resources</classifier> <scope>runtime</scope> <type>zip</type> - <version>0.0.1-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.nifi.minifi</groupId> <artifactId>minifi-bootstrap</artifactId> - <version>0.0.1-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.nifi.minifi</groupId> <artifactId>minifi-runtime</artifactId> - <version>0.0.1-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.nifi.minifi</groupId> <artifactId>minifi-provenance-reporting-nar</artifactId> - <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi.minifi</groupId> + <artifactId>minifi-provenance-repository-nar</artifactId> <type>nar</type> </dependency> @@ -234,7 +235,7 @@ limitations under the License. <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port> <!-- persistent provenance repository properties --> - <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation> + <nifi.provenance.repository.implementation>org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository</nifi.provenance.repository.implementation> <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default> <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time> <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 633cce2..e7383ad 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -83,6 +83,9 @@ public final class ConfigTransformer { public static final String OUT_THREADS_KEY = "out threads"; + public static final String PROVENANCE_REPO_KEY = "Provenance Repository"; + public static final String PROVENANCE_REPO_ROLLOVER_TIME_KEY = "provenance rollover time"; + public static final String CONTENT_REPO_KEY = "Content Repository"; public static final String CONTENT_CLAIM_MAX_APPENDABLE_SIZE_KEY = "content claim max appendable size"; public static final String CONTENT_CLAIM_MAX_FLOW_FILES_KEY = "content claim max flow files"; @@ -216,6 +219,7 @@ public final class ConfigTransformer { Map<String, Object> flowfileRepo = (Map<String, Object>) topLevelYaml.get(FLOWFILE_REPO_KEY); Map<String, Object> swapProperties = (Map<String, Object>) flowfileRepo.get(SWAP_PROPS_KEY); Map<String, Object> contentRepo = (Map<String, Object>) topLevelYaml.get(CONTENT_REPO_KEY); + Map<String, Object> provenanceRepo = (Map<String, Object>) topLevelYaml.get(PROVENANCE_REPO_KEY); Map<String, Object> componentStatusRepo = (Map<String, Object>) topLevelYaml.get(COMPONENT_STATUS_REPO_KEY); Map<String, Object> securityProperties = (Map<String, Object>) topLevelYaml.get(SECURITY_PROPS_KEY); Map<String, Object> sensitiveProperties = (Map<String, Object>) securityProperties.get(SENSITIVE_PROPS_KEY); @@ -278,7 +282,8 @@ public final class ConfigTransformer { writer.println("nifi.content.repository.always.sync=" + getValueString(contentRepo, ALWAYS_SYNC_KEY)); writer.println(); writer.println("# Provenance Repository Properties"); - writer.println("nifi.provenance.repository.implementation=org.apache.nifi.provenance.VolatileProvenanceRepository"); + writer.println("nifi.provenance.repository.implementation=org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository"); + writer.println("nifi.provenance.repository.rollover.time=" + getValueString(provenanceRepo, PROVENANCE_REPO_ROLLOVER_TIME_KEY)); writer.println(); writer.println("# Volatile Provenance Respository Properties"); writer.println("nifi.provenance.repository.buffer.size=100000"); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-bootstrap/src/test/resources/config.yml ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/config.yml b/minifi-bootstrap/src/test/resources/config.yml index 0da4bf3..bb4dec4 100644 --- a/minifi-bootstrap/src/test/resources/config.yml +++ b/minifi-bootstrap/src/test/resources/config.yml @@ -39,6 +39,9 @@ Content Repository: content claim max flow files: 100 always sync: false +Provenance Repository: + provenance rollover time: 1 min + Component Status Repository: buffer size: 1440 snapshot frequency: 1 min http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-bootstrap/src/test/resources/default.yml ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/default.yml b/minifi-bootstrap/src/test/resources/default.yml index 064a746..5043900 100644 --- a/minifi-bootstrap/src/test/resources/default.yml +++ b/minifi-bootstrap/src/test/resources/default.yml @@ -39,6 +39,9 @@ Content Repository: content claim max flow files: 100 always sync: false +Provenance Repository: + provenance rollover time: 1 min + Component Status Repository: buffer size: 1440 snapshot frequency: 1 min http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-commons/minifi-utils/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-commons/minifi-utils/pom.xml b/minifi-commons/minifi-utils/pom.xml new file mode 100644 index 0000000..efcb2c7 --- /dev/null +++ b/minifi-commons/minifi-utils/pom.xml @@ -0,0 +1,28 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi.minifi</groupId> + <artifactId>minifi-commons</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> + + <artifactId>minifi-utils</artifactId> + <packaging>jar</packaging> + + +</project> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-commons/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-commons/pom.xml b/minifi-commons/pom.xml index 992dbdd..8d44c71 100644 --- a/minifi-commons/pom.xml +++ b/minifi-commons/pom.xml @@ -25,7 +25,11 @@ limitations under the License. <version>0.0.1-SNAPSHOT</version> </parent> <artifactId>minifi-commons</artifactId> - <packaging>jar</packaging> + <packaging>pom</packaging> + + <modules> + <module>minifi-utils</module> + </modules> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-docs/Properties_Guide.md ---------------------------------------------------------------------- diff --git a/minifi-docs/Properties_Guide.md b/minifi-docs/Properties_Guide.md index 6306483..1e2b09c 100644 --- a/minifi-docs/Properties_Guide.md +++ b/minifi-docs/Properties_Guide.md @@ -94,6 +94,12 @@ content claim max appendable size | The maximum size for a content claim. The de content claim max flow files | The maximum number of FlowFiles to assign to one content claim. The default value is 100. always sync | If set to _true_, any change to the repository will be synchronized to the disk, meaning that NiFi will ask the operating system not to cache the information. This is very expensive and can significantly reduce NiFi performance. However, if it is _false_, there could be the potential for data loss if either there is a sudden power loss or the operating system crashes. The default value is _false_. +## Provenance Repository + +*Property* | *Description* +-------------------------------- | ------------- +provenance rollover time | The amount of time to wait before rolling over the latest data provenance information so that it is available to be accessed by components. The default value is 1 min. + ## *Component Status Repository* The Component Status Repository contains the information for the Component Status History tool in the User Interface. These http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml index cd78d0b..84a4ed2 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml @@ -50,7 +50,6 @@ limitations under the License. <dependency> <groupId>org.apache.nifi.minifi</groupId> <artifactId>minifi-runtime</artifactId> - <version>0.0.1-SNAPSHOT</version> <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml ---------------------------------------------------------------------- diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml index 064a746..5043900 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml @@ -39,6 +39,9 @@ Content Repository: content claim max flow files: 100 always sync: false +Provenance Repository: + provenance rollover time: 1 min + Component Status Repository: buffer size: 1440 snapshot frequency: 1 min http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml index be74790..b08a081 100644 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml @@ -30,7 +30,6 @@ <dependency> <groupId>org.apache.nifi.minifi</groupId> <artifactId>minifi-provenance-reporting-task</artifactId> - <version>0.0.1-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.nifi</groupId> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml index 0d34004..d9bfc04 100644 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml @@ -28,10 +28,15 @@ <module>minifi-provenance-reporting-task</module> <module>minifi-provenance-reporting-nar</module> </modules> - + <dependencyManagement> <dependencies> <dependency> + <groupId>org.apache.nifi.minifi</groupId> + <artifactId>minifi-provenance-reporting-task</artifactId> + <version>0.0.1-SNAPSHOT</version> + </dependency> + <dependency> <groupId>org.glassfish.jersey.core</groupId> <artifactId>jersey-client</artifactId> <version>2.19</version> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/fb554819/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/pom.xml b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/pom.xml new file mode 100644 index 0000000..9916ccd --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/pom.xml @@ -0,0 +1,57 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi.minifi</groupId> + <artifactId>minifi-provenance-repository-bundle</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> + + <artifactId>minifi-persistent-provenance-repository</artifactId> + + <dependencies> + <dependency> + <groupId>org.apache.nifi.minifi</groupId> + <artifactId>minifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-utils</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-properties</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-data-provenance-utils</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-persistent-provenance-repository</artifactId> + <version>0.6.0</version> + </dependency> + </dependencies> +</project> \ No newline at end of file
