NIFI-244: Initial import of GetTwitter processor
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/e9cb3b30 Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/e9cb3b30 Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/e9cb3b30 Branch: refs/heads/NIFI-271 Commit: e9cb3b300c6b5e223744925cee1d0c59dd97d29a Parents: 178c5cd Author: Mark Payne <marka...@hotmail.com> Authored: Thu Apr 9 17:56:52 2015 -0400 Committer: Mark Payne <marka...@hotmail.com> Committed: Thu Apr 9 17:56:52 2015 -0400 ---------------------------------------------------------------------- nifi/nifi-assembly/NOTICE | 49 + nifi/nifi-assembly/pom.xml | 925 ++++++++++--------- nifi/nifi-commons/pom.xml | 1 + .../nifi-social-media-nar/pom.xml | 36 + .../nifi-twitter-processors/.gitignore | 1 + .../nifi-twitter-processors/pom.xml | 60 ++ .../nifi/processors/twitter/GetTwitter.java | 360 ++++++++ .../org.apache.nifi.processor.Processor | 16 + .../nifi-social-media-bundle/pom.xml | 33 + nifi/nifi-nar-bundles/pom.xml | 4 + nifi/pom.xml | 24 + 11 files changed, 1060 insertions(+), 449 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-assembly/NOTICE ---------------------------------------------------------------------- diff --git a/nifi/nifi-assembly/NOTICE b/nifi/nifi-assembly/NOTICE index 8d7db8d..d95e2ff 100644 --- a/nifi/nifi-assembly/NOTICE +++ b/nifi/nifi-assembly/NOTICE @@ -501,6 +501,38 @@ The following binary components are provided under the Apache Software License v Apache License Version 2.0 http://www.apache.org/licenses/. (c) Daniel Lemire, http://lemire.me/en/ + (ASLv2) Twitter4J + The following NOTICE information applies: + Copyright 2007 Yusuke Yamamoto + + Twitter4J includes software from JSON.org to parse JSON response from the Twitter API. You can see the license term at http://www.JSON.org/license.html + + (ASLv2) JOAuth + The following NOTICE information applies: + JOAuth + Copyright 2010-2013 Twitter, Inc + + Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + + (ASLv2) Hosebird Client + The following NOTICE information applies: + Hosebird Client (hbc) + Copyright 2013 Twitter, Inc. + + Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + + (ASLv2) GeoIP2 Java API + The following NOTICE information applies: + GeoIP2 Java API + This software is Copyright (c) 2013 by MaxMind, Inc. + + This is free software, licensed under the Apache License, Version 2.0. + + (ASLv2) Google HTTP Client Library for Java + The following NOTICE information applies: + Google HTTP Client Library for Java + + This is free software, licensed under the Apache License, Version 2.0. ************************ Common Development and Distribution License 1.1 @@ -541,6 +573,14 @@ The following binary components are provided under the Common Development and Di (CDDL 1.0) SR 250 Common Annotations For The JavaTM Platform (javax.annotation:jsr250-api:jar:1.0 - http://jcp.org/aboutJava/communityprocess/final/jsr250/index.html) ************************ +Creative Commons Attribution-ShareAlike 3.0 +************************ + +The following binary components are provided under the Creative Commons Attribution-ShareAlike 3.0. See project link for details. + + (CCAS 3.0) MaxMind DB (https://github.com/maxmind/MaxMind-DB) + +************************ Eclipse Public License 1.0 ************************ @@ -560,6 +600,15 @@ The following binary components are provided under the Mozilla Public License v2 (MPL 2.0) Saxon HE (net.sf.saxon:Saxon-HE:jar:9.6.0-4 - http://www.saxonica.com/) ***************** +Mozilla Public License v1.1 +***************** + +The following binary components are provided under the Mozilla Public License v1.1. See project link for details. + + (MPL 1.1) HAPI Base (ca.uhn.hapi:hapi-base:2.2 - http://http://hl7api.sourceforge.net/) + (MPL 1.1) HAPI Structures (ca.uhn.hapi:hapi-structures-v*:2.2 - http://http://hl7api.sourceforge.net/) + +***************** Public Domain ***************** http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-assembly/pom.xml b/nifi/nifi-assembly/pom.xml index a26f214..13ffba8 100644 --- a/nifi/nifi-assembly/pom.xml +++ b/nifi/nifi-assembly/pom.xml @@ -1,457 +1,484 @@ <?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi</artifactId> - <version>0.1.0-incubating-SNAPSHOT</version> - </parent> - <artifactId>nifi-assembly</artifactId> - <packaging>pom</packaging> - <description>This is the assembly Apache NiFi (incubating)</description> - <build> - <plugins> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <finalName>nifi-${project.version}</finalName> - <attach>false</attach> - </configuration> - <executions> - <execution> - <id>make shared resource</id> - <goals> - <goal>single</goal> - </goals> - <phase>package</phase> - <configuration> - <descriptors> - <descriptor>src/main/assembly/dependencies.xml</descriptor> - </descriptors> - <tarLongFileMode>posix</tarLongFileMode> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - <dependencies> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jcl-over-slf4j</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jul-to-slf4j</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>log4j-over-slf4j</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-runtime</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-bootstrap</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-resources</artifactId> - <classifier>resources</classifier> - <scope>runtime</scope> - <type>zip</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-docs</artifactId> - <classifier>resources</classifier> - <scope>runtime</scope> - <type>zip</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-framework-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-provenance-repository-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-standard-services-api-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-ssl-context-service-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-distributed-cache-services-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-standard-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-jetty-bundle</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-update-attribute-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-hadoop-libraries-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-hadoop-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-kafka-nar</artifactId> - <type>nar</type> - </dependency> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + </parent> + <artifactId>nifi-assembly</artifactId> + <packaging>pom</packaging> + <description>This is the assembly Apache NiFi (incubating)</description> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <finalName>nifi-${project.version}</finalName> + <attach>false</attach> + </configuration> + <executions> + <execution> + <id>make shared resource</id> + <goals> + <goal>single</goal> + </goals> + <phase>package</phase> + <configuration> + <descriptors> + <descriptor>src/main/assembly/dependencies.xml</descriptor> + </descriptors> + <tarLongFileMode>posix</tarLongFileMode> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <dependencies> <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-http-context-map-nar</artifactId> - <type>nar</type> - </dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <scope>compile</scope> + </dependency> <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-kite-nar</artifactId> - <type>nar</type> - </dependency> - </dependencies> - - <properties> - <!--Wrapper Properties--> - <nifi.wrapper.jvm.heap.initial.mb>256</nifi.wrapper.jvm.heap.initial.mb> - <nifi.wrapper.jvm.heap.max.mb>512</nifi.wrapper.jvm.heap.max.mb> - <nifi.initial.permgen.size.mb>128</nifi.initial.permgen.size.mb> - <nifi.max.permgen.size.mb>128</nifi.max.permgen.size.mb> - <nifi.wrapper.logfile.maxsize>10m</nifi.wrapper.logfile.maxsize> - <nifi.wrapper.logfile.maxfiles>10</nifi.wrapper.logfile.maxfiles> - - <!-- nifi.properties: core properties --> - <nifi.version>${project.version}</nifi.version> - <nifi.flowcontroller.autoResumeState>true</nifi.flowcontroller.autoResumeState> - <nifi.flowcontroller.graceful.shutdown.period>10 sec</nifi.flowcontroller.graceful.shutdown.period> - <nifi.flowservice.writedelay.interval>500 ms</nifi.flowservice.writedelay.interval> - <nifi.administrative.yield.duration>30 sec</nifi.administrative.yield.duration> - <nifi.bored.yield.duration>10 millis</nifi.bored.yield.duration> + <groupId>org.slf4j</groupId> + <artifactId>jcl-over-slf4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jul-to-slf4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-runtime</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-bootstrap</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-resources</artifactId> + <classifier>resources</classifier> + <scope>runtime</scope> + <type>zip</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-docs</artifactId> + <classifier>resources</classifier> + <scope>runtime</scope> + <type>zip</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-framework-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-provenance-repository-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-services-api-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-ssl-context-service-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-distributed-cache-services-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-jetty-bundle</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-update-attribute-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-hadoop-libraries-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-hadoop-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kafka-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-http-context-map-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kite-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-social-media-nar</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-hl7-nar</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-language-translation-nar</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-geo-nar</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + <type>nar</type> + </dependency> + </dependencies> + + <properties> + <!--Wrapper Properties --> + <nifi.wrapper.jvm.heap.initial.mb>256</nifi.wrapper.jvm.heap.initial.mb> + <nifi.wrapper.jvm.heap.max.mb>512</nifi.wrapper.jvm.heap.max.mb> + <nifi.initial.permgen.size.mb>128</nifi.initial.permgen.size.mb> + <nifi.max.permgen.size.mb>128</nifi.max.permgen.size.mb> + <nifi.wrapper.logfile.maxsize>10m</nifi.wrapper.logfile.maxsize> + <nifi.wrapper.logfile.maxfiles>10</nifi.wrapper.logfile.maxfiles> + + <!-- nifi.properties: core properties --> + <nifi.version>${project.version}</nifi.version> + <nifi.flowcontroller.autoResumeState>true</nifi.flowcontroller.autoResumeState> + <nifi.flowcontroller.graceful.shutdown.period>10 sec</nifi.flowcontroller.graceful.shutdown.period> + <nifi.flowservice.writedelay.interval>500 ms</nifi.flowservice.writedelay.interval> + <nifi.administrative.yield.duration>30 sec</nifi.administrative.yield.duration> + <nifi.bored.yield.duration>10 millis</nifi.bored.yield.duration> + + <nifi.flow.configuration.file>./conf/flow.xml.gz</nifi.flow.configuration.file> + <nifi.flow.configuration.archive.dir>./conf/archive/</nifi.flow.configuration.archive.dir> + <nifi.authority.provider.configuration.file>./conf/authority-providers.xml</nifi.authority.provider.configuration.file> + <nifi.templates.directory>./conf/templates</nifi.templates.directory> + <nifi.database.directory>./database_repository</nifi.database.directory> + + <nifi.flowfile.repository.implementation>org.apache.nifi.controller.repository.WriteAheadFlowFileRepository</nifi.flowfile.repository.implementation> + <nifi.flowfile.repository.directory>./flowfile_repository</nifi.flowfile.repository.directory> + <nifi.flowfile.repository.partitions>256</nifi.flowfile.repository.partitions> + <nifi.flowfile.repository.checkpoint.interval>2 mins</nifi.flowfile.repository.checkpoint.interval> + <nifi.flowfile.repository.always.sync>false</nifi.flowfile.repository.always.sync> + <nifi.swap.manager.implementation>org.apache.nifi.controller.FileSystemSwapManager</nifi.swap.manager.implementation> + <nifi.queue.swap.threshold>20000</nifi.queue.swap.threshold> + <nifi.swap.in.period>5 sec</nifi.swap.in.period> + <nifi.swap.in.threads>1</nifi.swap.in.threads> + <nifi.swap.out.period>5 sec</nifi.swap.out.period> + <nifi.swap.out.threads>4</nifi.swap.out.threads> + + <nifi.content.repository.implementation>org.apache.nifi.controller.repository.FileSystemRepository</nifi.content.repository.implementation> + <nifi.content.claim.max.appendable.size>10 MB</nifi.content.claim.max.appendable.size> + <nifi.content.claim.max.flow.files>100</nifi.content.claim.max.flow.files> + <nifi.content.repository.directory.default>./content_repository</nifi.content.repository.directory.default> + <nifi.content.repository.archive.max.retention.period /> + <nifi.content.repository.archive.max.usage.percentage /> + <nifi.content.repository.archive.enabled>false</nifi.content.repository.archive.enabled> + <nifi.content.repository.always.sync>false</nifi.content.repository.always.sync> + <nifi.content.viewer.url /> + + <nifi.restore.directory /> + <nifi.ui.banner.text /> + <nifi.ui.autorefresh.interval>30 sec</nifi.ui.autorefresh.interval> + <nifi.nar.library.directory>./lib</nifi.nar.library.directory> + <nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory> + <nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory> + + <nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm> + <nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider> + <nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append> + + <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port> + + <!-- persistent provenance repository properties --> + <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation> + <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default> + <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time> + <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size> + <nifi.provenance.repository.rollover.time>5 mins</nifi.provenance.repository.rollover.time> + <nifi.provenance.repository.rollover.size>100 MB</nifi.provenance.repository.rollover.size> + <nifi.provenance.repository.query.threads>2</nifi.provenance.repository.query.threads> + <nifi.provenance.repository.compress.on.rollover>true</nifi.provenance.repository.compress.on.rollover> + <nifi.provenance.repository.indexed.fields>EventType, FlowFileUUID, + Filename, ProcessorID</nifi.provenance.repository.indexed.fields> + <nifi.provenance.repository.indexed.attributes /> + <nifi.provenance.repository.index.shard.size>500 MB</nifi.provenance.repository.index.shard.size> + <nifi.provenance.repository.always.sync>false</nifi.provenance.repository.always.sync> + <nifi.provenance.repository.journal.count>16</nifi.provenance.repository.journal.count> + + <!-- volatile provenance repository properties --> + <nifi.provenance.repository.buffer.size>100000</nifi.provenance.repository.buffer.size> + + <!-- Component status repository properties --> + <nifi.components.status.repository.implementation>org.apache.nifi.controller.status.history.VolatileComponentStatusRepository</nifi.components.status.repository.implementation> + <nifi.components.status.repository.buffer.size>288</nifi.components.status.repository.buffer.size> + <nifi.components.status.snapshot.frequency>5 mins</nifi.components.status.snapshot.frequency> + + <!-- nifi.properties: web properties --> + <nifi.web.war.directory>./lib</nifi.web.war.directory> + <nifi.web.http.host /> + <nifi.web.http.port>8080</nifi.web.http.port> + <nifi.web.https.host /> + <nifi.web.https.port /> + <nifi.jetty.work.dir>./work/jetty</nifi.jetty.work.dir> + <nifi.web.jetty.threads>200</nifi.web.jetty.threads> - <nifi.flow.configuration.file>./conf/flow.xml.gz</nifi.flow.configuration.file> - <nifi.flow.configuration.archive.dir>./conf/archive/</nifi.flow.configuration.archive.dir> - <nifi.authority.provider.configuration.file>./conf/authority-providers.xml</nifi.authority.provider.configuration.file> - <nifi.templates.directory>./conf/templates</nifi.templates.directory> - <nifi.database.directory>./database_repository</nifi.database.directory> + <!-- nifi.properties: security properties --> + <nifi.security.keystore /> + <nifi.security.keystoreType /> + <nifi.security.keystorePasswd /> + <nifi.security.keyPasswd /> + <nifi.security.truststore /> + <nifi.security.truststoreType /> + <nifi.security.truststorePasswd /> + <nifi.security.needClientAuth /> + <nifi.security.authorizedUsers.file>./conf/authorized-users.xml</nifi.security.authorizedUsers.file> + <nifi.security.user.credential.cache.duration>24 hours</nifi.security.user.credential.cache.duration> + <nifi.security.user.authority.provider>file-provider</nifi.security.user.authority.provider> + <nifi.security.x509.principal.extractor /> + <nifi.security.support.new.account.requests /> + <nifi.security.ocsp.responder.url /> + <nifi.security.ocsp.responder.certificate /> - <nifi.flowfile.repository.implementation>org.apache.nifi.controller.repository.WriteAheadFlowFileRepository</nifi.flowfile.repository.implementation> - <nifi.flowfile.repository.directory>./flowfile_repository</nifi.flowfile.repository.directory> - <nifi.flowfile.repository.partitions>256</nifi.flowfile.repository.partitions> - <nifi.flowfile.repository.checkpoint.interval>2 mins</nifi.flowfile.repository.checkpoint.interval> - <nifi.flowfile.repository.always.sync>false</nifi.flowfile.repository.always.sync> - <nifi.swap.manager.implementation>org.apache.nifi.controller.FileSystemSwapManager</nifi.swap.manager.implementation> - <nifi.queue.swap.threshold>20000</nifi.queue.swap.threshold> - <nifi.swap.in.period>5 sec</nifi.swap.in.period> - <nifi.swap.in.threads>1</nifi.swap.in.threads> - <nifi.swap.out.period>5 sec</nifi.swap.out.period> - <nifi.swap.out.threads>4</nifi.swap.out.threads> - - <nifi.content.repository.implementation>org.apache.nifi.controller.repository.FileSystemRepository</nifi.content.repository.implementation> - <nifi.content.claim.max.appendable.size>10 MB</nifi.content.claim.max.appendable.size> - <nifi.content.claim.max.flow.files>100</nifi.content.claim.max.flow.files> - <nifi.content.repository.directory.default>./content_repository</nifi.content.repository.directory.default> - <nifi.content.repository.archive.max.retention.period /> - <nifi.content.repository.archive.max.usage.percentage /> - <nifi.content.repository.archive.enabled>false</nifi.content.repository.archive.enabled> - <nifi.content.repository.always.sync>false</nifi.content.repository.always.sync> - <nifi.content.viewer.url /> - - <nifi.restore.directory /> - <nifi.ui.banner.text /> - <nifi.ui.autorefresh.interval>30 sec</nifi.ui.autorefresh.interval> - <nifi.nar.library.directory>./lib</nifi.nar.library.directory> - <nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory> - <nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory> - - <nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm> - <nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider> - <nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append> + <!-- nifi.properties: cluster common properties (cluster manager and nodes + must have same values) --> + <nifi.cluster.protocol.heartbeat.interval>5 sec</nifi.cluster.protocol.heartbeat.interval> + <nifi.cluster.protocol.is.secure>false</nifi.cluster.protocol.is.secure> + <nifi.cluster.protocol.socket.timeout>30 sec</nifi.cluster.protocol.socket.timeout> + <nifi.cluster.protocol.connection.handshake.timeout>45 sec</nifi.cluster.protocol.connection.handshake.timeout> + <nifi.cluster.protocol.use.multicast>false</nifi.cluster.protocol.use.multicast> + <nifi.cluster.protocol.multicast.address /> + <nifi.cluster.protocol.multicast.port /> + <nifi.cluster.protocol.multicast.service.broadcast.delay>500 ms</nifi.cluster.protocol.multicast.service.broadcast.delay> + <nifi.cluster.protocol.multicast.service.locator.attempts>3</nifi.cluster.protocol.multicast.service.locator.attempts> + <nifi.cluster.protocol.multicast.service.locator.attempts.delay>1 sec</nifi.cluster.protocol.multicast.service.locator.attempts.delay> - <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port> - - <!-- persistent provenance repository properties --> - <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation> - <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default> - <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time> - <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size> - <nifi.provenance.repository.rollover.time>5 mins</nifi.provenance.repository.rollover.time> - <nifi.provenance.repository.rollover.size>100 MB</nifi.provenance.repository.rollover.size> - <nifi.provenance.repository.query.threads>2</nifi.provenance.repository.query.threads> - <nifi.provenance.repository.compress.on.rollover>true</nifi.provenance.repository.compress.on.rollover> - <nifi.provenance.repository.indexed.fields>EventType, FlowFileUUID, Filename, ProcessorID</nifi.provenance.repository.indexed.fields> - <nifi.provenance.repository.indexed.attributes /> - <nifi.provenance.repository.index.shard.size>500 MB</nifi.provenance.repository.index.shard.size> - <nifi.provenance.repository.always.sync>false</nifi.provenance.repository.always.sync> - <nifi.provenance.repository.journal.count>16</nifi.provenance.repository.journal.count> - - <!-- volatile provenance repository properties --> - <nifi.provenance.repository.buffer.size>100000</nifi.provenance.repository.buffer.size> - - <!-- Component status repository properties --> - <nifi.components.status.repository.implementation>org.apache.nifi.controller.status.history.VolatileComponentStatusRepository</nifi.components.status.repository.implementation> - <nifi.components.status.repository.buffer.size>288</nifi.components.status.repository.buffer.size> - <nifi.components.status.snapshot.frequency>5 mins</nifi.components.status.snapshot.frequency> - - <!-- nifi.properties: web properties --> - <nifi.web.war.directory>./lib</nifi.web.war.directory> - <nifi.web.http.host /> - <nifi.web.http.port>8080</nifi.web.http.port> - <nifi.web.https.host /> - <nifi.web.https.port /> - <nifi.jetty.work.dir>./work/jetty</nifi.jetty.work.dir> - <nifi.web.jetty.threads>200</nifi.web.jetty.threads> - - <!-- nifi.properties: security properties --> - <nifi.security.keystore /> - <nifi.security.keystoreType /> - <nifi.security.keystorePasswd /> - <nifi.security.keyPasswd /> - <nifi.security.truststore /> - <nifi.security.truststoreType /> - <nifi.security.truststorePasswd /> - <nifi.security.needClientAuth /> - <nifi.security.authorizedUsers.file>./conf/authorized-users.xml</nifi.security.authorizedUsers.file> - <nifi.security.user.credential.cache.duration>24 hours</nifi.security.user.credential.cache.duration> - <nifi.security.user.authority.provider>file-provider</nifi.security.user.authority.provider> - <nifi.security.x509.principal.extractor /> - <nifi.security.support.new.account.requests /> - <nifi.security.ocsp.responder.url /> - <nifi.security.ocsp.responder.certificate /> - - <!-- nifi.properties: cluster common properties (cluster manager and nodes must have same values) --> - <nifi.cluster.protocol.heartbeat.interval>5 sec</nifi.cluster.protocol.heartbeat.interval> - <nifi.cluster.protocol.is.secure>false</nifi.cluster.protocol.is.secure> - <nifi.cluster.protocol.socket.timeout>30 sec</nifi.cluster.protocol.socket.timeout> - <nifi.cluster.protocol.connection.handshake.timeout>45 sec</nifi.cluster.protocol.connection.handshake.timeout> - <nifi.cluster.protocol.use.multicast>false</nifi.cluster.protocol.use.multicast> - <nifi.cluster.protocol.multicast.address /> - <nifi.cluster.protocol.multicast.port /> - <nifi.cluster.protocol.multicast.service.broadcast.delay>500 ms</nifi.cluster.protocol.multicast.service.broadcast.delay> - <nifi.cluster.protocol.multicast.service.locator.attempts>3</nifi.cluster.protocol.multicast.service.locator.attempts> - <nifi.cluster.protocol.multicast.service.locator.attempts.delay>1 sec</nifi.cluster.protocol.multicast.service.locator.attempts.delay> + <!-- nifi.properties: cluster node properties (only configure for cluster + nodes) --> + <nifi.cluster.is.node>false</nifi.cluster.is.node> + <nifi.cluster.node.address /> + <nifi.cluster.node.protocol.port /> + <nifi.cluster.node.protocol.threads>2</nifi.cluster.node.protocol.threads> + <nifi.cluster.node.unicast.manager.address /> + <nifi.cluster.node.unicast.manager.protocol.port /> - <!-- nifi.properties: cluster node properties (only configure for cluster nodes) --> - <nifi.cluster.is.node>false</nifi.cluster.is.node> - <nifi.cluster.node.address /> - <nifi.cluster.node.protocol.port /> - <nifi.cluster.node.protocol.threads>2</nifi.cluster.node.protocol.threads> - <nifi.cluster.node.unicast.manager.address /> - <nifi.cluster.node.unicast.manager.protocol.port /> - - <!-- nifi.properties: cluster manager properties (only configure for cluster manager) --> - <nifi.cluster.is.manager>false</nifi.cluster.is.manager> - <nifi.cluster.manager.address /> - <nifi.cluster.manager.protocol.port /> - <nifi.cluster.manager.node.firewall.file /> - <nifi.cluster.manager.node.event.history.size>10</nifi.cluster.manager.node.event.history.size> - <nifi.cluster.manager.node.api.connection.timeout>30 sec</nifi.cluster.manager.node.api.connection.timeout> - <nifi.cluster.manager.node.api.read.timeout>30 sec</nifi.cluster.manager.node.api.read.timeout> - <nifi.cluster.manager.node.api.request.threads>10</nifi.cluster.manager.node.api.request.threads> - <nifi.cluster.manager.flow.retrieval.delay>5 sec</nifi.cluster.manager.flow.retrieval.delay> - <nifi.cluster.manager.protocol.threads>10</nifi.cluster.manager.protocol.threads> - <nifi.cluster.manager.safemode.duration>0 sec</nifi.cluster.manager.safemode.duration> - </properties> - <profiles> - <profile> - <id>rpm</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>unpack-shared-resources</id> - <goals> - <goal>unpack-dependencies</goal> - </goals> - <phase>generate-resources</phase> - <configuration> - <outputDirectory>${project.build.directory}/generated-resources</outputDirectory> - <includeArtifactIds>nifi-resources</includeArtifactIds> - <includeGroupIds>org.apache.nifi</includeGroupIds> - <excludeTransitive>false</excludeTransitive> - </configuration> - </execution> - <execution> - <id>unpack-docs</id> - <goals> - <goal>unpack-dependencies</goal> - </goals> - <phase>generate-resources</phase> - <configuration> - <outputDirectory>${project.build.directory}/generated-docs</outputDirectory> - <includeArtifactIds>nifi-docs</includeArtifactIds> - <includeGroupIds>org.apache.nifi</includeGroupIds> - <excludeTransitive>false</excludeTransitive> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>rpm-maven-plugin</artifactId> - <configuration> - <summary>Apache NiFi (incubating)</summary> - <description>Apache Nifi (incubating) is dataflow system based on the Flow-Based Programming concepts.</description> - <license>Apache License, Version 2.0 and others (see included LICENSE file)</license> - <url>http://nifi.incubator.apache.org</url> - <group>Utilities</group> - <prefix>/opt/nifi</prefix> - <defineStatements> - <defineStatement>_use_internal_dependency_generator 0</defineStatement> - </defineStatements> - <defaultDirmode>750</defaultDirmode> - <defaultFilemode>640</defaultFilemode> - <defaultUsername>root</defaultUsername> - <defaultGroupname>root</defaultGroupname> - </configuration> - <executions> - <execution> - <id>build-bin-rpm</id> - <goals> - <goal>attached-rpm</goal> - </goals> - <configuration> - <classifier>bin</classifier> - <provides> - <provide>nifi</provide> - </provides> - <mappings> - <mapping> - <directory>/opt/nifi/nifi-${project.version}</directory> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}</directory> - <sources> - <source> - <location>./LICENSE</location> - </source> - <source> - <location>./NOTICE</location> - </source> - <source> - <location>../DISCLAIMER</location> - </source> - <source> - <location>./README.md</location> - <destination>README</destination> - </source> - </sources> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/bin</directory> - <filemode>750</filemode> - <sources> - <source> - <location>${project.build.directory}/generated-resources/bin/nifi.sh</location> - <destination>nifi.sh</destination> - <filter>true</filter> - </source> - </sources> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/conf</directory> - <configuration>true</configuration> - <sources> - <source> - <location>${project.build.directory}/generated-resources/conf</location> - <filter>true</filter> - </source> - </sources> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/lib</directory> - <dependency> - <excludes> - <exclude>org.apache.nifi:nifi-bootstrap</exclude> - <exclude>org.apache.nifi:nifi-resources</exclude> - <exclude>org.apache.nifi:nifi-docs</exclude> - </excludes> - </dependency> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/lib/bootstrap</directory> - <dependency> - <includes> - <include>org.apache.nifi:nifi-bootstrap</include> - </includes> - </dependency> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/docs</directory> - <sources> - <source> - <location>${project.build.directory}/generated-docs</location> - </source> - </sources> - </mapping> - </mappings> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> + <!-- nifi.properties: cluster manager properties (only configure for cluster + manager) --> + <nifi.cluster.is.manager>false</nifi.cluster.is.manager> + <nifi.cluster.manager.address /> + <nifi.cluster.manager.protocol.port /> + <nifi.cluster.manager.node.firewall.file /> + <nifi.cluster.manager.node.event.history.size>10</nifi.cluster.manager.node.event.history.size> + <nifi.cluster.manager.node.api.connection.timeout>30 sec</nifi.cluster.manager.node.api.connection.timeout> + <nifi.cluster.manager.node.api.read.timeout>30 sec</nifi.cluster.manager.node.api.read.timeout> + <nifi.cluster.manager.node.api.request.threads>10</nifi.cluster.manager.node.api.request.threads> + <nifi.cluster.manager.flow.retrieval.delay>5 sec</nifi.cluster.manager.flow.retrieval.delay> + <nifi.cluster.manager.protocol.threads>10</nifi.cluster.manager.protocol.threads> + <nifi.cluster.manager.safemode.duration>0 sec</nifi.cluster.manager.safemode.duration> + </properties> + <profiles> + <profile> + <id>rpm</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>unpack-shared-resources</id> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <phase>generate-resources</phase> + <configuration> + <outputDirectory>${project.build.directory}/generated-resources</outputDirectory> + <includeArtifactIds>nifi-resources</includeArtifactIds> + <includeGroupIds>org.apache.nifi</includeGroupIds> + <excludeTransitive>false</excludeTransitive> + </configuration> + </execution> + <execution> + <id>unpack-docs</id> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <phase>generate-resources</phase> + <configuration> + <outputDirectory>${project.build.directory}/generated-docs</outputDirectory> + <includeArtifactIds>nifi-docs</includeArtifactIds> + <includeGroupIds>org.apache.nifi</includeGroupIds> + <excludeTransitive>false</excludeTransitive> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>rpm-maven-plugin</artifactId> + <configuration> + <summary>Apache NiFi (incubating)</summary> + <description>Apache Nifi (incubating) is dataflow system based on + the Flow-Based Programming concepts.</description> + <license>Apache License, Version 2.0 and others (see included + LICENSE file)</license> + <url>http://nifi.incubator.apache.org</url> + <group>Utilities</group> + <prefix>/opt/nifi</prefix> + <defineStatements> + <defineStatement>_use_internal_dependency_generator 0</defineStatement> + </defineStatements> + <defaultDirmode>750</defaultDirmode> + <defaultFilemode>640</defaultFilemode> + <defaultUsername>root</defaultUsername> + <defaultGroupname>root</defaultGroupname> + </configuration> + <executions> + <execution> + <id>build-bin-rpm</id> + <goals> + <goal>attached-rpm</goal> + </goals> + <configuration> + <classifier>bin</classifier> + <provides> + <provide>nifi</provide> + </provides> + <mappings> + <mapping> + <directory>/opt/nifi/nifi-${project.version}</directory> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}</directory> + <sources> + <source> + <location>./LICENSE</location> + </source> + <source> + <location>./NOTICE</location> + </source> + <source> + <location>../DISCLAIMER</location> + </source> + <source> + <location>./README.md</location> + <destination>README</destination> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/bin</directory> + <filemode>750</filemode> + <sources> + <source> + <location>${project.build.directory}/generated-resources/bin/nifi.sh</location> + <destination>nifi.sh</destination> + <filter>true</filter> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/conf</directory> + <configuration>true</configuration> + <sources> + <source> + <location>${project.build.directory}/generated-resources/conf</location> + <filter>true</filter> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/lib</directory> + <dependency> + <excludes> + <exclude>org.apache.nifi:nifi-bootstrap</exclude> + <exclude>org.apache.nifi:nifi-resources</exclude> + <exclude>org.apache.nifi:nifi-docs</exclude> + </excludes> + </dependency> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/lib/bootstrap</directory> + <dependency> + <includes> + <include>org.apache.nifi:nifi-bootstrap</include> + </includes> + </dependency> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/docs</directory> + <sources> + <source> + <location>${project.build.directory}/generated-docs</location> + </source> + </sources> + </mapping> + </mappings> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-commons/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/pom.xml b/nifi/nifi-commons/pom.xml index 5358054..1d2ce46 100644 --- a/nifi/nifi-commons/pom.xml +++ b/nifi/nifi-commons/pom.xml @@ -36,5 +36,6 @@ <module>nifi-processor-utilities</module> <module>nifi-write-ahead-log</module> <module>nifi-site-to-site-client</module> + <module>nifi-hl7-query-language</module> </modules> </project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml new file mode 100644 index 0000000..6da74dd --- /dev/null +++ b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-social-media-bundle</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + </parent> + + <artifactId>nifi-social-media-nar</artifactId> + <packaging>nar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-twitter-processors</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/.gitignore ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/.gitignore b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/.gitignore @@ -0,0 +1 @@ +/target/ http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/pom.xml b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/pom.xml new file mode 100644 index 0000000..45af0ce --- /dev/null +++ b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/pom.xml @@ -0,0 +1,60 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-social-media-bundle</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + </parent> + + <artifactId>nifi-twitter-processors</artifactId> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-processor-utils</artifactId> + </dependency> + + <dependency> + <groupId>com.twitter</groupId> + <artifactId>hbc-twitter4j</artifactId> + <version>2.2.0</version> + </dependency> + + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-mock</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java new file mode 100644 index 0000000..45b1ae1 --- /dev/null +++ b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.twitter; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.MalformedURLException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.regex.Pattern; + +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.ProcessorInitializationContext; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.io.OutputStreamCallback; +import org.apache.nifi.processor.util.StandardValidators; + +import com.twitter.hbc.ClientBuilder; +import com.twitter.hbc.core.Client; +import com.twitter.hbc.core.Constants; +import com.twitter.hbc.core.endpoint.StatusesFilterEndpoint; +import com.twitter.hbc.core.endpoint.StatusesFirehoseEndpoint; +import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint; +import com.twitter.hbc.core.endpoint.StreamingEndpoint; +import com.twitter.hbc.core.event.Event; +import com.twitter.hbc.core.processor.StringDelimitedProcessor; +import com.twitter.hbc.httpclient.auth.Authentication; +import com.twitter.hbc.httpclient.auth.OAuth1; + +@SupportsBatching +@Tags({"twitter", "tweets", "social media", "status", "json"}) +@CapabilityDescription("Pulls status changes from Twitter's streaming API") +@WritesAttribute(attribute="mime.type", description="Sets mime type to application/json") +public class GetTwitter extends AbstractProcessor { + + static final AllowableValue ENDPOINT_SAMPLE = new AllowableValue("Sample Endpoint", "Sample Endpoint", "The endpoint that provides public data, aka a 'garden hose'"); + static final AllowableValue ENDPOINT_FIREHOSE = new AllowableValue("Firehose Endpoint", "Firehose Endpoint", "The endpoint that provides access to all tweets"); + static final AllowableValue ENDPOINT_FILTER = new AllowableValue("Filter Endpoint", "Filter Endpoint", "Endpoint that allows the stream to be filtered by specific terms or User IDs"); + + public static final PropertyDescriptor ENDPOINT = new PropertyDescriptor.Builder() + .name("Twitter Endpoint") + .description("Specifies which endpoint data should be pulled from") + .required(true) + .allowableValues(ENDPOINT_SAMPLE, ENDPOINT_FIREHOSE, ENDPOINT_FILTER) + .defaultValue(ENDPOINT_SAMPLE.getValue()) + .build(); + public static final PropertyDescriptor CONSUMER_KEY = new PropertyDescriptor.Builder() + .name("Consumer Key") + .description("The Consumer Key provided by Twitter") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final PropertyDescriptor CONSUMER_SECRET = new PropertyDescriptor.Builder() + .name("Consumer Secret") + .description("The Consumer Secret provided by Twitter") + .required(true) + .sensitive(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final PropertyDescriptor ACCESS_TOKEN = new PropertyDescriptor.Builder() + .name("Access Token") + .description("The Acces Token provided by Twitter") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final PropertyDescriptor ACCESS_TOKEN_SECRET = new PropertyDescriptor.Builder() + .name("Access Token Secret") + .description("The Access Token Secret provided by Twitter") + .required(true) + .sensitive(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final PropertyDescriptor LANGUAGES = new PropertyDescriptor.Builder() + .name("Languages") + .description("A comma-separated list of languages for which tweets should be fetched") + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final PropertyDescriptor FOLLOWING = new PropertyDescriptor.Builder() + .name("IDs to Follow") + .description("A comma-separated list of Twitter User ID's to follow. Ignored unless Endpoint is set to 'Filter Endpoint'.") + .required(false) + .addValidator(new FollowingValidator()) + .build(); + public static final PropertyDescriptor TERMS = new PropertyDescriptor.Builder() + .name("Terms to Filter On") + .description("A comma-separated list of terms to filter on. Ignored unless Endpoint is set to 'Filter Endpoint'. The filter works such that if any term matches, the status update will be retrieved; multiple terms separated by a space function as an 'AND'. I.e., 'it was, hello' will retrieve status updates that have either 'hello' or both 'it' AND 'was'") + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + + public static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("All status updates will be routed to this relationship") + .build(); + + private List<PropertyDescriptor> descriptors; + private Set<Relationship> relationships; + + private final BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<Event>(1000); + + private volatile Client client; + private volatile BlockingQueue<String> messageQueue; + + @Override + protected void init(final ProcessorInitializationContext context) { + final List<PropertyDescriptor> descriptors = new ArrayList<PropertyDescriptor>(); + descriptors.add(ENDPOINT); + descriptors.add(CONSUMER_KEY); + descriptors.add(CONSUMER_SECRET); + descriptors.add(ACCESS_TOKEN); + descriptors.add(ACCESS_TOKEN_SECRET); + descriptors.add(LANGUAGES); + descriptors.add(TERMS); + descriptors.add(FOLLOWING); + this.descriptors = Collections.unmodifiableList(descriptors); + + final Set<Relationship> relationships = new HashSet<Relationship>(); + relationships.add(REL_SUCCESS); + this.relationships = Collections.unmodifiableSet(relationships); + } + + @Override + public Set<Relationship> getRelationships() { + return this.relationships; + } + + @Override + public final List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return descriptors; + } + + @Override + protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { + return new PropertyDescriptor.Builder() + .name(propertyDescriptorName) + .description("Adds a query parameter with name '" + propertyDescriptorName + "' to the Twitter query") + .required(false) + .dynamic(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + } + + @Override + protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) { + final List<ValidationResult> results = new ArrayList<>(); + final String endpointName = validationContext.getProperty(ENDPOINT).getValue(); + + if ( ENDPOINT_FILTER.getValue().equals(endpointName) ) { + if ( !validationContext.getProperty(TERMS).isSet() && !validationContext.getProperty(FOLLOWING).isSet() ) { + results.add(new ValidationResult.Builder().input("").subject(FOLLOWING.getName()).valid(false).explanation("When using the 'Filter Endpoint', at least one of '" + TERMS.getName() + "' or '" + FOLLOWING.getName() + "' must be set").build()); + } + } + + return results; + } + + @Override + public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) { + // if any property is modified, the results are no longer valid. Destroy all messages in teh queue. + messageQueue.clear(); + } + + @OnScheduled + public void onScheduled(final ProcessContext context) throws MalformedURLException { + messageQueue = new LinkedBlockingQueue<>(100000); + + final String endpointName = context.getProperty(ENDPOINT).getValue(); + final Authentication oauth = new OAuth1(context.getProperty(CONSUMER_KEY).getValue(), + context.getProperty(CONSUMER_SECRET).getValue(), + context.getProperty(ACCESS_TOKEN).getValue(), + context.getProperty(ACCESS_TOKEN_SECRET).getValue()); + + final ClientBuilder clientBuilder = new ClientBuilder(); + clientBuilder.name("GetTwitter[id=" + getIdentifier() + "]") + .authentication(oauth) + .eventMessageQueue(eventQueue) + .processor(new StringDelimitedProcessor(messageQueue)); + + final String languageString = context.getProperty(LANGUAGES).getValue(); + final List<String> languages; + if ( languageString == null ) { + languages = null; + } else { + languages = new ArrayList<>(); + for ( final String language : context.getProperty(LANGUAGES).getValue().split(",") ) { + languages.add(language.trim()); + } + } + + final String host; + final StreamingEndpoint streamingEndpoint; + if ( ENDPOINT_SAMPLE.getValue().equals(endpointName) ) { + host = Constants.STREAM_HOST; + final StatusesSampleEndpoint sse = new StatusesSampleEndpoint(); + streamingEndpoint = sse; + if ( languages != null ) { + sse.languages(languages); + } + } else if ( ENDPOINT_FIREHOSE.getValue().equals(endpointName) ) { + host = Constants.STREAM_HOST; + final StatusesFirehoseEndpoint firehoseEndpoint = new StatusesFirehoseEndpoint(); + streamingEndpoint = firehoseEndpoint; + if ( languages != null ) { + firehoseEndpoint.languages(languages); + } + } else if ( ENDPOINT_FILTER.getValue().equals(endpointName) ) { + host = Constants.STREAM_HOST; + final StatusesFilterEndpoint filterEndpoint = new StatusesFilterEndpoint(); + + final String followingString = context.getProperty(FOLLOWING).getValue(); + final List<Long> followingIds; + if ( followingString == null ) { + followingIds = Collections.emptyList(); + } else { + followingIds = new ArrayList<>(); + + for ( final String split : followingString.split(",") ) { + final Long id = Long.parseLong(split.trim()); + followingIds.add(id); + } + } + + final String termString = context.getProperty(TERMS).getValue(); + final List<String> terms; + if ( termString == null ) { + terms = Collections.emptyList(); + } else { + terms = new ArrayList<>(); + for ( final String split : termString.split(",") ) { + terms.add(split.trim()); + } + } + + if ( !terms.isEmpty() ) { + filterEndpoint.trackTerms(terms); + } + + if ( !followingIds.isEmpty() ) { + filterEndpoint.followings(followingIds); + } + + if ( languages != null ) { + filterEndpoint.languages(languages); + } + streamingEndpoint = filterEndpoint; + } else { + throw new AssertionError("Endpoint was invalid value: " + endpointName); + } + + clientBuilder.hosts(host).endpoint(streamingEndpoint); + client = clientBuilder.build(); + client.connect(); + } + + @OnStopped + public void shutdownClient() { + if ( client != null ) { + client.stop(); + } + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + final Event event = eventQueue.poll(); + if ( event != null ) { + switch (event.getEventType()) { + case STOPPED_BY_ERROR: + getLogger().error("Received error {}: {} due to {}. Will not attempt to reconnect", new Object[] {event.getEventType(), event.getMessage(), event.getUnderlyingException()}); + break; + case CONNECTION_ERROR: + case HTTP_ERROR: + getLogger().error("Received error {}: {}. Will attempt to reconnect", new Object[] {event.getEventType(), event.getMessage()}); + client.reconnect(); + break; + default: + break; + } + } + + final String tweet = messageQueue.poll(); + if ( tweet == null ) { + context.yield(); + return; + } + + FlowFile flowFile = session.create(); + flowFile = session.write(flowFile, new OutputStreamCallback() { + @Override + public void process(final OutputStream out) throws IOException { + out.write(tweet.getBytes(StandardCharsets.UTF_8)); + } + }); + + final Map<String, String> attributes = new HashMap<>(); + attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json"); + attributes.put(CoreAttributes.FILENAME.key(), flowFile.getAttribute(CoreAttributes.FILENAME.key()) + ".json"); + flowFile = session.putAllAttributes(flowFile, attributes); + + session.transfer(flowFile, REL_SUCCESS); + session.getProvenanceReporter().receive(flowFile, Constants.STREAM_HOST + client.getEndpoint().getURI().toString()); + } + + private static class FollowingValidator implements Validator { + private static final Pattern NUMBER_PATTERN = Pattern.compile("\\d+"); + + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + final String[] splits = input.split(","); + for ( final String split : splits ) { + if ( !NUMBER_PATTERN.matcher(split.trim()).matches() ) { + return new ValidationResult.Builder().input(input).subject(subject).valid(false).explanation("Must be comma-separted list of User ID's").build(); + } + } + + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + } + + } +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 0000000..9504a11 --- /dev/null +++ b/nifi/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.nifi.processors.twitter.GetTwitter \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-nar-bundles/nifi-social-media-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-social-media-bundle/pom.xml b/nifi/nifi-nar-bundles/nifi-social-media-bundle/pom.xml new file mode 100644 index 0000000..5aadbce --- /dev/null +++ b/nifi/nifi-nar-bundles/nifi-social-media-bundle/pom.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-nar-bundles</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + </parent> + + <artifactId>nifi-social-media-bundle</artifactId> + <packaging>pom</packaging> + + <modules> + <module>nifi-twitter-processors</module> + <module>nifi-social-media-nar</module> + </modules> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/nifi-nar-bundles/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/pom.xml b/nifi/nifi-nar-bundles/pom.xml index e7c122d..50a9407 100644 --- a/nifi/nifi-nar-bundles/pom.xml +++ b/nifi/nifi-nar-bundles/pom.xml @@ -35,6 +35,10 @@ <module>nifi-update-attribute-bundle</module> <module>nifi-kafka-bundle</module> <module>nifi-kite-bundle</module> + <module>nifi-social-media-bundle</module> + <module>nifi-geo-bundle</module> + <module>nifi-hl7-bundle</module> + <module>nifi-language-translation-bundle</module> </modules> <dependencyManagement> <dependencies> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e9cb3b30/nifi/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/pom.xml b/nifi/pom.xml index 2e2346a..9b8bfb4 100644 --- a/nifi/pom.xml +++ b/nifi/pom.xml @@ -800,6 +800,30 @@ </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-social-media-nar</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-hl7-nar</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-language-translation-nar</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-geo-nar</artifactId> + <version>0.1.0-incubating-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-properties</artifactId> <version>0.1.0-incubating-SNAPSHOT</version> </dependency>