NIFI-40: Added new fields to index/search on
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/6ea2c72b Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/6ea2c72b Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/6ea2c72b Branch: refs/heads/prov-query-language Commit: 6ea2c72b981ac9740fc3e046968c439c9199c4a6 Parents: 6e0575c Author: Mark Payne <marka...@hotmail.com> Authored: Fri Mar 13 17:21:50 2015 -0400 Committer: Mark Payne <marka...@hotmail.com> Committed: Fri Mar 13 17:21:50 2015 -0400 ---------------------------------------------------------------------- .../nifi/provenance/SearchableFields.java | 3 +- .../org/apache/nifi/pql/ProvenanceQueryLexer.g | 5 +- .../org/apache/nifi/pql/ProvenanceQueryParser.g | 2 +- .../org/apache/nifi/pql/ProvenanceQuery.java | 126 +++++++------------ .../evaluation/extraction/DetailsEvaluator.java | 39 ++++++ .../java/org/apache/nifi/pql/TestQuery.java | 14 +++ .../journaling/index/LuceneIndexWriter.java | 7 +- 7 files changed, 110 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6ea2c72b/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java b/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java index 97c9880..81c7413 100644 --- a/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java +++ b/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java @@ -36,6 +36,7 @@ public class SearchableFields { public static final SearchableField EventType = new NamedSearchableField("EventType", "eventType", "Event Type", false); public static final SearchableField TransitURI = new NamedSearchableField("TransitURI", "transitUri", "Transit URI", false); public static final SearchableField ComponentID = new NamedSearchableField("ProcessorID", "processorId", "Component ID", false); + public static final SearchableField ComponentType = new NamedSearchableField("ComponentType", "componentType", "Component Type", false); public static final SearchableField AlternateIdentifierURI = new NamedSearchableField("AlternateIdentifierURI", "alternateIdentifierUri", "Alternate Identifier URI", false); public static final SearchableField FileSize = new NamedSearchableField("FileSize", "fileSize", "File Size", false, SearchableFieldType.DATA_SIZE); public static final SearchableField Details = new NamedSearchableField("Details", "details", "Details", false, SearchableFieldType.STRING); @@ -55,7 +56,7 @@ public class SearchableFields { static { final SearchableField[] searchableFields = new SearchableField[]{ EventTime, FlowFileUUID, Filename, EventType, TransitURI, - ComponentID, AlternateIdentifierURI, FileSize, Relationship, Details, + ComponentID, ComponentType, AlternateIdentifierURI, FileSize, Relationship, Details, LineageStartDate, LineageIdentifier, ContentClaimSection, ContentClaimContainer, ContentClaimIdentifier, ContentClaimOffset, SourceQueueIdentifier}; http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6ea2c72b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g index 0815475..4b12c13 100644 --- a/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g +++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g @@ -81,7 +81,6 @@ ASC : 'ASC' | 'asc' | 'Asc'; DESC : 'DESC' | 'desc' | 'Desc'; GROUP_BY : 'GROUP BY' | 'group by' | 'Group By'; EVENT : 'EVENT' | 'event' | 'Event'; -RELATIONSHIP : 'RELATIONSHIP' | 'relationship' | 'Relationship'; // Operators @@ -118,7 +117,11 @@ TIMESTAMP : 'TIME' | 'time' | 'Time'; FILESIZE : 'SIZE' | 'size' | 'Size'; TYPE : 'TYPE' | 'type' | 'Type'; COMPONENT_ID : 'COMPONENTID' | 'componentid' | 'ComponentId' | 'componentId' | 'componentID' | 'ComponentID'; +COMPONENT_TYPE : 'COMPONENTTYPE' | 'componenttype' | 'ComponentType' | 'componentType'; UUID : 'UUID' | 'uuid' | 'Uuid'; +RELATIONSHIP : 'RELATIONSHIP' | 'relationship' | 'Relationship'; +DETAILS : 'DETAILS' | 'details' | 'Details'; + // Event Types RECEIVE : 'RECEIVE' | 'receive' | 'Receive'; http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6ea2c72b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g index 25410bb..88ce3e8 100644 --- a/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g +++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g @@ -75,7 +75,7 @@ query : selectClause selectClause : SELECT^ selectable (COMMA! selectable)*; -selectable : function | (selectableSource ( (DOT! eventProperty^) | (LBRACKET! attribute^ RBRACKET!) )?); +selectable : (function^ | (selectableSource ( (DOT! eventProperty^) | (LBRACKET! attribute^ RBRACKET!) )?)^) (AS IDENTIFIER)?; selectableSource : EVENT | IDENTIFIER; http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6ea2c72b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java index 4dda39b..808ae8b 100644 --- a/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java +++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java @@ -1,42 +1,6 @@ package org.apache.nifi.pql; -import static org.apache.nifi.pql.ProvenanceQueryParser.AND; -import static org.apache.nifi.pql.ProvenanceQueryParser.ASC; -import static org.apache.nifi.pql.ProvenanceQueryParser.ATTRIBUTE; -import static org.apache.nifi.pql.ProvenanceQueryParser.AVG; -import static org.apache.nifi.pql.ProvenanceQueryParser.COMPONENT_ID; -import static org.apache.nifi.pql.ProvenanceQueryParser.COUNT; -import static org.apache.nifi.pql.ProvenanceQueryParser.DAY; -import static org.apache.nifi.pql.ProvenanceQueryParser.EQUALS; -import static org.apache.nifi.pql.ProvenanceQueryParser.EVENT; -import static org.apache.nifi.pql.ProvenanceQueryParser.EVENT_PROPERTY; -import static org.apache.nifi.pql.ProvenanceQueryParser.FILESIZE; -import static org.apache.nifi.pql.ProvenanceQueryParser.FROM; -import static org.apache.nifi.pql.ProvenanceQueryParser.GROUP_BY; -import static org.apache.nifi.pql.ProvenanceQueryParser.GT; -import static org.apache.nifi.pql.ProvenanceQueryParser.HOUR; -import static org.apache.nifi.pql.ProvenanceQueryParser.IDENTIFIER; -import static org.apache.nifi.pql.ProvenanceQueryParser.LIMIT; -import static org.apache.nifi.pql.ProvenanceQueryParser.LT; -import static org.apache.nifi.pql.ProvenanceQueryParser.MATCHES; -import static org.apache.nifi.pql.ProvenanceQueryParser.MINUTE; -import static org.apache.nifi.pql.ProvenanceQueryParser.MONTH; -import static org.apache.nifi.pql.ProvenanceQueryParser.NOT; -import static org.apache.nifi.pql.ProvenanceQueryParser.NOT_EQUALS; -import static org.apache.nifi.pql.ProvenanceQueryParser.NUMBER; -import static org.apache.nifi.pql.ProvenanceQueryParser.OR; -import static org.apache.nifi.pql.ProvenanceQueryParser.ORDER_BY; -import static org.apache.nifi.pql.ProvenanceQueryParser.RELATIONSHIP; -import static org.apache.nifi.pql.ProvenanceQueryParser.SECOND; -import static org.apache.nifi.pql.ProvenanceQueryParser.STARTS_WITH; -import static org.apache.nifi.pql.ProvenanceQueryParser.STRING_LITERAL; -import static org.apache.nifi.pql.ProvenanceQueryParser.SUM; -import static org.apache.nifi.pql.ProvenanceQueryParser.TIMESTAMP; -import static org.apache.nifi.pql.ProvenanceQueryParser.TRANSIT_URI; -import static org.apache.nifi.pql.ProvenanceQueryParser.TYPE; -import static org.apache.nifi.pql.ProvenanceQueryParser.UUID; -import static org.apache.nifi.pql.ProvenanceQueryParser.WHERE; -import static org.apache.nifi.pql.ProvenanceQueryParser.YEAR; +import static org.apache.nifi.pql.ProvenanceQueryParser.*; import java.io.IOException; import java.util.ArrayList; @@ -72,6 +36,8 @@ import org.apache.nifi.pql.evaluation.comparison.StartsWithEvaluator; import org.apache.nifi.pql.evaluation.conversion.StringToLongEvaluator; import org.apache.nifi.pql.evaluation.extraction.AttributeEvaluator; import org.apache.nifi.pql.evaluation.extraction.ComponentIdEvaluator; +import org.apache.nifi.pql.evaluation.extraction.ComponentTypeEvaluator; +import org.apache.nifi.pql.evaluation.extraction.DetailsEvaluator; import org.apache.nifi.pql.evaluation.extraction.RelationshipEvaluator; import org.apache.nifi.pql.evaluation.extraction.SizeEvaluator; import org.apache.nifi.pql.evaluation.extraction.TimestampEvaluator; @@ -140,7 +106,16 @@ public class ProvenanceQuery { private ProvenanceQuery(final Tree tree, final String pql, final Collection<SearchableField> searchableFields, final Collection<SearchableField> searchableAttributes) { this.tree = tree; this.pql = pql; - this.searchableFields = searchableFields == null ? null : Collections.unmodifiableSet(new HashSet<>(searchableFields)); + + if ( searchableFields == null ) { + this.searchableFields = null; + } else { + final Set<SearchableField> addressableFields = new HashSet<>(searchableFields); + addressableFields.add(SearchableFields.EventTime); + addressableFields.add(SearchableFields.EventType); + this.searchableFields = Collections.unmodifiableSet(addressableFields); + } + if (searchableAttributes == null) { this.searchableAttributes = null; } else { @@ -296,6 +271,13 @@ public class ProvenanceQuery { } private String getLabel(final Tree tree) { + if ( tree.getChildCount() > 0 ) { + final Tree childTree = tree.getChild(tree.getChildCount() - 1); + if ( childTree.getType() == AS ) { + return childTree.getChild(0).getText(); + } + } + final int type = tree.getType(); switch (type) { @@ -313,6 +295,17 @@ public class ProvenanceQuery { return tree.getText(); } + + private void ensureSearchable(final SearchableField field, final boolean addToReferencedFields) { + if ( searchableFields != null && !searchableFields.contains(field) ) { + throw new ProvenanceQueryLanguageException("Query cannot reference " + field.getFriendlyName() + " because this field is not searchable by the repository"); + } + if ( addToReferencedFields ) { + referencedFields.add(field.getSearchableFieldName()); + } + } + + private OperandEvaluator<?> buildOperandEvaluator(final Tree tree, final Clause clause) { // When events are pulled back from an index, for efficiency purposes, we may want to know which // fields to pull back. The fields in the WHERE clause are irrelevant because they are not shown @@ -323,61 +316,32 @@ public class ProvenanceQuery { case EVENT_PROPERTY: switch (tree.getChild(0).getType()) { case FILESIZE: - if ( searchableFields != null && !searchableFields.contains(SearchableFields.FileSize) ) { - throw new ProvenanceQueryLanguageException("Query cannot reference FileSize because this field is not searchable by the repository"); - } - if ( isReferenceInteresting ) { - referencedFields.add(SearchableFields.FileSize.getSearchableFieldName()); - } + ensureSearchable(SearchableFields.FileSize, isReferenceInteresting); return new SizeEvaluator(); case TRANSIT_URI: - if ( searchableFields != null && !searchableFields.contains(SearchableFields.TransitURI) ) { - throw new ProvenanceQueryLanguageException("Query cannot reference TransitURI because this field is not searchable by the repository"); - } - if ( isReferenceInteresting ) { - referencedFields.add(SearchableFields.TransitURI.getSearchableFieldName()); - } + ensureSearchable(SearchableFields.TransitURI, isReferenceInteresting); return new TransitUriEvaluator(); case TIMESTAMP: - // time is always indexed - if ( isReferenceInteresting ) { - referencedFields.add(SearchableFields.EventTime.getSearchableFieldName()); - } + ensureSearchable(SearchableFields.EventTime, isReferenceInteresting); return new TimestampEvaluator(); case TYPE: - // type is always indexed so no need to check it - if ( isReferenceInteresting ) { - referencedFields.add(SearchableFields.EventType.getSearchableFieldName()); - } - return new TypeEvaluator(); + ensureSearchable(SearchableFields.EventType, isReferenceInteresting); + return new TypeEvaluator(); case COMPONENT_ID: - if ( searchableFields != null && !searchableFields.contains(SearchableFields.ComponentID) ) { - throw new ProvenanceQueryLanguageException("Query cannot reference Component ID because this field is not searchable by the repository"); - } - if ( isReferenceInteresting ) { - referencedFields.add(SearchableFields.ComponentID.getSearchableFieldName()); - } - + ensureSearchable(SearchableFields.ComponentID, isReferenceInteresting); return new ComponentIdEvaluator(); - // TODO: Allow Component Type to be indexed and searched + case COMPONENT_TYPE: + ensureSearchable(SearchableFields.ComponentType, isReferenceInteresting); + return new ComponentTypeEvaluator(); case RELATIONSHIP: - if ( searchableFields != null && !searchableFields.contains(SearchableFields.Relationship) ) { - throw new ProvenanceQueryLanguageException("Query cannot reference Relationship because this field is not searchable by the repository"); - } - if ( isReferenceInteresting ) { - referencedFields.add(SearchableFields.Relationship.getSearchableFieldName()); - } - + ensureSearchable(SearchableFields.Relationship, isReferenceInteresting); return new RelationshipEvaluator(); case UUID: - if ( searchableFields != null && !searchableFields.contains(SearchableFields.FlowFileUUID) ) { - throw new ProvenanceQueryLanguageException("Query cannot reference FlowFile UUID because this field is not searchable by the repository"); - } - if ( isReferenceInteresting ) { - referencedFields.add(SearchableFields.FlowFileUUID.getSearchableFieldName()); - } - + ensureSearchable(SearchableFields.FlowFileUUID, isReferenceInteresting); return new UuidEvaluator(); + case DETAILS: + ensureSearchable(SearchableFields.Details, isReferenceInteresting); + return new DetailsEvaluator(); default: // TODO: IMPLEMENT throw new UnsupportedOperationException("Haven't implemented extraction of property " + tree.getChild(0).getText()); http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6ea2c72b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/evaluation/extraction/DetailsEvaluator.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/evaluation/extraction/DetailsEvaluator.java b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/evaluation/extraction/DetailsEvaluator.java new file mode 100644 index 0000000..de69316 --- /dev/null +++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/evaluation/extraction/DetailsEvaluator.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.pql.evaluation.extraction; + +import org.apache.nifi.pql.evaluation.OperandEvaluator; +import org.apache.nifi.provenance.ProvenanceEventRecord; + +public class DetailsEvaluator implements OperandEvaluator<String> { + + @Override + public String evaluate(final ProvenanceEventRecord record) { + return record.getDetails(); + } + + @Override + public int getEvaluatorType() { + return org.apache.nifi.pql.ProvenanceQueryParser.DETAILS; + } + + @Override + public Class<String> getType() { + return String.class; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6ea2c72b/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java b/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java index 3317c21..e640419 100644 --- a/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java +++ b/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java @@ -19,6 +19,7 @@ import org.apache.nifi.provenance.query.ProvenanceResultSet; import org.apache.nifi.util.NiFiProperties; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class TestQuery { @@ -117,6 +118,19 @@ public class TestQuery { dump(ProvenanceQuery.execute("SELECT Event['filename'], SUM(Event.size) GROUP BY Event['filename']", repo)); } + @Test + @Ignore("Not entirely implemented yet") + public void testAlias() throws IOException { + createRecords(); + final ProvenanceQuery query = ProvenanceQuery.compile("SELECT SUM(Event.Size) AS TotalSize, COUNT(Event) AS NumEvents", null, null); + + final ProvenanceResultSet rs = query.execute(repo); + dump(rs); + + assertEquals(2, rs.getLabels().size()); + assertEquals("TotalSize", rs.getLabels().get(0)); + assertEquals("NumEvents", rs.getLabels().get(1)); + } @Test public void testGroupBy() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6ea2c72b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java index 1f84891..4c14c05 100644 --- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java +++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java @@ -147,6 +147,7 @@ public class LuceneIndexWriter implements EventIndexWriter { addField(doc, SearchableFields.FlowFileUUID, event.getFlowFileUuid(), STORE_FIELDS); addField(doc, SearchableFields.Filename, attributes.get(CoreAttributes.FILENAME.key()), STORE_FIELDS); addField(doc, SearchableFields.ComponentID, event.getComponentId(), STORE_FIELDS); + addField(doc, SearchableFields.ComponentType, event.getComponentType(), STORE_FIELDS); addField(doc, SearchableFields.AlternateIdentifierURI, event.getAlternateIdentifierUri(), STORE_FIELDS); addField(doc, SearchableFields.EventType, event.getEventType().name(), STORE_FIELDS); addField(doc, SearchableFields.Relationship, event.getRelationship(), STORE_FIELDS); @@ -176,8 +177,10 @@ public class LuceneIndexWriter implements EventIndexWriter { doc.add(new LongField(IndexedFieldNames.BLOCK_INDEX, location.getBlockIndex(), Store.YES)); doc.add(new LongField(IndexedFieldNames.EVENT_ID, location.getEventId(), Store.YES)); - for (final String lineageIdentifier : event.getLineageIdentifiers()) { - addField(doc, SearchableFields.LineageIdentifier, lineageIdentifier, STORE_FIELDS); + if ( nonAttributeSearchableFields.contains(SearchableFields.LineageIdentifier) ) { + for (final String lineageIdentifier : event.getLineageIdentifiers()) { + addField(doc, SearchableFields.LineageIdentifier, lineageIdentifier, STORE_FIELDS); + } } // If it's event is a FORK, or JOIN, add the FlowFileUUID for all child/parent UUIDs.
