NIFI-40: Added new fields to index/search on
diff --git a/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java b/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
index 97c9880..81c7413 100644
--- a/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
+++ b/nifi/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
@@ -36,6 +36,7 @@
public static final SearchableField EventType = new NamedSearchableField("EventType", "eventType", "Event Type", false);
public static final SearchableField TransitURI = new NamedSearchableField("TransitURI", "transitUri", "Transit URI", false);
public static final SearchableField ComponentID = new NamedSearchableField("ProcessorID", "processorId", "Component ID", false);
+ public static final SearchableField ComponentType = new NamedSearchableField("ComponentType", "componentType", "Component Type", false);
public static final SearchableField AlternateIdentifierURI = new NamedSearchableField("AlternateIdentifierURI", "alternateIdentifierUri", "Alternate Identifier URI", false);
public static final SearchableField FileSize = new NamedSearchableField("FileSize", "fileSize", "File Size", false, SearchableFieldType.DATA_SIZE);
public static final SearchableField Details = new NamedSearchableField("Details", "details", "Details", false, SearchableFieldType.STRING);
@@ -55,7 +56,7 @@
static {
final SearchableField[] searchableFields = new SearchableField[]{
EventTime, FlowFileUUID, Filename, EventType, TransitURI,
- ComponentID, AlternateIdentifierURI, FileSize, Relationship, Details,
+ ComponentID, ComponentType, AlternateIdentifierURI, FileSize, Relationship, Details,
LineageStartDate, LineageIdentifier, ContentClaimSection, ContentClaimContainer, ContentClaimIdentifier,
ContentClaimOffset, SourceQueueIdentifier};
diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g
index 0815475..4b12c13 100644
--- a/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g
+++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryLexer.g
@@ -81,7 +81,6 @@
DESC : 'DESC' | 'desc' | 'Desc';
GROUP_BY : 'GROUP BY' | 'group by' | 'Group By';
EVENT : 'EVENT' | 'event' | 'Event';
-RELATIONSHIP : 'RELATIONSHIP' | 'relationship' | 'Relationship';
// Operators
@@ -118,7 +117,11 @@
FILESIZE : 'SIZE' | 'size' | 'Size';
TYPE : 'TYPE' | 'type' | 'Type';
COMPONENT_ID : 'COMPONENTID' | 'componentid' | 'ComponentId' | 'componentId' | 'componentID' | 'ComponentID';
+COMPONENT_TYPE : 'COMPONENTTYPE' | 'componenttype' | 'ComponentType' | 'componentType';
UUID : 'UUID' | 'uuid' | 'Uuid';
+RELATIONSHIP : 'RELATIONSHIP' | 'relationship' | 'Relationship';
+DETAILS : 'DETAILS' | 'details' | 'Details';
+
// Event Types
RECEIVE : 'RECEIVE' | 'receive' | 'Receive';
diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g
index 25410bb..88ce3e8 100644
--- a/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g
+++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/antlr3/org/apache/nifi/pql/ProvenanceQueryParser.g
@@ -75,7 +75,7 @@
selectClause : SELECT^ selectable (COMMA! selectable)*;
-selectable : function | (selectableSource ( (DOT! eventProperty^) | (LBRACKET! attribute^ RBRACKET!) )?);
+selectable : (function^ | (selectableSource ( (DOT! eventProperty^) | (LBRACKET! attribute^ RBRACKET!) )?)^) (AS IDENTIFIER)?;
selectableSource : EVENT | IDENTIFIER;
diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java
index 4dda39b..808ae8b 100644
--- a/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java
+++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/ProvenanceQuery.java
@@ -1,42 +1,6 @@
package org.apache.nifi.pql;
-import static org.apache.nifi.pql.ProvenanceQueryParser.AND;
-import static org.apache.nifi.pql.ProvenanceQueryParser.ASC;
-import static org.apache.nifi.pql.ProvenanceQueryParser.ATTRIBUTE;
-import static org.apache.nifi.pql.ProvenanceQueryParser.AVG;
-import static org.apache.nifi.pql.ProvenanceQueryParser.COMPONENT_ID;
-import static org.apache.nifi.pql.ProvenanceQueryParser.COUNT;
-import static org.apache.nifi.pql.ProvenanceQueryParser.DAY;
-import static org.apache.nifi.pql.ProvenanceQueryParser.EQUALS;
-import static org.apache.nifi.pql.ProvenanceQueryParser.EVENT;
-import static org.apache.nifi.pql.ProvenanceQueryParser.EVENT_PROPERTY;
-import static org.apache.nifi.pql.ProvenanceQueryParser.FILESIZE;
-import static org.apache.nifi.pql.ProvenanceQueryParser.FROM;
-import static org.apache.nifi.pql.ProvenanceQueryParser.GROUP_BY;
-import static org.apache.nifi.pql.ProvenanceQueryParser.GT;
-import static org.apache.nifi.pql.ProvenanceQueryParser.HOUR;
-import static org.apache.nifi.pql.ProvenanceQueryParser.IDENTIFIER;
-import static org.apache.nifi.pql.ProvenanceQueryParser.LIMIT;
-import static org.apache.nifi.pql.ProvenanceQueryParser.LT;
-import static org.apache.nifi.pql.ProvenanceQueryParser.MATCHES;
-import static org.apache.nifi.pql.ProvenanceQueryParser.MINUTE;
-import static org.apache.nifi.pql.ProvenanceQueryParser.MONTH;
-import static org.apache.nifi.pql.ProvenanceQueryParser.NOT;
-import static org.apache.nifi.pql.ProvenanceQueryParser.NOT_EQUALS;
-import static org.apache.nifi.pql.ProvenanceQueryParser.NUMBER;
-import static org.apache.nifi.pql.ProvenanceQueryParser.OR;
-import static org.apache.nifi.pql.ProvenanceQueryParser.ORDER_BY;
-import static org.apache.nifi.pql.ProvenanceQueryParser.RELATIONSHIP;
-import static org.apache.nifi.pql.ProvenanceQueryParser.SECOND;
-import static org.apache.nifi.pql.ProvenanceQueryParser.STARTS_WITH;
-import static org.apache.nifi.pql.ProvenanceQueryParser.STRING_LITERAL;
-import static org.apache.nifi.pql.ProvenanceQueryParser.SUM;
-import static org.apache.nifi.pql.ProvenanceQueryParser.TIMESTAMP;
-import static org.apache.nifi.pql.ProvenanceQueryParser.TRANSIT_URI;
-import static org.apache.nifi.pql.ProvenanceQueryParser.TYPE;
-import static org.apache.nifi.pql.ProvenanceQueryParser.UUID;
-import static org.apache.nifi.pql.ProvenanceQueryParser.WHERE;
-import static org.apache.nifi.pql.ProvenanceQueryParser.YEAR;
+import static org.apache.nifi.pql.ProvenanceQueryParser.*;
import java.io.IOException;
import java.util.ArrayList;
@@ -72,6 +36,8 @@
import org.apache.nifi.pql.evaluation.conversion.StringToLongEvaluator;
import org.apache.nifi.pql.evaluation.extraction.AttributeEvaluator;
import org.apache.nifi.pql.evaluation.extraction.ComponentIdEvaluator;
+import org.apache.nifi.pql.evaluation.extraction.ComponentTypeEvaluator;
+import org.apache.nifi.pql.evaluation.extraction.DetailsEvaluator;
import org.apache.nifi.pql.evaluation.extraction.RelationshipEvaluator;
import org.apache.nifi.pql.evaluation.extraction.SizeEvaluator;
import org.apache.nifi.pql.evaluation.extraction.TimestampEvaluator;
@@ -140,7 +106,16 @@
private ProvenanceQuery(final Tree tree, final String pql, final Collection<SearchableField> searchableFields, final Collection<SearchableField> searchableAttributes) {
this.tree = tree;
this.pql = pql;
- this.searchableFields = searchableFields == null ? null : Collections.unmodifiableSet(new HashSet<>(searchableFields));
+
+ if ( searchableFields == null ) {
+ this.searchableFields = null;
+ } else {
+ final Set<SearchableField> addressableFields = new HashSet<>(searchableFields);
+ addressableFields.add(SearchableFields.EventTime);
+ addressableFields.add(SearchableFields.EventType);
+ this.searchableFields = Collections.unmodifiableSet(addressableFields);
+ }
+
if (searchableAttributes == null) {
this.searchableAttributes = null;
} else {
@@ -296,6 +271,13 @@
}
private String getLabel(final Tree tree) {
+ if ( tree.getChildCount() > 0 ) {
+ final Tree childTree = tree.getChild(tree.getChildCount() - 1);
+ if ( childTree.getType() == AS ) {
+ return childTree.getChild(0).getText();
+ }
+ }
+
final int type = tree.getType();
switch (type) {
@@ -313,6 +295,17 @@
return tree.getText();
}
+
+ private void ensureSearchable(final SearchableField field, final boolean addToReferencedFields) {
+ if ( searchableFields != null && !searchableFields.contains(field) ) {
+ throw new ProvenanceQueryLanguageException("Query cannot reference " + field.getFriendlyName() + " because this field is not searchable by the repository");
+ }
+ if ( addToReferencedFields ) {
+ referencedFields.add(field.getSearchableFieldName());
+ }
+ }
+
+
private OperandEvaluator<?> buildOperandEvaluator(final Tree tree, final Clause clause) {
// When events are pulled back from an index, for efficiency purposes, we may want to know which
// fields to pull back. The fields in the WHERE clause are irrelevant because they are not shown
@@ -323,61 +316,32 @@
case EVENT_PROPERTY:
switch (tree.getChild(0).getType()) {
case FILESIZE:
- if ( searchableFields != null && !searchableFields.contains(SearchableFields.FileSize) ) {
- throw new ProvenanceQueryLanguageException("Query cannot reference FileSize because this field is not searchable by the repository");
- }
- if ( isReferenceInteresting ) {
- referencedFields.add(SearchableFields.FileSize.getSearchableFieldName());
- }
+ ensureSearchable(SearchableFields.FileSize, isReferenceInteresting);
return new SizeEvaluator();
case TRANSIT_URI:
- if ( searchableFields != null && !searchableFields.contains(SearchableFields.TransitURI) ) {
- throw new ProvenanceQueryLanguageException("Query cannot reference TransitURI because this field is not searchable by the repository");
- }
- if ( isReferenceInteresting ) {
- referencedFields.add(SearchableFields.TransitURI.getSearchableFieldName());
- }
+ ensureSearchable(SearchableFields.TransitURI, isReferenceInteresting);
return new TransitUriEvaluator();
case TIMESTAMP:
- // time is always indexed
- if ( isReferenceInteresting ) {
- referencedFields.add(SearchableFields.EventTime.getSearchableFieldName());
- }
+ ensureSearchable(SearchableFields.EventTime, isReferenceInteresting);
return new TimestampEvaluator();
case TYPE:
- // type is always indexed so no need to check it
- if ( isReferenceInteresting ) {
- referencedFields.add(SearchableFields.EventType.getSearchableFieldName());
- }
- return new TypeEvaluator();
+ ensureSearchable(SearchableFields.EventType, isReferenceInteresting);
+ return new TypeEvaluator();
case COMPONENT_ID:
- if ( searchableFields != null && !searchableFields.contains(SearchableFields.ComponentID) ) {
- throw new ProvenanceQueryLanguageException("Query cannot reference Component ID because this field is not searchable by the repository");
- }
- if ( isReferenceInteresting ) {
- referencedFields.add(SearchableFields.ComponentID.getSearchableFieldName());
- }
-
+ ensureSearchable(SearchableFields.ComponentID, isReferenceInteresting);
return new ComponentIdEvaluator();
- // TODO: Allow Component Type to be indexed and searched
+ case COMPONENT_TYPE:
+ ensureSearchable(SearchableFields.ComponentType, isReferenceInteresting);
+ return new ComponentTypeEvaluator();
case RELATIONSHIP:
- if ( searchableFields != null && !searchableFields.contains(SearchableFields.Relationship) ) {
- throw new ProvenanceQueryLanguageException("Query cannot reference Relationship because this field is not searchable by the repository");
- }
- if ( isReferenceInteresting ) {
- referencedFields.add(SearchableFields.Relationship.getSearchableFieldName());
- }
-
+ ensureSearchable(SearchableFields.Relationship, isReferenceInteresting);
return new RelationshipEvaluator();
case UUID:
- if ( searchableFields != null && !searchableFields.contains(SearchableFields.FlowFileUUID) ) {
- throw new ProvenanceQueryLanguageException("Query cannot reference FlowFile UUID because this field is not searchable by the repository");
- }
- if ( isReferenceInteresting ) {
- referencedFields.add(SearchableFields.FlowFileUUID.getSearchableFieldName());
- }
-
+ ensureSearchable(SearchableFields.FlowFileUUID, isReferenceInteresting);
return new UuidEvaluator();
+ case DETAILS:
+ ensureSearchable(SearchableFields.Details, isReferenceInteresting);
+ return new DetailsEvaluator();
default:
// TODO: IMPLEMENT
throw new UnsupportedOperationException("Haven't implemented extraction of property " + tree.getChild(0).getText());
diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/evaluation/extraction/DetailsEvaluator.java b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/evaluation/extraction/DetailsEvaluator.java
new file mode 100644
index 0000000..de69316
--- /dev/null
+++ b/nifi/nifi-commons/nifi-provenance-query-language/src/main/java/org/apache/nifi/pql/evaluation/extraction/DetailsEvaluator.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.pql.evaluation.extraction;
+
+import org.apache.nifi.pql.evaluation.OperandEvaluator;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+
+public class DetailsEvaluator implements OperandEvaluator<String> {
+
+ @Override
+ public String evaluate(final ProvenanceEventRecord record) {
+ return record.getDetails();
+ }
+
+ @Override
+ public int getEvaluatorType() {
+ return org.apache.nifi.pql.ProvenanceQueryParser.DETAILS;
+ }
+
+ @Override
+ public Class<String> getType() {
+ return String.class;
+ }
+
+}
diff --git a/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java b/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java
index 3317c21..e640419 100644
--- a/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java
+++ b/nifi/nifi-commons/nifi-provenance-query-language/src/test/java/org/apache/nifi/pql/TestQuery.java
@@ -19,6 +19,7 @@
import org.apache.nifi.util.NiFiProperties;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
public class TestQuery {
@@ -117,6 +118,19 @@
dump(ProvenanceQuery.execute("SELECT Event['filename'], SUM(Event.size) GROUP BY Event['filename']", repo));
}
+ @Test
+ @Ignore("Not entirely implemented yet")
+ public void testAlias() throws IOException {
+ createRecords();
+ final ProvenanceQuery query = ProvenanceQuery.compile("SELECT SUM(Event.Size) AS TotalSize, COUNT(Event) AS NumEvents", null, null);
+
+ final ProvenanceResultSet rs = query.execute(repo);
+ dump(rs);
+
+ assertEquals(2, rs.getLabels().size());
+ assertEquals("TotalSize", rs.getLabels().get(0));
+ assertEquals("NumEvents", rs.getLabels().get(1));
+ }
@Test
public void testGroupBy() throws IOException {
diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java
index 1f84891..4c14c05 100644
--- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java
+++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-journaling-provenance-repository/src/main/java/org/apache/nifi/provenance/journaling/index/LuceneIndexWriter.java
@@ -147,6 +147,7 @@
addField(doc, SearchableFields.FlowFileUUID, event.getFlowFileUuid(), STORE_FIELDS);
addField(doc, SearchableFields.Filename, attributes.get(CoreAttributes.FILENAME.key()), STORE_FIELDS);
addField(doc, SearchableFields.ComponentID, event.getComponentId(), STORE_FIELDS);
+ addField(doc, SearchableFields.ComponentType, event.getComponentType(), STORE_FIELDS);
addField(doc, SearchableFields.AlternateIdentifierURI, event.getAlternateIdentifierUri(), STORE_FIELDS);
addField(doc, SearchableFields.EventType, event.getEventType().name(), STORE_FIELDS);
addField(doc, SearchableFields.Relationship, event.getRelationship(), STORE_FIELDS);
@@ -176,8 +177,10 @@
doc.add(new LongField(IndexedFieldNames.BLOCK_INDEX, location.getBlockIndex(), Store.YES));
doc.add(new LongField(IndexedFieldNames.EVENT_ID, location.getEventId(), Store.YES));
- for (final String lineageIdentifier : event.getLineageIdentifiers()) {
- addField(doc, SearchableFields.LineageIdentifier, lineageIdentifier, STORE_FIELDS);
+ if ( nonAttributeSearchableFields.contains(SearchableFields.LineageIdentifier) ) {
+ for (final String lineageIdentifier : event.getLineageIdentifiers()) {
+ addField(doc, SearchableFields.LineageIdentifier, lineageIdentifier, STORE_FIELDS);
+ }
}
// If it's event is a FORK, or JOIN, add the FlowFileUUID for all child/parent UUIDs.
