ATLAS-2775: Enhance basic search to search on entities with any classifications or no classifications
diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
index 39ebf23..aac6b5a 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
@@ -55,6 +55,10 @@
private FilterCriteria tagFilters;
private Set<String> attributes;
+ public static final String WILDCARD_CLASSIFICATIONS = "*";
+ public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED";
+ public static final String NO_CLASSIFICATIONS = "_NOT_CLASSIFIED";
+
/**
* @return The type of query
*/
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 588ad3e..c00bfb6 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -50,6 +50,12 @@
import java.util.Map;
import java.util.Set;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
+import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
@@ -88,7 +94,9 @@
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
// for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
- boolean useIndexSearch = classificationType != SearchContext.MATCH_ALL_CLASSIFICATION &&
+ boolean useIndexSearch = classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION &&
+ classificationType != MATCH_ALL_CLASSIFIED &&
+ classificationType != MATCH_ALL_NOT_CLASSIFIED &&
typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS &&
CollectionUtils.isEmpty(graphAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false);
@@ -149,25 +157,32 @@
LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
}
} else {
- tagGraphQueryWithAttributes = null;
- List<AtlasGraphQuery> orConditions = new LinkedList<>();
+ tagGraphQueryWithAttributes = null;
+ List<AtlasGraphQuery> orConditions = new LinkedList<>();
- if (classificationType != SearchContext.MATCH_ALL_CLASSIFICATION) {
- orConditions.add(graph.query().createChildQuery().in(Constants.TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
- orConditions.add(graph.query().createChildQuery().in(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
+ if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
+ orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
+ orConditions.add(graph.query().createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.orPredicate(
- SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class),
- SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class));
+ SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
+ SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
+ orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null).has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
+
+ entityGraphQueryTraitNames = graph.query().or(orConditions);
+ entityPredicateTraitNames = PredicateUtils.andPredicate(
+ SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
+ SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else {
- orConditions.add(graph.query().createChildQuery().has(Constants.TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
- orConditions.add(graph.query().createChildQuery().has(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
+ orConditions.add(graph.query().createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
+ orConditions.add(graph.query().createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.orPredicate(
- SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, null, List.class),
- SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class),
+ SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class));
}
if (context.getSearchParameters().getExcludeDeletedEntities()) {
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index abe859f..c4b5ffb 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -40,6 +40,12 @@
import java.util.List;
import java.util.Set;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
+import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
public class EntitySearchProcessor extends SearchProcessor {
@@ -93,12 +99,15 @@
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
final Predicate traitPredicate;
- if (classificationType == SearchContext.MATCH_ALL_CLASSIFICATION) {
- traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, null, List.class),
- SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
+ traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
+ SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
+ traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
+ SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else {
- traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class),
- SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
+ traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class),
+ SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
}
processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
@@ -154,12 +163,15 @@
if (filterClassification) {
List<AtlasGraphQuery> orConditions = new LinkedList<>();
- if (classificationType == SearchContext.MATCH_ALL_CLASSIFICATION) {
- orConditions.add(query.createChildQuery().has(Constants.TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
- orConditions.add(query.createChildQuery().has(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
+ if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
+ orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
+ orConditions.add(query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
+ } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
+ orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null)
+ .has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
} else {
- orConditions.add(query.createChildQuery().in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
- orConditions.add(query.createChildQuery().in(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
+ orConditions.add(query.createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
+ orConditions.add(query.createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
}
query.or(orConditions);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 2c84369..0ffd61c 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -31,6 +31,10 @@
import java.util.Iterator;
import java.util.List;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
+
public class FullTextSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(FullTextSearchProcessor.class);
@@ -61,7 +65,9 @@
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
- if (context.getClassificationType() != null && context.getClassificationType() != SearchContext.MATCH_ALL_CLASSIFICATION) {
+ if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION &&
+ context.getClassificationType() != MATCH_ALL_CLASSIFIED &&
+ context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) {
String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 1f40076..b4542b7 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -45,6 +45,10 @@
import java.util.List;
import java.util.Set;
+import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
+import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
+import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
+
/*
* Search context captures elements required for performing a basic search
* For every search request the search context will determine the execution sequence of the search processor(s) and the
@@ -61,7 +65,9 @@
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
- public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef("*"));
+ public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
+ public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
+ public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
String classificationName = searchParameters.getClassification();
@@ -199,8 +205,12 @@
private AtlasClassificationType getClassificationType(String classificationName) {
AtlasClassificationType ret;
- if (StringUtils.equals(classificationName, MATCH_ALL_CLASSIFICATION.getTypeName())) {
- ret = MATCH_ALL_CLASSIFICATION;
+ if (StringUtils.equals(classificationName, MATCH_ALL_WILDCARD_CLASSIFICATION.getTypeName())) {
+ ret = MATCH_ALL_WILDCARD_CLASSIFICATION;
+ } else if (StringUtils.equals(classificationName, MATCH_ALL_CLASSIFIED.getTypeName())) {
+ ret = MATCH_ALL_CLASSIFIED;
+ } else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) {
+ ret = MATCH_ALL_NOT_CLASSIFIED;
} else {
ret = typeRegistry.getClassificationTypeByName(classificationName);
}
