Merge pull request #118 from apache/UNOMI-225-es7-fix-counts
UNOMI-225 ElasticSearch 7 fix for changes in the total hits
diff --git a/api/src/main/java/org/apache/unomi/api/PartialList.java b/api/src/main/java/org/apache/unomi/api/PartialList.java
index a78eb73..2128f93 100644
--- a/api/src/main/java/org/apache/unomi/api/PartialList.java
+++ b/api/src/main/java/org/apache/unomi/api/PartialList.java
@@ -35,10 +35,21 @@
private long offset;
private long pageSize;
private long totalSize;
+ private Relation totalSizeRelation;
private String scrollIdentifier = null;
private String scrollTimeValidity = null;
/**
+ * This enum exists to replicate Lucene's total hits relation in a back-end agnostic way. Basically Lucene will
+ * by default not report accurate total hit counts above a certain threshold for performance reasons. Using the
+ * relation we can understand if we are in the case of an accurate hit or not.
+ */
+ public enum Relation {
+ EQUAL,
+ GREATER_THAN_OR_EQUAL_TO
+ }
+
+ /**
* Instantiates a new PartialList.
*/
public PartialList() {
@@ -46,6 +57,7 @@
offset = 0;
pageSize = 0;
totalSize = 0;
+ totalSizeRelation = Relation.EQUAL;
}
/**
@@ -56,11 +68,12 @@
* @param pageSize the number of elements this PartialList contains
* @param totalSize the total size of elements in the original List
*/
- public PartialList(List<T> list, long offset, long pageSize, long totalSize) {
+ public PartialList(List<T> list, long offset, long pageSize, long totalSize, Relation totalSizeRelation) {
this.list = list;
this.offset = offset;
this.pageSize = pageSize;
this.totalSize = totalSize;
+ this.totalSizeRelation = totalSizeRelation;
}
/**
@@ -164,4 +177,17 @@
public void setScrollTimeValidity(String scrollTimeValidity) {
this.scrollTimeValidity = scrollTimeValidity;
}
+
+ /**
+ * Retrieve the relation to the total site, wether it is equal to or greater than the value stored in the
+ * totalSize property.
+ * @return a Relation enum value that describes the type of total size we have in this object.
+ */
+ public Relation getTotalSizeRelation() {
+ return totalSizeRelation;
+ }
+
+ public void setTotalSizeRelation(Relation totalSizeRelation) {
+ this.totalSizeRelation = totalSizeRelation;
+ }
}
diff --git a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
index e60cb19..237a52a 100644
--- a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
+++ b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
@@ -51,7 +51,7 @@
for (UserList definition : userLists.getList()) {
metadata.add(definition.getMetadata());
}
- return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize());
+ return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize(), userLists.getTotalSizeRelation());
}
public PartialList<Metadata> getListMetadatas(Query query) {
@@ -64,7 +64,7 @@
for (UserList definition : userLists.getList()) {
metadata.add(definition.getMetadata());
}
- return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize());
+ return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize(), userLists.getTotalSizeRelation());
}
@Override
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 3d36784..812c184 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -25,7 +25,7 @@
import org.apache.http.client.CredentialsProvider;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.apache.lucene.search.TotalHits;
import org.apache.unomi.api.Item;
import org.apache.unomi.api.PartialList;
import org.apache.unomi.api.TimestampedItem;
@@ -55,6 +55,8 @@
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.*;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.core.MainResponse;
import org.elasticsearch.client.indices.*;
import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -1346,13 +1348,13 @@
@Override
protected Long execute(Object... args) throws IOException {
- SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType));
+
+ CountRequest countRequest = new CountRequest(getIndexNameForQuery(itemType));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(filter);
- searchSourceBuilder.size(0);
- searchRequest.source(searchSourceBuilder);
- SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
- return response.getHits().getTotalHits().value;
+ countRequest.source(searchSourceBuilder);
+ CountResponse response = client.count(countRequest, RequestOptions.DEFAULT);
+ return response.getCount();
}
}.catchingExecuteInClassLoader(true);
}
@@ -1365,6 +1367,7 @@
List<T> results = new ArrayList<T>();
String scrollIdentifier = null;
long totalHits = 0;
+ PartialList.Relation totalHitsRelation = PartialList.Relation.EQUAL;
try {
String itemType = Item.getItemType(clazz);
TimeValue keepAlive = TimeValue.timeValueHours(1);
@@ -1448,6 +1451,7 @@
SearchHits searchHits = response.getHits();
scrollIdentifier = response.getScrollId();
totalHits = searchHits.getTotalHits().value;
+ totalHitsRelation = getTotalHitsRelation(searchHits.getTotalHits());
for (SearchHit searchHit : searchHits) {
String sourceAsString = searchHit.getSourceAsString();
final T value = ESCustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
@@ -1460,7 +1464,7 @@
throw new Exception("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
}
- PartialList<T> result = new PartialList<T>(results, offset, size, totalHits);
+ PartialList<T> result = new PartialList<T>(results, offset, size, totalHits, totalHitsRelation);
if (scrollIdentifier != null && totalHits != 0) {
result.setScrollIdentifier(scrollIdentifier);
result.setScrollTimeValidity(scrollTimeValidity);
@@ -1470,6 +1474,10 @@
}.catchingExecuteInClassLoader(true);
}
+ private PartialList.Relation getTotalHitsRelation(TotalHits totalHits) {
+ return TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO.equals(totalHits.relation) ? PartialList.Relation.GREATER_THAN_OR_EQUAL_TO : PartialList.Relation.EQUAL;
+ }
+
@Override
public <T extends Item> PartialList<T> continueScrollQuery(final Class<T> clazz, final String scrollIdentifier, final String scrollTimeValidity) {
return new InClassLoaderExecute<PartialList<T>>(metricsService, this.getClass().getName() + ".continueScrollQuery") {
@@ -1499,7 +1507,7 @@
results.add(value);
}
}
- PartialList<T> result = new PartialList<T>(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits().value);
+ PartialList<T> result = new PartialList<T>(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits().value, getTotalHitsRelation(response.getHits().getTotalHits()));
if (scrollIdentifier != null) {
result.setScrollIdentifier(scrollIdentifier);
result.setScrollTimeValidity(scrollTimeValidity);
@@ -1657,13 +1665,19 @@
}
}
if (aggregations.get("buckets") != null) {
+ long totalDocCount = 0;
MultiBucketsAggregation terms = aggregations.get("buckets");
for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) {
results.put(bucket.getKeyAsString(), bucket.getDocCount());
+ totalDocCount += bucket.getDocCount();
}
SingleBucketAggregation missing = aggregations.get("missing");
if (missing.getDocCount() > 0) {
results.put("_missing", missing.getDocCount());
+ totalDocCount += missing.getDocCount();
+ }
+ if (response.getHits() != null && TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO.equals(response.getHits().getTotalHits().relation)) {
+ results.put("_filtered", totalDocCount);
}
}
}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
index 22cf911..1368105 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
@@ -50,7 +50,7 @@
for (T definition : items.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize());
+ return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize(), items.getTotalSizeRelation());
}
protected <T extends MetadataItem> PartialList<Metadata> getMetadatas(Query query, Class<T> clazz) {
@@ -63,6 +63,6 @@
for (T definition : items.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize());
+ return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize(), items.getTotalSizeRelation());
}
}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
index 336eac9..cf2788e 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
@@ -360,7 +360,7 @@
details.add(campaignDetail);
}
}
- return new PartialList<>(details, campaigns.getOffset(), campaigns.getPageSize(), campaigns.getTotalSize());
+ return new PartialList<>(details, campaigns.getOffset(), campaigns.getPageSize(), campaigns.getTotalSize(), campaigns.getTotalSizeRelation());
}
public CampaignDetail getCampaignDetail(String id) {
diff --git a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
index 6506b42..4f7161a 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
@@ -314,7 +314,7 @@
for (Rule definition : rules.getList()) {
descriptions.add(definition.getMetadata());
}
- return new PartialList<>(descriptions, rules.getOffset(), rules.getPageSize(), rules.getTotalSize());
+ return new PartialList<>(descriptions, rules.getOffset(), rules.getPageSize(), rules.getTotalSize(), rules.getTotalSizeRelation());
}
public PartialList<Rule> getRuleDetails(Query query) {
@@ -325,7 +325,7 @@
PartialList<Rule> rules = persistenceService.query(query.getCondition(), query.getSortby(), Rule.class, query.getOffset(), query.getLimit());
List<Rule> details = new LinkedList<>();
details.addAll(rules.getList());
- return new PartialList<>(details, rules.getOffset(), rules.getPageSize(), rules.getTotalSize());
+ return new PartialList<>(details, rules.getOffset(), rules.getPageSize(), rules.getTotalSize(), rules.getTotalSizeRelation());
}
public Rule getRule(String ruleId) {
diff --git a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
index 047751c..212c1b1 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
@@ -198,7 +198,7 @@
for (Segment definition : segments.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, segments.getOffset(), segments.getPageSize(), segments.getTotalSize());
+ return new PartialList<>(details, segments.getOffset(), segments.getPageSize(), segments.getTotalSize(), segments.getTotalSizeRelation());
}
public PartialList<Metadata> getSegmentMetadatas(Query query) {
