Reverse merge current-release-line
diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml
index 464c935..2902c6b 100644
--- a/checkstyle/pom.xml
+++ b/checkstyle/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
<name>Lens Checkstyle Rules</name>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
<build>
<plugins>
diff --git a/contrib/clients/pom.xml b/contrib/clients/pom.xml
index 2bad7eb..d7677a3 100644
--- a/contrib/clients/pom.xml
+++ b/contrib/clients/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>lens-contrib</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-contrib-clients</artifactId>
diff --git a/contrib/clients/python/lens/client/query.py b/contrib/clients/python/lens/client/query.py
index df16cca..1cd3291 100644
--- a/contrib/clients/python/lens/client/query.py
+++ b/contrib/clients/python/lens/client/query.py
@@ -62,7 +62,7 @@
'DOUBLE': float,
'TIMESTAMP': long_type,
'BINARY': bin,
- 'ARRAY': list,
+ 'ARRAY': lambda x: list(y.strip() for y in x.split(",")),
'MAP': dict,
# 'STRUCT,': str,
# 'UNIONTYPE,': float,
diff --git a/contrib/clients/python/pom.xml b/contrib/clients/python/pom.xml
index 2084d96..8fd6eda 100644
--- a/contrib/clients/python/pom.xml
+++ b/contrib/clients/python/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>lens-contrib-clients</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-python-client</artifactId>
diff --git a/contrib/pom.xml b/contrib/pom.xml
index 5e9c4dc..b36add0 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-contrib</artifactId>
diff --git a/lens-api/pom.xml b/lens-api/pom.xml
index ee96fa7..ece4430 100644
--- a/lens-api/pom.xml
+++ b/lens-api/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-api</artifactId>
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
index 37eda42..f65578e 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
@@ -27,5 +27,5 @@
@XmlType
@XmlEnum
public enum QueryCostType {
- LOW, MEDIUM, HIGH
+ VERY_LOW, LOW, MEDIUM, HIGH
}
diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml
index a343d1b..3640fdd 100644
--- a/lens-cli/pom.xml
+++ b/lens-cli/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<properties>
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
index 26650c1..f37a380 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
@@ -322,6 +322,11 @@
}
@Override
+ protected APIResult doDropPartitions(String tableName, String storageName, String filter, String updatePeriod) {
+ return getClient().dropAllPartitionsOfDim(tableName, storageName, filter);
+ }
+
+ @Override
protected APIResult doUpdatePartition(String tableName, String storageName, String validPath) {
return getClient().updatePartitionOfDim(tableName, storageName, validPath);
}
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
index a872998..63dd992 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
@@ -198,9 +198,10 @@
/**
* Drop all partitions of fact.
*
- * @param tableName fact name
- * @param storageName storage name
- * @param filter partition query filter
+ * @param tableName fact name
+ * @param storageName storage name
+ * @param filter partition query filter
+ * @param updatePeriod update period of partition to be dropped
* @return the string
*/
@CliCommand(value = "fact drop partitions",
@@ -209,8 +210,9 @@
public String dropAllPartitionsOfFact(
@CliOption(key = {"", "fact_name"}, mandatory = true, help = "<fact_name>") String tableName,
@CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "update_period"}, mandatory = true, help = "<update_period>") String updatePeriod,
@CliOption(key = {"", "filter"}, mandatory = false, help = "<partition-filter>") String filter) {
- return dropPartitions(tableName, storageName, filter);
+ return dropPartitions(tableName, storageName, filter, updatePeriod);
}
/**
@@ -327,6 +329,11 @@
}
@Override
+ protected APIResult doDropPartitions(String tableName, String storageName, String filter, String updatePeriod) {
+ return getClient().dropAllPartitionsOfFact(tableName, storageName, filter, updatePeriod);
+ }
+
+ @Override
protected APIResult doUpdatePartition(String tableName, String storageName, String validPath) {
return getClient().updatePartitionOfFact(tableName, storageName, validPath);
}
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
index e0b60c2..5b0c1d0 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
@@ -96,6 +96,10 @@
return doDropPartitions(tableName, storageName, filter).toString().toLowerCase();
}
+ public String dropPartitions(String tableName, String storageName, String filter, String updatePeriod) {
+ return doDropPartitions(tableName, storageName, filter, updatePeriod).toString().toLowerCase();
+ }
+
protected abstract List<String> getAll(String filter);
public abstract List<String> getAllStorages(String name);
@@ -116,6 +120,9 @@
protected abstract APIResult doDropPartitions(String tableName, String storageName, String filter);
+ protected abstract APIResult doDropPartitions(String tableName, String storageName, String filter,
+ String updatePeriod);
+
protected abstract APIResult doUpdatePartition(String tableName, String storageName, String validPath);
protected abstract APIResult doUpdatePartitions(String tableName, String storageName, String validPath);
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
index 927b439..67242d7 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
@@ -432,7 +432,7 @@
assertTrue(timelines.containsAll(command.getTimelines("fact1", FACT_LOCAL, "hourly", "dt")));
assertEquals(command.getTimelines("fact1", null, null, "dt"), timelines);
assertEquals(command.getTimelines("fact1", FACT_LOCAL, null, "dt"), timelines);
- String dropPartitionsStatus = command.dropAllPartitionsOfFact("fact1", FACT_LOCAL, null);
+ String dropPartitionsStatus = command.dropAllPartitionsOfFact("fact1", FACT_LOCAL, null, null);
assertFalse(dropPartitionsStatus.contains("Syntax error, please try in following"));
result = command.getAllPartitionsOfFact("fact1", FACT_LOCAL, filter);
assertTrue(result.trim().isEmpty());
diff --git a/lens-client/pom.xml b/lens-client/pom.xml
index d8b4b4f..5b94800 100644
--- a/lens-client/pom.xml
+++ b/lens-client/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<properties>
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 1de99fe..76c8cad 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -609,6 +609,10 @@
return mc.dropPartitionsOfFactTable(fact, storage, list);
}
+ public APIResult dropAllPartitionsOfFact(String fact, String storage, String list, String updatePeriod) {
+ return mc.dropPartitionsOfFactTable(fact, storage, list, updatePeriod);
+ }
+
public APIResult dropAllPartitionsOfDim(String dim, String storage) {
return mc.dropAllPartitionsOfDimensionTable(dim, storage);
}
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index 8a05952..7b37769 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -602,16 +602,21 @@
return getPartitionsOfFactTable(factName, storage, "");
}
- public APIResult dropPartitionsOfFactTable(String factName, String storage, String filter) {
+ public APIResult dropPartitionsOfFactTable(String factName, String storage, String filter, String updatePeriod) {
WebTarget target = getMetastoreWebTarget();
return translate(target.path("facts").path(factName)
.path("storages").path(storage).path("partitions")
.queryParam("sessionid", this.connection.getSessionHandle())
.queryParam("filter", filter)
+ .queryParam("updatePeriod", updatePeriod)
.request(MediaType.APPLICATION_XML)
.delete());
}
+ public APIResult dropPartitionsOfFactTable(String factName, String storage, String filter) {
+ return dropPartitionsOfFactTable(factName, storage, filter, null);
+ }
+
public APIResult dropPartitionsOfFactTable(String factName, String storage) {
return dropPartitionsOfFactTable(factName, storage, "");
}
diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml
index 706a288..deaeb36 100644
--- a/lens-cube/pom.xml
+++ b/lens-cube/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-cube</artifactId>
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index c57a9c1..43e827a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -393,6 +393,11 @@
return this.getName();
}
+ @Override
+ public Map<String, String> getSourceFactProperties() {
+ return getProperties();
+ }
+
public String getTablePrefix(String storage, UpdatePeriod updatePeriod) {
return storagePrefixUpdatePeriodMap.get(storage).get(updatePeriod);
}
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
index 8b55985..96948cc 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
@@ -190,4 +190,9 @@
public String getSourceFactName() {
return this.sourceCubeFactTable.getName();
}
+
+ @Override
+ public Map<String, String> getSourceFactProperties() {
+ return getSourceCubeFactTable().getProperties();
+ }
}
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
index a463c47..e351d7f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -163,4 +163,5 @@
*/
String getSourceFactName();
+ Map<String, String> getSourceFactProperties();
}
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 1cff4a4..9125f4f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -169,7 +169,8 @@
}
}
@Deprecated
- private List<UnionCandidate> getCombinations(final List<Candidate> candidates, CubeQueryContext cubeql) {
+ private List<UnionCandidate> getCombinations(final List<Candidate> candidates, CubeQueryContext cubeql)
+ throws LensException {
List<UnionCandidate> combinations = new LinkedList<>();
int size = candidates.size();
int threshold = Double.valueOf(Math.pow(2, size)).intValue() - 1;
@@ -187,6 +188,7 @@
}
UnionCandidate uc = new UnionCandidate(individualCombinationList, cubeql);
if (isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
+ uc.cloneChildren();
combinations.add(uc);
}
}
@@ -207,7 +209,8 @@
* @param cubeql
* @return
*/
- private List<UnionCandidate> getCombinationTailIterative(List<Candidate> candidates, CubeQueryContext cubeql) {
+ private List<UnionCandidate> getCombinationTailIterative(List<Candidate> candidates, CubeQueryContext cubeql)
+ throws LensException {
LinkedList<Candidate> candidateLinkedList = Lists.newLinkedList(candidates);
List<List<Candidate>> incompleteCombinations = Lists.newArrayList();
incompleteCombinations.add(Lists.newArrayList());
@@ -221,6 +224,7 @@
incompleteCombination.add(candidate);
UnionCandidate unionCandidate = new UnionCandidate(incompleteCombination, cubeql);
if (isCandidateCoveringTimeRanges(unionCandidate, cubeql.getTimeRanges())) {
+ unionCandidate.cloneChildren();
unionCandidates.add(unionCandidate);
} else {
moreIncomplete.add(incompleteCombination);
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
index eeaa3af..4768fb8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
@@ -48,6 +48,7 @@
public static final String PROCESS_TIME_PART_COL = "lens.cube.query.process.time" + ".partition.column";
public static final String COMPLETENESS_CHECK_PART_COL = "lens.cube.query.completeness.check.partition.column";
public static final String LOOK_AHEAD_PT_PARTS_PFX = "lens.cube.query.lookahead.ptparts.forinterval.";
+ public static final String LOOK_AHEAD_TIME_PARTS_PFX = "lens.cube.query.lookahead.timeparts.forinterval.";
public static final String ENABLE_GROUP_BY_TO_SELECT = "lens.cube.query.promote.groupby.toselect";
public static final String ENABLE_SELECT_TO_GROUPBY = "lens.cube.query.promote.select.togroupby";
public static final String ENABLE_ATTRFIELDS_ADD_DISTINCT = "lens.cube.query.enable.attrfields.add.distinct";
@@ -58,6 +59,7 @@
public static final String REPLACE_TIMEDIM_WITH_PART_COL = "lens.cube.query.replace.timedim";
public static final boolean DEFAULT_MULTI_TABLE_SELECT = true;
public static final int DEFAULT_LOOK_AHEAD_PT_PARTS = 1;
+ public static final int DEFAULT_LOOK_AHEAD_TIME_PARTS = 1;
public static final boolean DEFAULT_ENABLE_GROUP_BY_TO_SELECT = false;
public static final boolean DEFAULT_ENABLE_SELECT_TO_GROUPBY = false;
public static final boolean DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL = true;
@@ -66,6 +68,10 @@
return LOOK_AHEAD_PT_PARTS_PFX + interval.name().toLowerCase();
}
+ public static String getLookAheadTimePartsKey(UpdatePeriod interval) {
+ return LOOK_AHEAD_TIME_PARTS_PFX + interval.name().toLowerCase();
+ }
+
private static String getValidKeyCubePFX(String cubeName) {
return VALID_PFX + cubeName.toLowerCase();
}
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 553468f..e0a0d17 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -560,7 +560,7 @@
private ASTNode getDefaultExpr(ASTNode node) {
if (HQLParser.isAggregateAST(node)) {
- node.setChild(1, new ASTNode(new CommonToken(HiveParser.Identifier, "0.0")));
+ node.setChild(1, new ASTNode(new CommonToken(HiveParser.Identifier, "0")));
}
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 27835b3..b48be7f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -169,7 +169,6 @@
rangeToPartitions.put(entry.getKey(), new LinkedHashSet<>(entry.getValue()));
}
this.rangeToExtraWhereFallBack = sc.rangeToExtraWhereFallBack;
- this.answerableMeasurePhraseIndices = sc.answerableMeasurePhraseIndices;
}
public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext)
@@ -316,7 +315,7 @@
public Optional<Date> getColumnStartTime(String column) {
Date startTime = null;
- for (String key : getTable().getProperties().keySet()) {
+ for (String key : this.getFact().getSourceFactProperties().keySet()) {
if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) {
String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX);
if (column.equals(propCol)) {
@@ -330,7 +329,7 @@
@Override
public Optional<Date> getColumnEndTime(String column) {
Date endTime = null;
- for (String key : getTable().getProperties().keySet()) {
+ for (String key : this.getFact().getSourceFactProperties().keySet()) {
if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) {
String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX);
if (column.equals(propCol)) {
@@ -457,6 +456,10 @@
int lookAheadNumParts = getConf()
.getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS);
+
+ int lookAheadNumTimeParts = getConf()
+ .getInt(CubeQueryConfUtil.getLookAheadTimePartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_TIME_PARTS);
+
TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, maxInterval, 1).iterator();
// add partitions from ceilFrom to floorTo
while (iter.hasNext()) {
@@ -476,7 +479,7 @@
log.debug("part column is process time col");
} else if (updatePeriods.first().equals(maxInterval)) {
log.debug("Update period is the least update period");
- } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumParts) {
+ } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumTimeParts) {
// see if this is the part of the last-n look ahead partitions
log.debug("Not a look ahead partition");
} else {
@@ -487,16 +490,27 @@
// process time are present
TimeRange.Iterable.Iterator processTimeIter = TimeRange.iterable(nextDt, lookAheadNumParts, maxInterval, 1)
.iterator();
+ TimeRange.Iterable.Iterator timeIter = TimeRange.iterable(nextDt, lookAheadNumTimeParts, maxInterval, 1)
+ .iterator();
while (processTimeIter.hasNext()) {
Date pdt = processTimeIter.next();
Date nextPdt = processTimeIter.peekNext();
- FactPartition processTimePartition = new FactPartition(processTimePartCol, pdt, maxInterval, null,
- partWhereClauseFormat);
- updatePartitionStorage(processTimePartition);
- if (processTimePartition.isFound()) {
+ FactPartition currFactPartition;
+ boolean allProcessTimePartitionsFound = true;
+ while (timeIter.hasNext()){
+ Date date = timeIter.next();
+ currFactPartition = new FactPartition(processTimePartCol, date, maxInterval, null,
+ partWhereClauseFormat);
+ updatePartitionStorage(currFactPartition);
+ if (!currFactPartition.isFound()) {
+ log.debug("Looked ahead process time partition {} is not found : " + currFactPartition);
+ allProcessTimePartitionsFound = false;
+ break;
+ }
+ }
+ if (allProcessTimePartitionsFound) {
log.debug("Finer parts not required for look-ahead partition :{}", part);
} else {
- log.debug("Looked ahead process time partition {} is not found", processTimePartition);
TreeSet<UpdatePeriod> newset = new TreeSet<UpdatePeriod>();
newset.addAll(updatePeriods);
newset.remove(maxInterval);
@@ -516,7 +530,8 @@
partWhereClauseFormat);
updatePartitionStorage(innerPart);
innerPart.setFound(pPart.isFound());
- if (innerPart.isFound()) {
+ if (innerPart.isFound() || !failOnPartialData) {
+ this.participatingUpdatePeriods.add(maxInterval);
partitions.add(innerPart);
}
}
@@ -933,6 +948,7 @@
updatePeriodSpecificSc = copy();
updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(),
storageName, period));
+ updatePeriodSpecificSc.isStorageTblsAtUpdatePeriodLevel = false;
updatePeriodSpecificSc.truncatePartitions(period);
periodSpecificScList.add(updatePeriodSpecificSc);
}
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
index 21cdb61..432c0e4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
@@ -21,6 +21,7 @@
import static org.apache.lens.cube.metadata.MetastoreConstants.VIRTUAL_FACT_FILTER;
+import java.util.Collection;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -50,7 +51,17 @@
this.storageCandidate = storageCandidate;
this.rootCubeQueryContext = rootCubeQueryContext;
getCubeQueryContext().addRangeClauses(this);
- if (!Objects.equals(getStorageCandidate(), rootCubeQueryContext.getPickedCandidate())) {
+ boolean setNullHaving = true;
+ if (Objects.equals(getStorageCandidate(), rootCubeQueryContext.getPickedCandidate())) {
+ setNullHaving = false;
+ } else {
+ Collection<? extends Candidate> children = rootCubeQueryContext.getPickedCandidate().getChildren();
+ // children should not be null. Not checking for null here.
+ if (children.size() == 1 && Objects.equals(getStorageCandidate(), children.iterator().next())) {
+ setNullHaving = false;
+ }
+ }
+ if (setNullHaving) {
getQueryAst().setHavingAST(null);
}
}
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
index 510ea0c..231035e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
@@ -65,7 +65,12 @@
this.children = Lists.newArrayList(childCandidates);
this.cubeQueryContext = cubeQueryContext;
}
-
+ void cloneChildren() throws LensException {
+ ListIterator<Candidate> iter = children.listIterator();
+ while(iter.hasNext()) {
+ iter.set(iter.next().copy());
+ }
+ }
@Override
public Set<Integer> getAnswerableMeasurePhraseIndices() {
// All children in the UnionCandiate will be having common quriable measure
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index 4eb086b..9d0d0d2 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -54,7 +54,7 @@
private Map<StorageCandidateHQLContext, ASTNode> storageCandidateToSelectAstMap = new HashMap<>();
private CubeQueryContext cubeql;
static final ASTNode DEFAULT_MEASURE_AST;
- private static final String DEFAULT_MEASURE = "0.0";
+ private static final String DEFAULT_MEASURE = "0";
static {
try {
DEFAULT_MEASURE_AST = HQLParser.parseExpr(DEFAULT_MEASURE);
@@ -204,7 +204,8 @@
ASTNode outerOrderby = new ASTNode(child);
ASTNode tokNullsChild = (ASTNode) child.getChild(0);
ASTNode outerTokNullsChild = new ASTNode(tokNullsChild);
- if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.DOT) {
+ if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.DOT
+ || ((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.TOK_FUNCTION) {
outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) tokNullsChild)));
} else {
outerTokNullsChild.addChild(tokNullsChild);
@@ -216,7 +217,7 @@
}
/**
- * Get the select expression. In case of node is default retunrs "0.0" with alias
+ * Get the select expression. In case of node is default retunrs "0" with alias
* otherwise the select phrase with alias.
*
* @param nodeWithoutAlias
@@ -246,7 +247,7 @@
/**
* Get the aggregate node for the SelectPhrase index. A given measure might not be answerable
- * for a StorageCanddate. In that case get the non default aggregate node wcich ideally not "0.0",
+ * for a StorageCanddate. In that case get the non default aggregate node wcich ideally not "0",
* from otherStorage candidate.
*
* @param position
@@ -266,7 +267,7 @@
}
/**
- * Set the default value "0.0" in the non answerable aggreagte expressions.
+ * Set the default value "0" in the non answerable aggreagte expressions.
* @param node
* @param sc
* @return
@@ -386,7 +387,7 @@
/**
* Set the default value for the non queriable measures. If a measure is not
- * answerable from a StorageCandidate set it as 0.0
+ * answerable from a StorageCandidate set it as "0"
*
* @throws LensException
*/
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
index 3c157ee..6d845f8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.cube.query.cost;
+import static org.apache.lens.server.api.LensConfConstants.DRIVER_COST_TYPE_RANGES;
+import static org.apache.lens.server.api.LensConfConstants.DRIVER_QUERY_COST_TYPE_DEFAULT_RANGES;
+
import java.util.Map;
import java.util.Set;
@@ -26,9 +29,7 @@
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
-import org.apache.lens.server.api.query.cost.QueryCost;
-import org.apache.lens.server.api.query.cost.QueryCostCalculator;
+import org.apache.lens.server.api.query.cost.*;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
@@ -37,6 +38,8 @@
public static final String UPDATE_PERIOD_WEIGHT_PREFIX = "update.period.weight.";
+ protected QueryCostTypeDecider queryCostTypeDecider;
+
/**
* Calculates total cost based on weights of selected tables and their selected partitions
*
@@ -91,9 +94,19 @@
}
@Override
+ public void init(LensDriver lensDriver) {
+ queryCostTypeDecider = new RangeBasedQueryCostTypeDecider(
+ lensDriver.getConf().get(DRIVER_COST_TYPE_RANGES, DRIVER_QUERY_COST_TYPE_DEFAULT_RANGES));
+ }
+
+ @Override
public QueryCost calculateCost(final AbstractQueryContext queryContext, LensDriver driver) throws LensException {
Double cost = getTotalPartitionCost(queryContext, driver);
- return cost == null ? null : new FactPartitionBasedQueryCost(cost);
+ QueryCost queryCost = cost == null ? null : new FactPartitionBasedQueryCost(cost);
+ if (queryCost != null) {
+ queryCost.setQueryCostType(queryCostTypeDecider.decideCostType(queryCost));
+ }
+ return queryCost;
}
public Map<String, Set<?>> getAllPartitions(AbstractQueryContext queryContext, LensDriver driver) {
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/StaticCostCalculator.java b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/StaticCostCalculator.java
new file mode 100644
index 0000000..2fc069a
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/StaticCostCalculator.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.cube.query.cost;
+
+import static org.apache.lens.server.api.LensConfConstants.*;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.cost.*;
+
+public class StaticCostCalculator implements QueryCostCalculator {
+
+ private QueryCost queryCost;
+
+ @Override
+ public void init(LensDriver lensDriver) throws LensException {
+ QueryCostTypeDecider queryCostTypeDecider = new RangeBasedQueryCostTypeDecider(
+ lensDriver.getConf().get(DRIVER_COST_TYPE_RANGES, DRIVER_QUERY_COST_TYPE_DEFAULT_RANGES));
+ this.queryCost = new StaticQueryCost(lensDriver.getConf().getDouble(DRIVER_QUERY_COST, DEFAULT_DRIVER_QUERY_COST));
+ this.queryCost.setQueryCostType(queryCostTypeDecider.decideCostType(this.queryCost));
+ }
+
+ @Override
+ public QueryCost calculateCost(AbstractQueryContext queryContext, LensDriver driver) throws LensException {
+ return this.queryCost;
+ }
+
+}
diff --git a/lens-cube/src/main/resources/olap-query-conf.xml b/lens-cube/src/main/resources/olap-query-conf.xml
index b389d6a..d0e5eb4 100644
--- a/lens-cube/src/main/resources/olap-query-conf.xml
+++ b/lens-cube/src/main/resources/olap-query-conf.xml
@@ -153,6 +153,14 @@
</property>
<property>
+ <name>lens.cube.query.lookahead.timeparts.forinterval.${interval}</name>
+ <value>1</value>
+ <description>The number of time partitions for interval specified where look ahead is applied.
+ Interval can be any Update period.
+ </description>
+ </property>
+
+ <property>
<name>lens.cube.query.replace.timedim</name>
<value>true</value>
<description>Tells whether timedim attribute queried in the time range should be replaced with its corresponding
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
index 75f2abd..c4404f1 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
@@ -168,6 +168,7 @@
public static final String TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS;
public static final String TWO_DAYS_RANGE_TTD2;
public static final String TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS;
+ public static final String THREE_DAYS_RANGE_IT;
public static final String TWO_DAYS_RANGE_IT;
public static final String ONE_DAY_RANGE_IT;
public static final String THIS_YEAR_RANGE;
@@ -218,6 +219,7 @@
TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS = getTimeRangeString("test_time_dim", DAILY, -6, -4, HOURLY);
TWO_DAYS_RANGE_TTD2 = getTimeRangeString("test_time_dim2", DAILY, -2, 0, HOURLY);
TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS = getTimeRangeString("test_time_dim2", DAILY, -6, -4, HOURLY);
+ THREE_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -3, 0, HOURLY);
TWO_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -2, 0, HOURLY);
ONE_DAY_RANGE_IT = getTimeRangeString("it", DAILY, -1, 0, DAILY);
THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1);
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index 35cb2b5..db7b9b2 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -144,10 +144,10 @@
Assert.assertTrue(storageCandidates.contains("c1_testfact2_base"));
String hqlQuery = ctx.toHQL();
String expected1 =
- getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, sum((basecube.msr12)) as `alias1` FROM ", null,
+ getExpectedQuery(cubeName, "SELECT 0 as `alias0`, sum((basecube.msr12)) as `alias1` FROM ", null,
null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT sum((basecube.msr2)) as `alias0`, 0.0 as `alias1` FROM ", null,
+ getExpectedQuery(cubeName, "SELECT sum((basecube.msr2)) as `alias0`, 0 as `alias1` FROM ", null,
null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -171,14 +171,14 @@
Assert.assertTrue(storageCandidates.contains("c1_testfact2_base"));
Assert.assertTrue(storageCandidates.contains("c1_testfact3_base"));
String hqlQuery = ctx.toHQL();
- String expected1 = getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, 0.0 as `alias1`, "
+ String expected1 = getExpectedQuery(cubeName, "SELECT 0 as `alias0`, 0 as `alias1`, "
+ "sum((basecube.msr12)) as `alias2` FROM ", null, null,
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
- String expected2 = getExpectedQuery(cubeName, "SELECT sum((basecube.msr2)) as `alias0`, 0.0 as `alias1`, "
- + "0.0 as `alias2` FROM ", null,
+ String expected2 = getExpectedQuery(cubeName, "SELECT sum((basecube.msr2)) as `alias0`, 0 as `alias1`, "
+ + "0 as `alias2` FROM ", null,
null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
- String expected3 = getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, count((basecube.msr14)) as `alias1`, "
- + "0.0 as `alias2` FROM ", null, null,
+ String expected3 = getExpectedQuery(cubeName, "SELECT 0 as `alias0`, count((basecube.msr14)) as `alias1`, "
+ + "0 as `alias2` FROM ", null, null,
getWhereForDailyAndHourly2days(cubeName, "C1_testFact3_BASE"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -194,9 +194,9 @@
String hqlQuery = rewrite("select dim1, roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr2)) as `alias1`, "
- + "0.0 as `alias2` FROM ", null, " group by basecube.dim1",
+ + "0 as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
- String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, "
+ String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, "
+ "sum((basecube.msr12)) as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
compareContains(expected1, hqlQuery);
@@ -214,11 +214,11 @@
Configuration tConf = new Configuration(conf);
tConf.setBoolean(CubeQueryConfUtil.LIGHTEST_FACT_FIRST, true);
String hqlQuery = rewrite("select dim1, roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, tConf);
- String expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, "
+ String expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, "
+ "sum((basecube.msr12)) as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr2)) "
- + "as `alias1`, 0.0 as `alias2` FROM ", null, " group by basecube.dim1",
+ + "as `alias1`, 0 as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -235,11 +235,11 @@
tConf.setBoolean(CubeQueryConfUtil.LIGHTEST_FACT_FIRST, true);
String hqlQuery = rewrite("select dim1, roundedmsr2, flooredmsr12 from basecube" + " where "
+ TWO_DAYS_RANGE, tConf);
- String expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, "
+ String expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, "
+ "sum((basecube.msr12)) as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr2)) "
- + "as `alias1`, 0.0 as `alias2` FROM ", null, " group by basecube.dim1",
+ + "as `alias1`, 0 as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -256,10 +256,10 @@
String hqlQuery = rewrite("select dim1, msr12, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, "
- + "0.0 as `alias2` FROM", null, " group by basecube.dim1",
+ + "0 as `alias2` FROM", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", null,
+ "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -278,17 +278,17 @@
+ TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, " SELECT (basecube.dim1) as `alias0`, (basecube.d_time) as `alias1`, "
- + "sum((basecube.msr12)) as `alias2`, 0.0 as `alias3`, 0.0 as `alias4`, 0.0 as `alias5` FROM ",
+ + "sum((basecube.msr12)) as `alias2`, 0 as `alias3`, 0 as `alias4`, 0 as `alias5` FROM ",
null, " group by basecube.dim1, (basecube.d_time)",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 = getExpectedQuery(
cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.d_time) as `alias1`, 0.0 as `alias2`, "
- + "sum((basecube.msr2)) as `alias3`, 0.0 as `alias4`, max((basecube.msr3)) as `alias5` FROM ", null,
+ "SELECT (basecube.dim1) as `alias0`, (basecube.d_time) as `alias1`, 0 as `alias2`, "
+ + "sum((basecube.msr2)) as `alias3`, 0 as `alias4`, max((basecube.msr3)) as `alias5` FROM ", null,
" group by basecube.dim1, (basecube.d_time)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
String expected3 =
- getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.d_time) as `alias1`, 0.0 "
- + "as `alias2`, 0.0 as `alias3`, max((basecube.msr13)) as `alias4`, 0.0 as `alias5` FROM ", null,
+ getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.d_time) as `alias1`, 0 "
+ + "as `alias2`, 0 as `alias3`, max((basecube.msr13)) as `alias4`, 0 as `alias5` FROM ", null,
" group by basecube.dim1, (basecube.d_time)", getWhereForDailyAndHourly2days(cubeName, "c1_testfact3_base"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -307,10 +307,10 @@
// query two dim attributes
String hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf);
String expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, "
- + "sum((basecube.msr12)) as `alias2`, 0.0 as `alias3` FROM ", null, " group by basecube.dim1",
+ + "sum((basecube.msr12)) as `alias2`, 0 as `alias3` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, "
- + "0.0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1",
+ + "0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -328,9 +328,9 @@
String hqlQuery = rewrite("select dim1, msr11, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.msr11) as `alias1`, "
- + "0.0 as `alias2` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
+ + "0 as `alias2` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
String expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, round(((basecube.msr2) / 1000)) "
+ "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, round(((basecube.msr2) / 1000)) "
+ "as `alias2` FROM ", null, null,
getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
@@ -349,10 +349,10 @@
rewrite("select dim1 d1, msr12 `my msr12`, roundedmsr2 m2 from basecube where " + TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, "
- + "0.0 as `alias2` FROM ", null, " group by basecube.dim1",
+ + "0 as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) "
+ getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, sum((basecube.msr2)) "
+ "as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
@@ -371,10 +371,10 @@
+ TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, "
- + "0.0 as `alias2` FROM ", null, " group by basecube.dim1",
+ + "0 as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) "
+ getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, sum((basecube.msr2)) "
+ "as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
@@ -393,10 +393,10 @@
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, "
- + "0.0 as `alias2` FROM ", null, " group by basecube.dim1",
+ + "0 as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) "
+ getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, sum((basecube.msr2)) "
+ "as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
@@ -416,10 +416,10 @@
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, "
- + "0.0 as `alias2` FROM", null, " group by basecube.dim1",
+ + "0 as `alias2` FROM", null, " group by basecube.dim1",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) "
+ getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, sum((basecube.msr2)) "
+ "as `alias2` FROM", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
@@ -437,12 +437,12 @@
rewrite("select reverse(dim1), ltrim(dim1), msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, ltrim((basecube.dim1)) as `alias1`, "
- + "sum((basecube.msr12)) as `alias2`, 0.0 as `alias3` FROM ", null,
+ + "sum((basecube.msr12)) as `alias2`, 0 as `alias3` FROM ", null,
" group by reverse(basecube.dim1), ltrim(basecube.dim1)",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 =
getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, ltrim((basecube.dim1)) as `alias1`, "
- + "0.0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM ", null,
+ + "0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM ", null,
" group by reverse(basecube.dim1), ltrim(basecube.dim1)",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
@@ -462,11 +462,11 @@
conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, max((basecube.msr13)) as `alias1`, "
- + "count((basecube.msr14)) as `alias2`, 0.0 as `alias3` FROM", null,
+ + "count((basecube.msr14)) as `alias2`, 0 as `alias3` FROM", null,
" group by reverse(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact3_BASE"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, 0.0 as `alias1`, "
- + "0.0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM", null,
+ getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, 0 as `alias1`, "
+ + "0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM", null,
" group by reverse(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -484,10 +484,10 @@
String hqlQuery = rewrite("select dim1, avg(msr12), avg(msr2) from basecube where " + TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, avg((basecube.msr12)) as `alias1`,"
- + " 0.0 as `alias2` FROM ", null, " group by basecube.dim1",
+ + " 0 as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, avg((basecube.msr2)) "
+ getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, avg((basecube.msr2)) "
+ "as `alias2` FROM ", null, " group by basecube.dim1",
getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
@@ -503,13 +503,13 @@
// query with join
String hqlQuery = rewrite("select dim2chain.name, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf);
String expected1 = getExpectedQuery(cubeName,
- "SELECT (dim2chain.name) as `alias0`, sum((basecube.msr12)) as `alias1`, 0.0 as `alias2` FROM ",
+ "SELECT (dim2chain.name) as `alias0`, sum((basecube.msr12)) as `alias1`, 0 as `alias2` FROM ",
" JOIN " + getDbName()
+ "c1_testdim2tbl dim2chain ON basecube.dim2 = "
+ " dim2chain.id and (dim2chain.dt = 'latest') ", null,
" group by dim2chain.name", null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 = getExpectedQuery(cubeName,
- "SELECT (dim2chain.name) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", " JOIN "
+ "SELECT (dim2chain.name) as `alias0`, 0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", " JOIN "
+ getDbName()
+ "c1_testdim2tbl dim2chain ON basecube.dim2 = "
+ " dim2chain.id and (dim2chain.dt = 'latest') ", null,
@@ -528,12 +528,12 @@
// query with denorm variable
String hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf);
String expected1 = getExpectedQuery(cubeName, "SELECT (dim2chain.id) as `alias0`, max((basecube.msr13)) "
- + "as `alias1`, 0.0 as `alias2` FROM ", " JOIN " + getDbName()
+ + "as `alias1`, 0 as `alias2` FROM ", " JOIN " + getDbName()
+ "c1_testdim2tbl dim2chain ON basecube.dim12 = "
+ " dim2chain.id and (dim2chain.dt = 'latest') ", null, " group by dim2chain.id", null,
getWhereForHourly2days(cubeName, "C1_testFact3_RAW_BASE"));
String expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim2) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", null,
+ "SELECT (basecube.dim2) as `alias0`, 0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", null,
" group by basecube.dim2", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -550,12 +550,12 @@
String hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube where dim2 == 10 and " + TWO_DAYS_RANGE,
conf);
String expected1 = getExpectedQuery(cubeName, "SELECT (dim2chain.id) as `alias0`, max((basecube.msr13)) "
- + "as `alias1`, 0.0 as `alias2` FROM ", " JOIN " + getDbName()
+ + "as `alias1`, 0 as `alias2` FROM ", " JOIN " + getDbName()
+ "c1_testdim2tbl dim2chain ON basecube.dim12 = "
+ " dim2chain.id and (dim2chain.dt = 'latest') ", "dim2chain.id == 10", " group by dim2chain.id", null,
getWhereForHourly2days(cubeName, "C1_testFact3_RAW_BASE"));
String expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim2) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ",
+ "SELECT (basecube.dim2) as `alias0`, 0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ",
"basecube.dim2 == 10", " group by basecube.dim2",
getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
@@ -577,14 +577,14 @@
conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (((basecube.dim1) != 'x') and ((dim2chain.id) != 10)) as `alias0`, "
- + "0.0 as `alias1`, avg(((basecube.msr13) + (basecube.msr14))) as `alias2` FROM ", " JOIN "
+ + "0 as `alias1`, avg(((basecube.msr13) + (basecube.msr14))) as `alias2` FROM ", " JOIN "
+ getDbName() + "c1_testdim2tbl dim2chain ON basecube.dim12 = "
+ " dim2chain.id and (dim2chain.dt = 'latest') ", null,
" group by basecube.dim1 != 'x' AND dim2chain.id != 10", null,
getWhereForHourly2days(cubeName, "C1_testfact3_raw_base"));
String expected2 =
getExpectedQuery(cubeName, "SELECT (((basecube.dim1) != 'x') and ((basecube.dim2) != 10)) as `alias0`, "
- + "sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM", null,
+ + "sum((basecube.msr2)) as `alias1`, 0 as `alias2` FROM", null,
" group by basecube.dim1 != 'x' AND basecube.dim2 != 10",
getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
@@ -607,14 +607,14 @@
+ TWO_DAYS_RANGE, conf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (((basecube.dim1) != 'x') and ((dim2chain.id) != 10)) as `alias0`, "
- + "0.0 as `alias1`, avg(((basecube.msr13) + (basecube.msr14))) as `alias2` FROM ", " JOIN " + getDbName()
+ + "0 as `alias1`, avg(((basecube.msr13) + (basecube.msr14))) as `alias2` FROM ", " JOIN " + getDbName()
+ "c1_testdim2tbl dim2chain ON basecube.dim12 = " + " dim2chain.id and (dim2chain.dt = 'latest') ",
"(basecube.dim1 != 'x' AND dim2chain.id != 10) == true",
" group by basecube.dim1 != 'x' AND dim2chain.id != 10", null,
getWhereForHourly2days(cubeName, "C1_testfact3_raw_base"));
String expected2 =
getExpectedQuery(cubeName, "SELECT (((basecube.dim1) != 'x') and ((basecube.dim2) != 10)) as `alias0`, "
- + "sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM ",
+ + "sum((basecube.msr2)) as `alias1`, 0 as `alias2` FROM ",
"(basecube.dim1 != 'x' AND basecube.dim2 != 10) == true",
" group by basecube.dim1 != 'x' AND basecube.dim2 != 10",
getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
@@ -635,12 +635,12 @@
String hqlQuery =
rewrite("select booleancut, round(sum(msr2)/1000), msr13 from basecube where " + TWO_DAYS_RANGE, tconf);
String expected1 =
- getExpectedQuery(cubeName, "SELECT (basecube.booleancut) as `alias0`, 0.0 as `alias1`, "
+ getExpectedQuery(cubeName, "SELECT (basecube.booleancut) as `alias0`, 0 as `alias1`, "
+ "max((basecube.msr13)) as `alias2` FROM", null, " "
+ "group by basecube.booleancut", getWhereForDailyAndHourly2days(cubeName, "C1_testfact6_base"));
String expected2 =
getExpectedQuery(cubeName, "SELECT (basecube.booleancut) as `alias0`, sum((basecube.msr2)) as `alias1`, "
- + "0.0 as `alias2` FROM ", null, " group by basecube.booleancut",
+ + "0 as `alias2` FROM ", null, " group by basecube.booleancut",
getWhereForDailyAndHourly2days(cubeName, "C1_testfact5_base"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -660,10 +660,10 @@
+ TWO_DAYS_RANGE, tconf);
String expected1 =
getExpectedQuery(cubeName, "SELECT sum(case when ((basecube.dim22) = 'x') then (basecube.msr12) else 0 end) "
- + "as `alias0`, 0.0 as `alias1` FROM ", null, null,
+ + "as `alias0`, 0 as `alias1` FROM ", null, null,
getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, null,
+ getExpectedQuery(cubeName, "SELECT 0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, null,
getWhereForHourly2days(cubeName, "c1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -680,10 +680,10 @@
+ TWO_DAYS_RANGE, tconf);
String expected1 =
getExpectedQuery(cubeName, "SELECT sum(case when ((basecube.dim13) = 'x') then (basecube.msr12) else 0 end) "
- + "as `alias0`, 0.0 as `alias1` FROM ", null, null,
+ + "as `alias0`, 0 as `alias1` FROM ", null, null,
getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, null,
+ getExpectedQuery(cubeName, "SELECT 0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, null,
getWhereForHourly2days(cubeName, "c1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -700,11 +700,11 @@
+ "where " + TWO_DAYS_RANGE, tconf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum(case when ((basecube.dim13) = 'x') "
- + "then (basecube.msr12) else 0 end) as `alias1`, 0.0 as `alias2` FROM ", null,
+ + "then (basecube.msr12) else 0 end) as `alias1`, 0 as `alias2` FROM ", null,
" group by basecube.dim1 ",
getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr1)) "
+ getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, sum((basecube.msr1)) "
+ "as `alias2` FROM", null,
" group by basecube.dim1 ", getWhereForHourly2days(cubeName, "c1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
@@ -726,10 +726,10 @@
+ "and sum(msr1) > 500", tconf);
String expected1 =
getExpectedQuery(cubeName, "SELECT sum(case when ((basecube.dim13) = 'x') then (basecube.msr12) else 0 end) "
- + "as `alias0`, 0.0 as `alias1` FROM ", null, "",
+ + "as `alias0`, 0 as `alias1` FROM ", null, "",
getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, "",
+ getExpectedQuery(cubeName, "SELECT 0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, "",
getWhereForHourly2days(cubeName, "c1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
compareContains(expected2, hqlQuery);
@@ -748,10 +748,10 @@
+ "and sum(msr1) > 500", tconf);
String expected1 =
getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum(case when ((basecube.dim13) = 'x') then "
- + "(basecube.msr12) else 0 end) as `alias1`, 0.0 as `alias2` FROM", null, " group by basecube.dim1",
+ + "(basecube.msr12) else 0 end) as `alias1`, 0 as `alias2` FROM", null, " group by basecube.dim1",
getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
String expected2 =
- getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr1)) "
+ getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0 as `alias1`, sum((basecube.msr1)) "
+ "as `alias2` FROM", null, " group by basecube.dim1",
getWhereForHourly2days(cubeName, "c1_testfact1_raw_base"));
compareContains(expected1, hqlQuery);
@@ -876,10 +876,10 @@
+ "having roundedmsr2 > 0", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) "
- + "as `alias2`, 0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ + "as `alias2`, 0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -895,10 +895,10 @@
+ "having msr12 > 2 and roundedmsr2 > 0", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ + "0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -915,10 +915,10 @@
+ "having msr12 > 2 and roundedmsr2 > 0", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) "
- + "as `alias2`, 0.0 as `alias4` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ + "as `alias2`, 0 as `alias4` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias4` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -934,11 +934,11 @@
+ "having msr12+roundedmsr2 <= 1000 and msr12 > 2 and roundedmsr2 > 0", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ + "0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -954,10 +954,10 @@
+ "having msr12+roundedmsr2 <= 1000", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "0.0 as `alias3` FROM", null, " group by basecube.dim1, basecube.dim11",
+ + "0 as `alias3` FROM", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, sum((basecube.msr2)) "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, sum((basecube.msr2)) "
+ "as `alias3` FROM", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -974,10 +974,10 @@
+ "having round(msr12+roundedmsr2) <= 1000", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ + "0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- " SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ " SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -999,10 +999,10 @@
+ "having msr12 > 2 and roundedmsr2 > 0", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ + "0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
String begin = "select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11` from";
@@ -1017,10 +1017,10 @@
+ "having msr12 > 2 and roundedmsr2 > 0 and msr2 > 100", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ + "0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ", null,
" group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -1035,11 +1035,11 @@
+ "having msr12+roundedmsr2 <= 1000", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) "
- + "as `alias2`, 0.0 as `alias3` FROM ",
+ + "as `alias2`, 0 as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -1056,11 +1056,11 @@
+ "having msr12 > 2 and roundedmsr2 > 0 and msr12+roundedmsr2 <= 1000", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "0.0 as `alias3` FROM ",
+ + "0 as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -1078,11 +1078,11 @@
+ "having msr12 > 2 or roundedmsr2 > 0 or msr12+roundedmsr2 <= 1000", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "0.0 as `alias3` FROM ",
+ + "0 as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0 as `alias2`, "
+ "sum((basecube.msr2)) as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
index 9fbeb41..09dcf36 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
@@ -206,7 +206,7 @@
String query = "select usersports.name, msr2, msr12 from basecube where " + TWO_DAYS_RANGE;
String hqlQuery = rewrite(query, hConf);
String expected1 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM ",
+ "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 as `alias2` FROM ",
" join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from "
@@ -215,7 +215,7 @@
+ " group by user_interests.user_id) usersports" + " on userdim.id = usersports.user_id ", null,
"group by usersports.balias0", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
String expected2 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM ",
+ "SELECT (usersports.balias0) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM ",
" join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from "
@@ -238,7 +238,7 @@
query = "select sports, msr2, msr12 from basecube where " + TWO_DAYS_RANGE;
hqlQuery = rewrite(query, hConf);
expected1 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM ",
+ "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 as `alias2` FROM ",
" join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from "
+ getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName()
@@ -246,7 +246,7 @@
+ " group by user_interests.user_id) usersports" + " on userdim.id = usersports.user_id ", null,
"group by usersports.balias0", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
expected2 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) "
+ "SELECT (usersports.balias0) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from "
@@ -625,7 +625,7 @@
+ " and usersports.name = 'CRICKET'";
String hqlQuery = rewrite(query, hConf);
String expected1 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 "
+ "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from "
@@ -635,7 +635,7 @@
" and array_contains(usersports.balias0,'CRICKET') group by usersports.balias0", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
String expected2 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) "
+ "SELECT (usersports.balias0) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) "
+ "as `alias2` FROM", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from "
@@ -656,7 +656,7 @@
query = "select sports, msr2, msr12 from basecube where " + TWO_DAYS_RANGE + " and sports = 'CRICKET'";
hqlQuery = rewrite(query, hConf);
expected1 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 "
+ "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from "
@@ -666,7 +666,7 @@
"and array_contains(usersports.balias0,'CRICKET') group by usersports.balias0", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
expected2 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) "
+ "SELECT (usersports.balias0) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from "
@@ -693,7 +693,7 @@
+ " and usersports.name = 'CRICKET,FOOTBALL'";
String hqlQuery = rewrite(query, conf);
String expected1 = getExpectedQuery("basecube",
- "SELECT (usersports.name) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM ",
+ "SELECT (usersports.name) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 as `alias2` FROM ",
" join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from "
+ getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName()
@@ -702,7 +702,7 @@
" and usersports.name = 'CRICKET,FOOTBALL' group by usersports.name", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
String expected2 = getExpectedQuery("basecube",
- "SELECT (usersports.name) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM ",
+ "SELECT (usersports.name) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM ",
" join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from "
@@ -724,7 +724,7 @@
+ " and sports = 'CRICKET,FOOTBALL'";
hqlQuery = rewrite(query, conf);
expected1 = getExpectedQuery("basecube",
- "SELECT (usersports.name) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 "
+ "SELECT (usersports.name) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from "
@@ -734,7 +734,7 @@
" and usersports.name = 'CRICKET,FOOTBALL' group by usersports.name", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
expected2 = getExpectedQuery("basecube",
- "SELECT (usersports.name) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) "
+ "SELECT (usersports.name) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from "
@@ -878,7 +878,7 @@
+ " and usersports.name in ('CRICKET', 'FOOTBALL')";
String hqlQuery = rewrite(query, hConf);
String expected1 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 "
+ "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id, collect_set(substr(usersports.name, 3)) as balias0, "
@@ -890,7 +890,7 @@
+ " group by usersports.balias0", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
String expected2 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM "
+ "SELECT (usersports.balias0) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM "
, " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id, collect_set(substr(usersports.name, 3)) as balias0, "
@@ -915,7 +915,7 @@
+ "('CRICKET', 'FOOTBALL')";
hqlQuery = rewrite(query, hConf);
expected1 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 "
+ "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id, collect_set(substr((usersports.name), 3)) as balias0, "
@@ -927,7 +927,7 @@
+ " group by usersports.balias0", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
expected2 = getExpectedQuery("basecube",
- "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) "
+ "SELECT (usersports.balias0) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id, collect_set(substr((usersports.name), 3)) as balias0,"
@@ -957,7 +957,7 @@
+ " and usersports.name = 'CRICKET,FOOTBALL'";
String hqlQuery = rewrite(query, conf);
String expected1 = getExpectedQuery("basecube",
- "SELECT substr((usersports.name), 3) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 "
+ "SELECT substr((usersports.name), 3) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from "
@@ -967,7 +967,7 @@
" and usersports.name = 'CRICKET,FOOTBALL' group by substr(usersports.name, 3)", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
String expected2 = getExpectedQuery("basecube",
- "SELECT substr((usersports.name), 3) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) "
+ "SELECT substr((usersports.name), 3) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from "
@@ -990,7 +990,7 @@
+ "'CRICKET,FOOTBALL'";
hqlQuery = rewrite(query, conf);
expected1 = getExpectedQuery("basecube",
- "SELECT substr((usersports.name), 3) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 "
+ "SELECT substr((usersports.name), 3) as `alias0`, sum((basecube.msr2)) as `alias1`, 0 "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from "
@@ -1000,7 +1000,7 @@
" and usersports.name = 'CRICKET,FOOTBALL' group by substr(usersports.name, 3)", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
expected2 = getExpectedQuery("basecube",
- "SELECT substr((usersports.name), 3) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) "
+ "SELECT substr((usersports.name), 3) as `alias0`, 0 as `alias1`, sum((basecube.msr12)) "
+ "as `alias2` FROM ", " join " + getDbName()
+ "c1_usertable userdim ON basecube.userid = userdim.id "
+ " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from "
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 9b54957..4fdb822 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -1467,6 +1467,62 @@
.getParticipatingPartitions());
// pt does not exist beyond 1 day. So in this test, max look ahead possible is 3
assertEquals(partsQueried, expectedPartsQueried);
+
+ //New test case for testing look ahead time parts
+ expectedPartsQueried = Sets.newTreeSet();
+ conf.setInt(CubeQueryConfUtil.LOOK_AHEAD_TIME_PARTS_PFX + "daily", 2);
+
+ ceilDay = DAILY.getCeilDate(getDateWithOffset(DAILY, -3));
+ nextDay = DateUtils.addDays(ceilDay, 1);
+ nextToNextDay = DateUtils.addDays(nextDay, 1);
+
+ for (TimePartition p : Iterables.concat(
+ TimePartition.of(HOURLY, getDateWithOffset(DAILY, -3)).rangeUpto(TimePartition.of(HOURLY, ceilDay)),
+ TimePartition.of(DAILY, ceilDay).rangeUpto(TimePartition.of(DAILY, nextDay)),
+ TimePartition.of(DAILY, nextDay).rangeUpto(TimePartition.of(DAILY, nextToNextDay)),
+ TimePartition.of(HOURLY, nextToNextDay).rangeUpto(TimePartition.of(HOURLY, NOW)))) {
+ FactPartition fp = new FactPartition("it", p, null, storageTables);
+ expectedPartsQueried.add(fp);
+ }
+ for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) {
+ for (TimePartition pt : TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) {
+ FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables);
+ FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables);
+ expectedPartsQueried.add(itPartition);
+ }
+ }
+
+ ceilDay = DateUtils.addDays(ceilDay, 1);
+ nextDay = DateUtils.addDays(nextDay, 1);
+ nextToNextDay = DateUtils.addDays(nextToNextDay, 1);
+
+ for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) {
+ for (TimePartition pt : TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) {
+ FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables);
+ FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables);
+ expectedPartsQueried.add(itPartition);
+ }
+ }
+
+ ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + THREE_DAYS_RANGE_IT,
+ conf);
+ partsQueried = new TreeSet<>(((StorageCandidate)ctx.getCandidates().iterator().next())
+ .getParticipatingPartitions());
+ // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3
+ assertEquals(partsQueried, expectedPartsQueried);
+ }
+
+ @Test
+ public void testTimeRangeIn() throws Exception {
+ //check whether time_range_in is resolving in cube rewrite
+ Configuration conf = getConf();
+ conf.set(CubeQueryConfUtil.PROCESS_TIME_PART_COL, "pt");
+ conf.set(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "true");
+ conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class);
+ CubeQueryContext ctx = rewriteCtx("select dim1, sum(msr23)" + " from testCube" + " where " + ONE_DAY_RANGE_IT,
+ conf);
+ String rewrittenQuery = ctx.toHQL();
+ assertTrue(!rewrittenQuery.contains("time_range_in"));
}
@Test
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
index 7e1714b..72c67dd 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
@@ -157,15 +157,15 @@
getConf());
String query1, query2, query3;
query1 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, 0.0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", null,
+ "select testcube.cityid as alias0, 0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", null,
"group by testcube.cityid",
getWhereForDailyAndHourly2days("testcube", "c1_b1fact1"));
query2 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, 0.0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", null,
+ "select testcube.cityid as alias0, 0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", null,
"group by testcube.cityid",
getWhereForDailyAndHourly2days("testcube", "c0_b2fact1"));
query3 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, sum(testcube.msr2) as alias1, 0.0 as alias2 FROM ", null,
+ "select testcube.cityid as alias0, sum(testcube.msr2) as alias1, 0 as alias2 FROM ", null,
"group by testcube.cityid",
getWhereForHourly2days("testcube", "c1_testfact2"));
compareUnionQuery(ctx,
@@ -214,15 +214,15 @@
join3 = "join " + getDbName()
+ "c1_citytable cubecity ON testcube.cityid = cubecity.id AND (cubecity.dt = 'latest')";
query1 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, 0.0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", join1,
+ "select testcube.cityid as alias0, 0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", join1,
"cubecity1.name='blah'", "group by testcube.cityid", null,
getWhereForDailyAndHourly2days("testcube", "c1_b1fact1"));
query2 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, 0.0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", join2,
+ "select testcube.cityid as alias0, 0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", join2,
"cubecity2.name='blah'", "group by testcube.cityid", null,
getWhereForDailyAndHourly2days("testcube", "c0_b2fact1"));
query3 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, sum(testcube.msr2) as alias1, 0.0 as alias2 FROM ", join3,
+ "select testcube.cityid as alias0, sum(testcube.msr2) as alias1, 0 as alias2 FROM ", join3,
"cubecity.name='blah'", "group by testcube.cityid", null,
getWhereForHourly2days("testcube", "c1_testfact2"));
compareUnionQuery(ctx,
@@ -271,17 +271,17 @@
+ " xusersports.user_id";
query1 = getExpectedQuery("testcube",
"select (usersports.balias0) AS `alias0`, (xusersports.balias0) AS `alias1`, (yusersports.balias0) AS `alias2`, "
- + "sum((testcube.segmsr1)) AS `alias3`, 0.0 AS `alias4` FROM ", joinExpr, null,
+ + "sum((testcube.segmsr1)) AS `alias3`, 0 AS `alias4` FROM ", joinExpr, null,
"group by (usersports.balias0), (xusersports.balias0), (yusersports.balias0), ", null,
getWhereForDailyAndHourly2days("testcube", "c1_b1fact1"));
query2 = getExpectedQuery("testcube",
"select (usersports.balias0) AS `alias0`, (xusersports.balias0) AS `alias1`, (yusersports.balias0) AS `alias2`, "
- + "sum((testcube.segmsr1)) AS `alias3`, 0.0 AS `alias4` FROM ", joinExpr, null,
+ + "sum((testcube.segmsr1)) AS `alias3`, 0 AS `alias4` FROM ", joinExpr, null,
"group by (usersports.balias0), (xusersports.balias0), (yusersports.balias0)", null,
getWhereForDailyAndHourly2days("testcube", "c0_b2fact1"));
query3 = getExpectedQuery("testcube",
"select (usersports.balias0) AS `alias0`, (xusersports.balias0) AS `alias1`, (yusersports.balias0) AS `alias2`, "
- + "0.0 AS `alias3`, sum(testcube.msr2) AS `alias4` FROM ", joinExpr, null,
+ + "0 AS `alias3`, sum(testcube.msr2) AS `alias4` FROM ", joinExpr, null,
"group by (usersports.balias0), (xusersports.balias0), (yusersports.balias0)", null,
getWhereForHourly2days("testcube", "c1_testfact2"));
compareUnionQuery(ctx,
@@ -297,15 +297,15 @@
CubeQueryContext ctx = rewriteCtx(userQuery, getConf());
String query1, query2, query3;
query1 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, 0.0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", null,
+ "select testcube.cityid as alias0, 0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", null,
"group by testcube.cityid",
getWhereForDailyAndHourly2days("testcube", "c1_b1fact1"));
query2 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, 0.0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", null,
+ "select testcube.cityid as alias0, 0 as alias1, sum(testcube.segmsr1) as alias2 FROM ", null,
"group by testcube.cityid",
getWhereForDailyAndHourly2days("testcube", "c0_b2fact1"));
query3 = getExpectedQuery("testcube",
- "select testcube.cityid as alias0, sum(testcube.msr2) as alias1, 0.0 as alias2 FROM ", null,
+ "select testcube.cityid as alias0, sum(testcube.msr2) as alias1, 0 as alias2 FROM ", null,
"group by testcube.cityid",
getWhereForHourly2days("testcube", "c1_testfact2"));
compareUnionQuery(ctx,
@@ -316,11 +316,11 @@
@Test
public void testSegmentationWithSingleSegment() throws LensException {
- String userQuery = "select segmsr1 from basecube where " + TWO_DAYS_RANGE;
+ String userQuery = "select zipcode, segmsr1 from basecube where " + TWO_DAYS_RANGE + " having segmsr1 > 10";
String actual = rewrite(userQuery, getConf());
String expected = getExpectedQuery("basecube",
- "select sum(basecube.segmsr1) FROM ", null,
- null,
+ "select basecube.zipcode, sum(basecube.segmsr1) FROM ", null,
+ "group by basecube.zipcode having sum(basecube.segmsr1) > 10",
getWhereForDailyAndHourly2days("basecube", "c1_b1fact1"));
compareQueries(actual, expected);
}
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
index ccc3bf4..c16062c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
@@ -100,11 +100,11 @@
assertTrue(rewrittenQuery.contains("UNION ALL"));
String expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, sum(case "
+ "when ((cubecityjoinunionctx.name) = 'blr') then (basecube.union_join_ctx_msr1) else 0 end) "
- + "as `alias1`, 0.0 as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact1 basecube ";
+ + "as `alias1`, 0 as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact1 basecube ";
String expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, "
+ "sum(case when ((cubecityjoinunionctx.name) = 'blr') then (basecube.union_join_ctx_msr1) else 0 end) "
- + "as `alias1`, 0.0 as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact2 basecube";
- String expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0.0 as `alias1`, "
+ + "as `alias1`, 0 as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact2 basecube";
+ String expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0 as `alias1`, "
+ "sum(case when ((cubecityjoinunionctx.name) = 'blr') then (basecube.union_join_ctx_msr2) else 0 end) "
+ "as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact3 basecube";
String outerSelect = "SELECT (basecube.alias0) as `union_join_ctx_cityid`, sum((basecube.alias1)) "
@@ -129,12 +129,12 @@
String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf);
assertTrue(rewrittenQuery.contains("UNION ALL"));
String expectedInnerSelect1 = "SELECT case when (basecube.union_join_ctx_cityid) is null then 0 "
- + "else (basecube.union_join_ctx_cityid) end as `alias0`, 0.0 as `alias1`, "
+ + "else (basecube.union_join_ctx_cityid) end as `alias0`, 0 as `alias1`, "
+ "sum((basecube.union_join_ctx_msr2)) as `alias2`, sum((basecube.union_join_ctx_msr4)) as `alias3` "
+ "FROM TestQueryRewrite.c1_union_join_ctx_fact4 basecube";
String expectedInnerSelect2 = "SELECT case when (basecube.union_join_ctx_cityid) is null then 0 "
+ "else (basecube.union_join_ctx_cityid) end as `alias0`, (basecube.union_join_ctx_msr22) as `alias1`, "
- + "0.0 as `alias2`, 0.0 as `alias3` FROM TestQueryRewrite.c1_union_join_ctx_fact3 basecube";
+ + "0 as `alias2`, 0 as `alias3` FROM TestQueryRewrite.c1_union_join_ctx_fact3 basecube";
String outerSelect = "SELECT (basecube.alias0) as `union_join_ctx_notnullcityid`, (basecube.alias1) "
+ "as `union_join_ctx_msr22`, case when ((sum((basecube.alias2)) + 0) = 0) then 0 else "
+ "(((sum((basecube.alias3)) + 0) * 100) / (sum((basecube.alias2)) + 0)) end as "
@@ -150,19 +150,19 @@
@Test
public void testDuplicateMeasureProjectionInJoinCandidate() throws ParseException, LensException {
// union_join_ctx_msr2 is common between two storage candidates and it should be answered from one
- // and the other fact will have it replaced with 0.0
+ // and the other fact will have it replaced with 0
String colsSelected = " union_join_ctx_notnullcityid, sum(union_join_ctx_msr22) , "
+ "sum(union_join_ctx_msr2), sum(union_join_ctx_msr4) ";
String whereCond = "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")";
String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf);
assertTrue(rewrittenQuery.contains("UNION ALL"));
String expectedInnerSelect1 = "SELECT case when (basecube.union_join_ctx_cityid) is null then 0 "
- + "else (basecube.union_join_ctx_cityid) end as `alias0`, 0.0 as `alias1`, "
+ + "else (basecube.union_join_ctx_cityid) end as `alias0`, 0 as `alias1`, "
+ "sum((basecube.union_join_ctx_msr2)) as `alias2`, sum((basecube.union_join_ctx_msr4)) "
+ "as `alias3` FROM TestQueryRewrite.c1_union_join_ctx_fact4 basecube";
String expectedInnerSelect2 = "SELECT case when (basecube.union_join_ctx_cityid) is null then 0 else "
+ "(basecube.union_join_ctx_cityid) end as `alias0`, sum((basecube.union_join_ctx_msr22)) as `alias1`, "
- + "0.0 as `alias2`, 0.0 as `alias3` FROM TestQueryRewrite.c1_union_join_ctx_fact3 basecube";
+ + "0 as `alias2`, 0 as `alias3` FROM TestQueryRewrite.c1_union_join_ctx_fact3 basecube";
String outerSelect = "SELECT (basecube.alias0) as `union_join_ctx_notnullcityid`, sum((basecube.alias1)) "
+ "as `sum(union_join_ctx_msr22)`, sum((basecube.alias2)) as `sum(union_join_ctx_msr2)`, "
+ "sum((basecube.alias3)) as `sum(union_join_ctx_msr4)` FROM";
@@ -182,12 +182,12 @@
String whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and "
+ "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")";
String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond + having, conf);
- String expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0.0 as `alias1`, "
+ String expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0 as `alias1`, "
+ "sum((basecube.union_join_ctx_msr1)) as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact1 basecube ";
- String expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0.0 as `alias1`, "
+ String expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0 as `alias1`, "
+ "sum((basecube.union_join_ctx_msr1)) as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact2 basecube ";
String expectedInnerSelect3 = " SELECT (basecube.union_join_ctx_cityid) as `alias0`, "
- + "sum((basecube.union_join_ctx_msr2)) as `alias1`, 0.0 as `alias2` "
+ + "sum((basecube.union_join_ctx_msr2)) as `alias1`, 0 as `alias2` "
+ "FROM TestQueryRewrite.c1_union_join_ctx_fact3 basecube ";
String outerHaving = "HAVING (sum((basecube.alias2)) > 100)";
compareContains(expectedInnerSelect1, rewrittenQuery);
@@ -208,14 +208,14 @@
expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) "
+ "as `alias1`, case when (basecube.union_join_ctx_cityid) is null then 0 else "
+ "(basecube.union_join_ctx_cityid) end as `alias2`, sum((basecube.union_join_ctx_msr1)) as `alias3`, "
- + "0.0 as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact1 basecube";
+ + "0 as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact1 basecube";
expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) "
+ "as `alias1`, case when (basecube.union_join_ctx_cityid) is null then 0 else "
+ "(basecube.union_join_ctx_cityid) end as `alias2`, sum((basecube.union_join_ctx_msr1)) as `alias3`, "
- + "0.0 as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact2";
+ + "0 as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact2";
expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) "
+ "as `alias1`, case when (basecube.union_join_ctx_cityid) is null then 0 else "
- + "(basecube.union_join_ctx_cityid) end as `alias2`, 0.0 as `alias3`, "
+ + "(basecube.union_join_ctx_cityid) end as `alias2`, 0 as `alias3`, "
+ "sum((basecube.union_join_ctx_msr2)) as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact3";
String outerGroupBy = "GROUP BY (basecube.alias0), (basecube.alias1), (basecube.alias2)";
compareContains(outerSelect, rewrittenQuery);
@@ -239,12 +239,12 @@
+ "as `(sum(union_join_ctx_msr1) + 10)` FROM ";
expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, "
+ "(cubecityjoinunionctx.name) as `alias1`, sum((basecube.union_join_ctx_msr1)) as `alias2`, "
- + "0.0 as `alias3`, 0.0 as `alias4` FROM";
+ + "0 as `alias3`, 0 as `alias4` FROM";
expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, "
+ "(cubecityjoinunionctx.name) as `alias1`, sum((basecube.union_join_ctx_msr1)) as `alias2`, "
- + "0.0 as `alias3`, 0.0 as `alias4` FROM";
+ + "0 as `alias3`, 0 as `alias4` FROM";
expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) "
- + "as `alias1`, 0.0 as `alias2`, sum((basecube.union_join_ctx_msr2)) as `alias3`, "
+ + "as `alias1`, 0 as `alias2`, sum((basecube.union_join_ctx_msr2)) as `alias3`, "
+ "sum(case when ((basecube.union_join_ctx_msr2) > 0) then (basecube.union_join_ctx_msr2) else 0 end) "
+ "as `alias4` FROM";
String innerGroupBy = "GROUP BY (basecube.union_join_ctx_cityid), (cubecityjoinunionctx.name)";
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
index 94e836c..fe6b20d 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
@@ -86,11 +86,11 @@
public void testVirtualFactUnionQuery() throws Exception {
String expectedInnerSelect = getExpectedQuery("virtualcube", "SELECT (virtualcube.cityid) AS `alias0`,"
- + " sum((virtualcube.msr2)) AS `alias1`,0.0 AS `alias2` FROM ",
+ + " sum((virtualcube.msr2)) AS `alias1`,0 AS `alias2` FROM ",
null, null, "GROUP BY (virtualcube.cityid)", null,
getWhereForDailyAndHourly2days("virtualcube", "c1_testfact8_base"))
+ " UNION ALL " + getExpectedQuery("virtualcube", "SELECT (virtualcube.cityid) AS `alias0`,"
- + "0.0 AS `alias1`, sum((virtualcube.msr3)) AS `alias2` FROM ",
+ + "0 AS `alias1`, sum((virtualcube.msr3)) AS `alias2` FROM ",
null, null, "GROUP BY (virtualcube.cityid)", null,
getWhereForDailyAndHourly2days("virtualcube", "c1_testfact7_base"));
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
index b5e73fd..8402253 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
@@ -47,7 +47,8 @@
public class TestFactPartitionBasedQueryCostCalculator {
AbstractQueryContext queryContext;
- FactPartitionBasedQueryCostCalculator calculator = new FactPartitionBasedQueryCostCalculator();
+ FactPartitionBasedQueryCostCalculator calculator =
+ new FactPartitionBasedQueryCostCalculator();
LensDriver driver;
private static String latest = "latest";
@@ -56,6 +57,7 @@
driver = mock(LensDriver.class);
when(driver.getConf()).thenReturn(new Configuration());
queryContext = mock(AbstractQueryContext.class);
+ calculator.init(driver);
ImmutableMap<String, Double> tableWeights = new ImmutableMap.Builder<String, Double>().build();
diff --git a/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml b/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml
index c23a029..5a600ae 100644
--- a/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml
+++ b/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml
@@ -45,5 +45,6 @@
<attr_name>union_join_ctx_dup_cityname</attr_name>
<attr_name>d_time</attr_name>
<attr_name>union_join_ctx_cityid</attr_name>
+ <attr_name>zipcode</attr_name>
</dim_attr_names>
</x_derived_cube>
diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml
index 2afa04d..fb49931 100644
--- a/lens-dist/pom.xml
+++ b/lens-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-dist</artifactId>
diff --git a/lens-docker/lens-test/Dockerfile b/lens-docker/lens-test/Dockerfile
index 665e519..52680b5 100644
--- a/lens-docker/lens-test/Dockerfile
+++ b/lens-docker/lens-test/Dockerfile
@@ -24,7 +24,7 @@
ENV SPARK_HOME /usr/local/spark-1.3.0-bin-hadoop2.4
RUN rm spark-1.3.0-bin-hadoop2.4.tar
-ENV LENS_VERSION 2.7.0-SNAPSHOT
+ENV LENS_VERSION 2.8.0-SNAPSHOT
ENV BASEDIR /opt/lens
ENV LENS_HOME $BASEDIR/lens-dist/target/apache-lens-${LENS_VERSION}-bin/apache-lens-${LENS_VERSION}-bin/server
ENV LENS_CLIENT $BASEDIR/lens-dist/target/apache-lens-${LENS_VERSION}-bin/apache-lens-${LENS_VERSION}-bin/client
diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh
index 0d480e1..6a15110 100644
--- a/lens-docker/lens-test/lens-bootstrap.sh
+++ b/lens-docker/lens-test/lens-bootstrap.sh
@@ -37,7 +37,7 @@
export SPARK_YARN_JAR
echo "SPARK_YARN_JAR " $SPARK_YARN_JAR
-HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.7.0-SNAPSHOT.jar,$SPARK_YARN_JAR
+HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.8.0-SNAPSHOT.jar,$SPARK_YARN_JAR
export HIVE_AUX_JARS_PATH
echo "HIVE_AUX_JARS_PATH " $HIVE_AUX_JARS_PATH
diff --git a/lens-driver-es/pom.xml b/lens-driver-es/pom.xml
index e5c0a62..7823a62 100644
--- a/lens-driver-es/pom.xml
+++ b/lens-driver-es/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.lens</groupId>
<artifactId>apache-lens</artifactId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-driver-es</artifactId>
diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml
index e4d3ce4..51205e9 100644
--- a/lens-driver-hive/pom.xml
+++ b/lens-driver-hive/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<properties>
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 0b4db9a..2eb94aa 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -1,4 +1,4 @@
-/**
+ /**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -44,9 +44,7 @@
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
-import org.apache.lens.server.api.query.cost.QueryCost;
-import org.apache.lens.server.api.query.cost.QueryCostCalculator;
+import org.apache.lens.server.api.query.cost.*;
import org.apache.lens.server.api.query.priority.CostRangePriorityDecider;
import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf;
import org.apache.lens.server.api.query.priority.QueryPriorityDecider;
@@ -337,6 +335,7 @@
isEmbedded = (connectionClass.getName().equals(EmbeddedThriftConnection.class.getName()));
connectionExpiryTimeout = getConf().getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY);
whetherCalculatePriority = getConf().getBoolean(HS2_CALCULATE_PRIORITY, true);
+
Class<? extends QueryCostCalculator> queryCostCalculatorClass = getConf().getClass(HS2_COST_CALCULATOR,
FactPartitionBasedQueryCostCalculator.class, QueryCostCalculator.class);
try {
@@ -344,6 +343,8 @@
} catch (InstantiationException | IllegalAccessException e) {
throw new LensException("Can't instantiate query cost calculator of class: " + queryCostCalculatorClass, e);
}
+ //For initializing the decider class instance
+ queryCostCalculator.init(this);
queryPriorityDecider = new CostRangePriorityDecider(
new CostToPriorityRangeConf(getConf().get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES))
);
diff --git a/lens-driver-hive/src/main/resources/hivedriver-default.xml b/lens-driver-hive/src/main/resources/hivedriver-default.xml
index a13d3b0..2b2dc0b 100644
--- a/lens-driver-hive/src/main/resources/hivedriver-default.xml
+++ b/lens-driver-hive/src/main/resources/hivedriver-default.xml
@@ -165,4 +165,16 @@
implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.</description>
</property>
+ <property>
+ <name>lens.driver.cost.query.decider.class</name>
+ <value>org.apache.lens.server.api.query.cost.RangeBasedQueryCostTypeDecider</value>
+ <description>Decider class which looks at ranges passed in config and decides the querycosttype </description>
+ </property>
+
+ <property>
+ <name>lens.driver.cost.type.ranges</name>
+ <value>LOW,0.0,HIGH</value>
+ <description>Cost based Query type mapping</description>
+ </property>
+
</configuration>
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 1261409..49bfb68 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -89,7 +89,6 @@
protected SessionState ss;
private CostRangePriorityDecider alwaysNormalPriorityDecider
= new CostRangePriorityDecider(new CostToPriorityRangeConf(""));
-
/**
* Before test.
*
diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml
index cdd6e38..5acd5d2 100644
--- a/lens-driver-jdbc/pom.xml
+++ b/lens-driver-jdbc/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-driver-jdbc</artifactId>
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 528f857..e810e7c 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -40,6 +40,7 @@
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.cube.parse.HQLParser;
+import org.apache.lens.cube.query.cost.StaticCostCalculator;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.error.LensDriverErrorCode;
@@ -51,8 +52,7 @@
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory;
-import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
-import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.cost.*;
import org.apache.lens.server.api.query.rewrite.QueryRewriter;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.lens.server.model.LogSegregationContext;
@@ -99,6 +99,8 @@
private boolean isStatementCancelSupported;
+ QueryCostCalculator queryCostCalculator;
+
/**
* Data related to a query submitted to JDBCDriver.
*/
@@ -382,6 +384,17 @@
super.configure(conf, driverType, driverName);
init();
configured = true;
+ Class<? extends QueryCostCalculator> queryCostCalculatorClass = getConf().getClass(JDBC_COST_CALCULATOR,
+ StaticCostCalculator.class, QueryCostCalculator.class);
+ try {
+ queryCostCalculator = queryCostCalculatorClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new LensException("Can't instantiate query cost calculator of class: " + queryCostCalculatorClass, e);
+ }
+
+ //For initializing the decider class instance
+ queryCostCalculator.init(this);
+
log.info("JDBC Driver {} configured", getFullyQualifiedName());
}
@@ -421,6 +434,9 @@
DEFAULT_STATEMENT_CANCEL_SUPPORTED);
}
+ public QueryCost calculateQueryCost(AbstractQueryContext qctx) throws LensException {
+ return queryCostCalculator.calculateCost(qctx, this);
+ }
/**
* Check configured.
*
@@ -513,22 +529,21 @@
return rewrittenQuery;
}
- static final QueryCost JDBC_DRIVER_COST = new FactPartitionBasedQueryCost(0);
-
/**
* Dummy JDBC query Plan class to get min cost selector working.
*/
private static class JDBCQueryPlan extends DriverQueryPlan {
+
+ @Getter
+ private final QueryCost cost;
+
+ JDBCQueryPlan(QueryCost cost){
+ this.cost = cost;
+ }
@Override
public String getPlan() {
return "";
}
-
- @Override
- public QueryCost getCost() {
- // this means that JDBC driver is only selected for tables with just DB storage.
- return JDBC_DRIVER_COST;
- }
}
private static final String VALIDATE_GAUGE = "validate-thru-prepare";
@@ -542,7 +557,7 @@
VALIDATE_GAUGE);
validate(qctx);
validateGauge.markSuccess();
- return JDBC_DRIVER_COST;
+ return calculateQueryCost(qctx);
}
/**
@@ -590,7 +605,7 @@
result.close();
}
}
- JDBCQueryPlan jqp = new JDBCQueryPlan();
+ JDBCQueryPlan jqp = new JDBCQueryPlan(calculateQueryCost(explainCtx));
explainCtx.getDriverContext().setDriverQueryPlan(this, jqp);
return jqp;
}
@@ -802,7 +817,7 @@
public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException {
checkConfigured();
prepare(pContext);
- return new JDBCQueryPlan();
+ return new JDBCQueryPlan(calculateQueryCost(pContext));
}
/**
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
index 51abc96..f2bfb69 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
@@ -51,6 +51,10 @@
/** The Constant JDBC_PASSWORD. */
public static final String JDBC_PASSWORD = JDBC_DRIVER_PFX + "db.password";
+ public static final String JDBC_COST_CALCULATOR = JDBC_DRIVER_PFX + "cost.calculator.class";
+
+ public static final String JDBC_COST_QUERY_DECIDER = JDBC_DRIVER_PFX + "cost.query.decider.class";
+
public enum ConnectionPoolProperties {
/** The Constant JDBC_POOL_MAX_SIZE_DEFAULT. */
JDBC_POOL_MAX_SIZE("maxPoolSize", JDBC_DRIVER_PFX + "pool.max.size", 15),
diff --git a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
index 5545fac..511fe71 100644
--- a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
+++ b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
@@ -269,4 +269,29 @@
<description>Flag to indicate Whether cancel on JDBC statement is supported. If not supported,
framework wont call cancel on JDBC statement.</description>
</property>
+
+ <property>
+ <name>lens.driver.jdbc.cost.calculator.class</name>
+ <value>org.apache.lens.cube.query.cost.StaticCostCalculator</value>
+ <description>Cost calculator class. By default calculating cost through static values </description>
+ </property>
+
+ <property>
+ <name>lens.driver.query.cost</name>
+ <value>0.0</value>
+ <description>Jdbc driver static cost value</description>
+ </property>
+
+ <property>
+ <name>lens.driver.cost.query.decider.class</name>
+ <value>org.apache.lens.server.api.query.cost.RangeBasedQueryCostTypeDecider</value>
+ <description>Decider class which looks at ranges passed in config and decides the querycosttype </description>
+ </property>
+
+ <property>
+ <name>lens.driver.cost.type.ranges</name>
+ <value>LOW,0.0,HIGH</value>
+ <description>Cost based Query type mapping</description>
+ </property>
+
</configuration>
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 446e759..506935b 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -21,6 +21,8 @@
import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.*;
import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.ConnectionPoolProperties.*;
+import static org.apache.lens.server.api.LensConfConstants.DRIVER_COST_TYPE_RANGES;
+
import static org.testng.Assert.*;
import java.sql.*;
@@ -29,6 +31,7 @@
import java.util.concurrent.TimeUnit;
import org.apache.lens.api.LensConf;
+import org.apache.lens.api.query.QueryCostType;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.ResultRow;
import org.apache.lens.server.api.LensConfConstants;
@@ -40,6 +43,7 @@
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.cost.StaticQueryCost;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -72,6 +76,8 @@
Collection<LensDriver> drivers;
+ static final StaticQueryCost JDBC_COST = new StaticQueryCost(0.0, QueryCostType.LOW);
+
/**
* Test create jdbc driver.
*
@@ -85,6 +91,7 @@
baseConf.set(JDBC_USER, "SA");
baseConf.set(JDBC_PASSWORD, "");
baseConf.set(JDBC_EXPLAIN_KEYWORD_PARAM, "explain plan for ");
+ baseConf.set(DRIVER_COST_TYPE_RANGES, "VERY_LOW,0.0,LOW,0.001,HIGH");
hConf = new HiveConf(baseConf, this.getClass());
driver = new JDBCDriver();
@@ -263,7 +270,7 @@
ExplainQueryContext ctx = createExplainContext(query1, baseConf);
Assert.assertNull(ctx.getFinalDriverQuery(driver));
QueryCost cost = driver.estimate(ctx);
- Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST);
+ Assert.assertEquals(cost, JDBC_COST);
Assert.assertNotNull(ctx.getFinalDriverQuery(driver));
// Test connection leak for estimate
@@ -336,7 +343,7 @@
// run estimate and execute - because server would first run estimate and then execute with same context
QueryContext ctx = createQueryContext(query1, metricConf);
QueryCost cost = driver.estimate(ctx);
- Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST);
+ Assert.assertEquals(cost, JDBC_COST);
LensResultSet result = driver.execute(ctx);
Assert.assertNotNull(result);
@@ -344,13 +351,13 @@
// run estimate and prepare - because server would first run estimate and then prepare with same context
PreparedQueryContext pContext = new PreparedQueryContext(query1, "SA", metricConf, drivers);
cost = driver.estimate(pContext);
- Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST);
+ Assert.assertEquals(cost, JDBC_COST);
driver.prepare(pContext);
// test explain and prepare
PreparedQueryContext pContext2 = new PreparedQueryContext(query1, "SA", metricConf, drivers);
cost = driver.estimate(pContext2);
- Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST);
+ Assert.assertEquals(cost, JDBC_COST);
driver.prepare(pContext2);
driver.explainAndPrepare(pContext2);
}
diff --git a/lens-examples/pom.xml b/lens-examples/pom.xml
index 1866d04..abf349f 100644
--- a/lens-examples/pom.xml
+++ b/lens-examples/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-examples</artifactId>
diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml
index a431a41..c75e2ab 100644
--- a/lens-ml-dist/pom.xml
+++ b/lens-ml-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-ml-dist</artifactId>
diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml
index 143373e..959efe8 100644
--- a/lens-ml-lib/pom.xml
+++ b/lens-ml-lib/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<properties>
diff --git a/lens-query-lib/pom.xml b/lens-query-lib/pom.xml
index 3a8b33c..c21e77f 100644
--- a/lens-query-lib/pom.xml
+++ b/lens-query-lib/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-query-lib</artifactId>
diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml
index 89cf44e..a4f857d 100644
--- a/lens-regression/pom.xml
+++ b/lens-regression/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-regression</artifactId>
diff --git a/lens-server-api/pom.xml b/lens-server-api/pom.xml
index d04e1f9..93f79ff 100644
--- a/lens-server-api/pom.xml
+++ b/lens-server-api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-server-api</artifactId>
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 7fd487c..d5273be 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -1253,4 +1253,26 @@
* Default Value of the config "lens.cube.metastore.enable.datacompleteness.check"
*/
public static final boolean DEFAULT_ENABLE_DATACOMPLETENESS_CHECK = false;
+
+ /**
+ * This property is for setting static cost to driver
+ */
+ public static final String DRIVER_QUERY_COST = DRIVER_PFX + "query.cost";
+
+ /**
+ * Config param for defining query type ranges.
+ */
+ public static final String DRIVER_COST_TYPE_RANGES = "lens.driver.cost.type.ranges";
+
+ /*
+ * Default query cost range
+ * */
+ public static final String DRIVER_QUERY_COST_TYPE_DEFAULT_RANGES = "LOW,0.0,HIGH";
+
+ /**
+ * Default Value of the config "lens.driver.query.cost"
+ */
+ public static final double DEFAULT_DRIVER_QUERY_COST = 0.0;
+
+ public static final String DRIVER_COST_QUERY_DECIDER = DRIVER_PFX + "cost.query.decider.class";
}
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
index 9d16533..4780955 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
@@ -536,6 +536,9 @@
void dropPartitionFromStorageByFilter(LensSessionHandle sessionid,
String cubeTableName, String storageName, String filter) throws LensException;
+ void dropPartitionFromStorageByFilter(LensSessionHandle sessionid,
+ String cubeTableName, String storageName, String filter, String updatePeriod) throws LensException;
+
/**
* Get flattened columns - all columns of table + all reachable columns
*
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
index eba8f0d..2e23dd4 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
@@ -27,17 +27,24 @@
import com.google.common.base.Preconditions;
import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
import lombok.ToString;
@ToString
@EqualsAndHashCode
public class FactPartitionBasedQueryCost implements QueryCost<FactPartitionBasedQueryCost>, Serializable {
+ @Getter
private final double partitionCost;
+ @Getter
+ @Setter
+ private QueryCostType queryCostType;
public FactPartitionBasedQueryCost(final double partitionCost) {
Preconditions.checkArgument(partitionCost >= 0, "Cost can't be negative");
this.partitionCost = partitionCost;
+ this.queryCostType = QueryCostType.HIGH;
}
@Override
@@ -46,11 +53,6 @@
}
@Override
- public QueryCostType getQueryCostType() {
- return partitionCost == 0 ? QueryCostType.LOW : QueryCostType.HIGH;
- }
-
- @Override
public long getEstimatedExecTimeMillis() throws UnsupportedOperationException {
throw new UnsupportedOperationException("Estimated time is not implemented");
}
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
index 9d7320a..fcbbeb6 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
@@ -21,7 +21,6 @@
*/
package org.apache.lens.server.api.query.cost;
-
import org.apache.lens.api.query.QueryCostType;
/**
@@ -34,8 +33,16 @@
QueryCostType getQueryCostType();
+ void setQueryCostType(QueryCostType queryCostType);
+
long getEstimatedExecTimeMillis() throws UnsupportedOperationException;
double getEstimatedResourceUsage() throws UnsupportedOperationException;
+ @Override
+ default int compareTo(QueryCost queryCost) {
+ return (new Double(this.getEstimatedResourceUsage()))
+ .compareTo(queryCost.getEstimatedResourceUsage());
+ }
+
}
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostCalculator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostCalculator.java
index 8c71c00..bf8e936 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostCalculator.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostCalculator.java
@@ -23,6 +23,8 @@
import org.apache.lens.server.api.query.AbstractQueryContext;
public interface QueryCostCalculator {
+
+ void init(LensDriver lensDriver) throws LensException;
/**
* @param queryContext
* @return calculated cost based on anything available in abstract query context
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTypeDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTypeDecider.java
new file mode 100644
index 0000000..8b2fdbe
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTypeDecider.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.cost;
+
+
+import org.apache.lens.api.query.QueryCostType;
+import org.apache.lens.server.api.error.LensException;
+
+public interface QueryCostTypeDecider {
+
+ /**
+ * @param cost
+ * @return calculate queryCostType based on the pre calculated query cost
+ * @throws LensException when can't decide queryCostType.
+ */
+ QueryCostType decideCostType(QueryCost cost) throws LensException;
+}
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTypeRangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTypeRangeConf.java
new file mode 100644
index 0000000..4a9daae
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTypeRangeConf.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.query.cost;
+
+
+import org.apache.lens.api.query.QueryCostType;
+import org.apache.lens.server.api.query.priority.RangeConf;
+
+public class QueryCostTypeRangeConf extends RangeConf<Double, QueryCostType> {
+ /**
+ * Super constructor
+ *
+ * @param confValue
+ * @see RangeConf#RangeConf(String)
+ */
+ public QueryCostTypeRangeConf(String confValue) {
+ super(confValue);
+ }
+
+ /**
+ * Parse key method
+ *
+ * @param s
+ * @return parsed float from string s
+ * @see RangeConf#parseKey(String)
+ */
+ @Override
+ protected Double parseKey(String s) {
+ return Double.parseDouble(s);
+ }
+
+ /**
+ * Parse value method
+ *
+ * @param s
+ * @return parsed QueryCostType from String s
+ * @see RangeConf#parseValue(String)
+ */
+ @Override
+ protected QueryCostType parseValue(String s) {
+ return QueryCostType.valueOf(s);
+ }
+
+ /**
+ * Default value is "HIGH".
+ * @return "HIGH"
+ */
+ @Override
+ protected String getDefaultConf() {
+ return QueryCostType.HIGH.toString();
+ }
+}
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/RangeBasedQueryCostTypeDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/RangeBasedQueryCostTypeDecider.java
new file mode 100644
index 0000000..bc97594
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/RangeBasedQueryCostTypeDecider.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.cost;
+
+import org.apache.lens.api.query.QueryCostType;
+import org.apache.lens.server.api.error.LensException;
+
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class RangeBasedQueryCostTypeDecider implements QueryCostTypeDecider {
+
+ @NonNull
+ private final QueryCostTypeRangeConf queryCostTypeRangeMap;
+
+ public RangeBasedQueryCostTypeDecider(String queryCostTypeRange) {
+ this.queryCostTypeRangeMap = new QueryCostTypeRangeConf(queryCostTypeRange);
+ }
+
+ @Override
+ public QueryCostType decideCostType(@NonNull final QueryCost cost) throws LensException {
+ QueryCostType q = queryCostTypeRangeMap.get(cost.getEstimatedResourceUsage());
+ log.info("cost was: {}, decided querytype: {}", cost, q);
+ return q;
+ }
+}
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/StaticQueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/StaticQueryCost.java
new file mode 100644
index 0000000..5377ab2
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/StaticQueryCost.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.query.cost;
+
+import java.io.Serializable;
+
+import org.apache.lens.api.query.QueryCostType;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+@ToString
+@EqualsAndHashCode
+public class StaticQueryCost implements QueryCost<StaticQueryCost>, Serializable {
+
+ private final double staticCost;
+ @Getter
+ @Setter
+ private QueryCostType queryCostType;
+
+ public StaticQueryCost(final double cost) {
+ this.staticCost = cost;
+ this.queryCostType = QueryCostType.HIGH;
+ }
+
+ //Added for testcase
+ public StaticQueryCost(final double cost, final QueryCostType queryCostType) {
+ this.staticCost = cost;
+ this.queryCostType = queryCostType;
+ }
+
+ @Override
+ public StaticQueryCost add(final StaticQueryCost other) {
+ return new StaticQueryCost(staticCost + other.staticCost);
+ }
+
+ @Override
+ public long getEstimatedExecTimeMillis() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("Estimated time is not implemented");
+ }
+
+ @Override
+ public double getEstimatedResourceUsage() throws UnsupportedOperationException {
+ return staticCost;
+ }
+
+ @Override
+ public int compareTo(final StaticQueryCost staticQueryCost) {
+ return new Double(staticCost).compareTo(staticQueryCost.staticCost);
+ }
+
+ @Override
+ public String toString() {
+ return getQueryCostType() + "(" + getEstimatedResourceUsage() + ")";
+ }
+}
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/RangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/RangeConf.java
index 43cc53c..fd44be6 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/RangeConf.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/RangeConf.java
@@ -62,7 +62,7 @@
*
* @param confValue
*/
- RangeConf(String confValue) {
+ public RangeConf(String confValue) {
if (confValue == null || confValue.isEmpty()) {
confValue = getDefaultConf();
}
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/MockQueryCostCalculator.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/MockQueryCostCalculator.java
index fb4085e..72223f7 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/MockQueryCostCalculator.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/MockQueryCostCalculator.java
@@ -25,6 +25,11 @@
public class MockQueryCostCalculator implements QueryCostCalculator {
@Override
+ public void init(LensDriver lensDriver) throws LensException {
+
+ }
+
+ @Override
public QueryCost calculateCost(AbstractQueryContext queryContext, LensDriver driver) throws LensException {
return new FactPartitionBasedQueryCost(10.0);
}
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
index e047c91..bd93d02 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
@@ -25,16 +25,34 @@
import org.apache.lens.api.query.QueryCostType;
import org.apache.lens.api.serialize.SerializationTest;
+import org.apache.lens.server.api.error.LensException;
+import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
public class TestFactPartitionBasedQueryCost {
+
+ RangeBasedQueryCostTypeDecider costRangeQueryTypeDecider =
+ new RangeBasedQueryCostTypeDecider(new QueryCostTypeRangeConf("VERY_LOW,0.0,LOW,0.1,HIGH"));
QueryCost cost0 = new FactPartitionBasedQueryCost(0.0);
QueryCost cost1 = new FactPartitionBasedQueryCost(0.2);
QueryCost cost11 = new FactPartitionBasedQueryCost(0.2);
QueryCost cost2 = new FactPartitionBasedQueryCost(0.3);
+ QueryCost scost0 = new StaticQueryCost(-1.0);
+ QueryCost scost1 = new StaticQueryCost(0.0);
+ QueryCost scost2 = new StaticQueryCost(1.0);
+ @BeforeTest
+ public void beforeTest() throws LensException {
+ cost0.setQueryCostType(costRangeQueryTypeDecider.decideCostType(cost0));
+ cost1.setQueryCostType(costRangeQueryTypeDecider.decideCostType(cost1));
+ cost11.setQueryCostType(costRangeQueryTypeDecider.decideCostType(cost11));
+ cost2.setQueryCostType(costRangeQueryTypeDecider.decideCostType(cost2));
+ scost0.setQueryCostType(costRangeQueryTypeDecider.decideCostType(scost0));
+ scost1.setQueryCostType(costRangeQueryTypeDecider.decideCostType(scost1));
+ scost2.setQueryCostType(costRangeQueryTypeDecider.decideCostType(scost2));
+ }
@Test(expectedExceptions = {IllegalArgumentException.class})
public void testInvalid() {
new FactPartitionBasedQueryCost(-0.5);
@@ -50,6 +68,7 @@
assertEquals(cost1.getQueryCostType(), QueryCostType.HIGH);
assertEquals(cost2.getQueryCostType(), QueryCostType.HIGH);
assertEquals(cost0.getQueryCostType(), QueryCostType.LOW);
+ assertEquals(scost0.getQueryCostType(), QueryCostType.VERY_LOW);
}
@Test(expectedExceptions = {UnsupportedOperationException.class})
@@ -67,6 +86,9 @@
assertEquals(cost1.compareTo(cost2), -1);
assertEquals(cost2.compareTo(cost1), 1);
assertEquals(cost1.compareTo(cost11), 0);
+ assertEquals(scost1.compareTo(scost0), 1);
+ assertEquals(scost0.compareTo(scost1), -1);
+ assertEquals(scost2.compareTo(scost0), 1);
}
@Test
@@ -80,4 +102,9 @@
public void testFactPartitionBasedQueryCostIsSerializable() {
new SerializationTest().verifySerializationAndDeserialization(new FactPartitionBasedQueryCost(Double.MAX_VALUE));
}
+
+ @Test
+ public void testStaticQueryCostIsSerializable() {
+ new SerializationTest().verifySerializationAndDeserialization(new StaticQueryCost(Double.MAX_VALUE));
+ }
}
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index cb4ed94..92e218f 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<properties>
diff --git a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
index d7ce434..b9fcdd8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
@@ -265,6 +265,11 @@
LensSessionHandle restoredSession = new LensSessionHandle(hiveSessionHandle.getHandleIdentifier().getPublicId(),
hiveSessionHandle.getHandleIdentifier().getSecretId());
SESSION_MAP.put(restoredSession.getPublicId().toString(), restoredSession);
+ SessionUser sessionUser = SESSION_USER_INSTANCE_MAP.get(userName);
+ if (sessionUser == null) {
+ sessionUser = new SessionUser(userName);
+ SESSION_USER_INSTANCE_MAP.put(userName, sessionUser);
+ }
updateSessionsPerUser(userName);
} catch (HiveSQLException e) {
throw new LensException("Error restoring session " + sessionHandle, e);
@@ -340,10 +345,14 @@
}
synchronized (sessionUser) {
Integer sessionCount = SESSIONS_PER_USER.get(userName);
- if (sessionCount == 1) {
- SESSIONS_PER_USER.remove(userName);
- } else {
- SESSIONS_PER_USER.put(userName, --sessionCount);
+ if (null != sessionCount) {
+ if (sessionCount == 1) {
+ SESSIONS_PER_USER.remove(userName);
+ } else {
+ SESSIONS_PER_USER.put(userName, --sessionCount);
+ }
+ }else {
+ log.info("Trying to decrement session count for non existing session {} for user {}: ",sessionHandle, userName);
}
}
}
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index 0a8635c..74806af 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -781,6 +781,10 @@
public void dropPartitionFromStorageByFilter(LensSessionHandle sessionid, String cubeTableName,
String storageName, String filter) throws LensException {
+ dropPartitionFromStorageByFilter(sessionid, cubeTableName, storageName, filter, null);
+ }
+ public void dropPartitionFromStorageByFilter(LensSessionHandle sessionid, String cubeTableName,
+ String storageName, String filter, String updatePeriodString) throws LensException {
try (SessionContext ignored = new SessionContext(sessionid)){
Set<String> storageTables = getAllTablesForStorage(sessionid, cubeTableName, storageName);
List<Partition> partitions = new ArrayList<>();
@@ -788,11 +792,15 @@
for (String tableName : storageTables) {
partitions.addAll(msClient.getPartitionsByFilter(tableName, filter));
}
+ if (updatePeriodString!=null) {
+ partitions.removeIf(part -> !part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD).
+ equals((updatePeriodString.toUpperCase())));
+ }
for (Partition part : partitions) {
try {
Map<String, Date> timeSpec = new HashMap<>();
Map<String, String> nonTimeSpec = new HashMap<>();
- UpdatePeriod updatePeriod = populatePartSpec(part, timeSpec, nonTimeSpec);
+ UpdatePeriod updatePeriod= populatePartSpec(part, timeSpec, nonTimeSpec);
msClient.dropPartition(cubeTableName, storageName, timeSpec, nonTimeSpec, updatePeriod);
} catch (HiveException e) {
if (!(e.getCause() instanceof NoSuchObjectException)) {
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index 0af6a34..8818844 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -1022,10 +1022,11 @@
public APIResult dropPartitionsOfFactStorageByFilter(@QueryParam("sessionid") LensSessionHandle sessionid,
@PathParam("factName") String factName,
@PathParam("storage") String storage,
- @QueryParam("filter") String filter) throws LensException {
+ @QueryParam("filter") String filter,
+ @QueryParam("updatePeriod") String updatePeriod) throws LensException {
checkSessionId(sessionid);
try {
- getSvc().dropPartitionFromStorageByFilter(sessionid, factName, storage, filter);
+ getSvc().dropPartitionFromStorageByFilter(sessionid, factName, storage, filter, updatePeriod);
} catch (LensException exc) {
log.warn("Got exception while dropping partition.", exc);
return partial(processLensException(exc));
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java
index 9a37afd..6e420bf 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java
@@ -26,15 +26,14 @@
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.cost.StaticQueryCost;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.map.MultiValueMap;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import lombok.NonNull;
import lombok.ToString;
@@ -160,18 +159,17 @@
private QueryCost getTotalQueryCost(final Collection<QueryContext> queries) {
if (queries.isEmpty()) {
- return new FactPartitionBasedQueryCost(0);
+ return new StaticQueryCost(0);
}
- QueryContext query0 = Iterables.get(queries, 0);
- QueryCost totalQueryCost = query0.getSelectedDriverQueryCost();
+ Double totalQueryCost = 0d;
for (QueryContext query : queries) {
- QueryCost queryCost = query.getSelectedDriverQueryCost();
- totalQueryCost = totalQueryCost.add(queryCost);
+ Double queryCost = query.getSelectedDriverQueryCost().getEstimatedResourceUsage();
+ totalQueryCost = totalQueryCost + queryCost;
}
log.debug("Total Query Cost:{}", totalQueryCost);
- return totalQueryCost;
+ return new StaticQueryCost(totalQueryCost);
}
@Override
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index b480d14..31ac358 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -347,7 +347,9 @@
public synchronized void start() {
super.start();
- sessionExpiryThread = Executors.newSingleThreadScheduledExecutor();
+ sessionExpiryThread = Executors.newSingleThreadScheduledExecutor(runnable
+ -> new Thread(runnable, "Session-expiry-thread"));
+
int sessionExpiryInterval = getSessionExpiryInterval();
sessionExpiryThread.scheduleWithFixedDelay(sessionExpiryRunnable, sessionExpiryInterval,
sessionExpiryInterval, TimeUnit.SECONDS);
@@ -420,12 +422,23 @@
@Override
public void writeExternal(ObjectOutput out) throws IOException {
// Write out all the sessions
- out.writeInt(SESSION_MAP.size());
+ List<LensSessionImpl> sessions = new ArrayList<>();
for (LensSessionHandle sessionHandle : SESSION_MAP.values()) {
- LensSessionImpl session = getSession(sessionHandle);
+ try {
+ sessions.add(getSession(sessionHandle));
+ } catch (ClientErrorException e) {
+ // warn for invalid/null session and continue.
+ log.warn("Cannot persist " + (sessionHandle != null ? sessionHandle.getPublicId() : "null ")
+ + " session. {}", e);
+ }
+ }
+
+ out.writeInt(sessions.size());
+ for (LensSessionImpl session : sessions) {
session.getLensSessionPersistInfo().writeExternal(out);
}
- log.info("Session service pesristed " + SESSION_MAP.size() + " sessions");
+
+ log.info("Session service pesristed " + sessions.size() + " sessions out of " + SESSION_MAP.size());
}
/**
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index ab24033..fdf4694 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -2527,7 +2527,7 @@
}
private XPartition createPartition(String cubeTableName, Date partDate) {
- return createPartition(cubeTableName, partDate, "dt");
+ return createPartition(cubeTableName, partDate, "dt", null);
}
private XTimePartSpecElement createTimePartSpecElement(Date partDate, String timeDimension) {
@@ -2539,11 +2539,17 @@
private XPartition createPartition(String cubeTableName, Date partDate, final String timeDimension) {
- return createPartition(cubeTableName, Lists.newArrayList(createTimePartSpecElement(partDate, timeDimension)));
+ return createPartition(cubeTableName, partDate, timeDimension, null);
}
- private XPartition createPartition(String cubeTableName, final List<XTimePartSpecElement> timePartSpecs) {
+ private XPartition createPartition(String cubeTableName, Date partDate, final String timeDimension,
+ String updatePeriod) {
+ return createPartition(cubeTableName, Lists.newArrayList(
+ createTimePartSpecElement(partDate, timeDimension)), updatePeriod);
+ }
+ private XPartition createPartition(String cubeTableName, final List<XTimePartSpecElement>
+ timePartSpecs, String updatePeriod) {
XPartition xp = cubeObjectFactory.createXPartition();
xp.setLocation(new Path(new File("target").getAbsolutePath(), "part/test_part").toString());
xp.setFactOrDimensionTableName(cubeTableName);
@@ -2554,10 +2560,19 @@
for (XTimePartSpecElement timePartSpec : timePartSpecs) {
xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpec);
}
- xp.setUpdatePeriod(XUpdatePeriod.HOURLY);
+ if (updatePeriod==null) {
+ xp.setUpdatePeriod(XUpdatePeriod.HOURLY);
+ } else {
+ XUpdatePeriod updatePeriod1=XUpdatePeriod.valueOf(updatePeriod.toUpperCase());
+ xp.setUpdatePeriod(updatePeriod1);
+ }
return xp;
}
+ private XPartition createPartition(String cubeTableName, final List<XTimePartSpecElement> timePartSpecs) {
+ return createPartition(cubeTableName, timePartSpecs, null);
+ }
+
@Test(dataProvider = "mediaTypeData")
public void testLatestDateWithInputTimeDimAbsentFromAtleastOneFactPartition(MediaType mediaType) throws Exception {
@@ -2704,6 +2719,7 @@
setCurrentDatabase(DB, mediaType);
createStorage("S1", mediaType);
createStorage("S2", mediaType);
+ createStorage("S3", mediaType);
try {
@@ -2717,6 +2733,7 @@
f.getStorageTables().getStorageTable().add(createStorageTblElement("S1", table, "HOURLY"));
f.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "DAILY"));
f.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "HOURLY"));
+ f.getStorageTables().getStorageTable().add(createStorageTblElement("S3", table, "HOURLY", "DAILY"));
APIResult result = target()
.path("metastore")
.path("facts").queryParam("sessionid", lensSessionId)
@@ -2903,6 +2920,47 @@
partitions = partitionsElement.getValue();
assertNotNull(partitions);
assertEquals(partitions.getPartition().size(), 0);
+
+ //adding 2 partition daily and hourly on S3 storage
+
+ xp = createPartition(table, partDate, "dt");
+ APIResult partAddResult1 = target().path("metastore/facts/").path(table).path("storages/S3/partition")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .post(Entity.entity(new GenericEntity<JAXBElement<XPartition>>(cubeObjectFactory.createXPartition(xp)) {
+ }, mediaType), APIResult.class);
+ assertSuccess(partAddResult1);
+
+ xp2 = createPartition(table, partDate, "dt", "daily");
+ APIResult partAddResult2 = target().path("metastore/facts/").path(table).path("storages/S3/partition")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .post(Entity.entity(new GenericEntity<JAXBElement<XPartition>>(cubeObjectFactory.createXPartition(xp2)) {
+ }, mediaType), APIResult.class);
+ assertSuccess(partAddResult2);
+
+ String filter="dt<='" + HOURLY.format(partDate) + "'";
+
+ //deleting partition based on filter and updatePeriod
+
+ dropResult = target().path("metastore/facts").path(table).path("storages/S3/partitions")
+ .queryParam("sessionid", lensSessionId).queryParam("filter", filter)
+ .queryParam("updatePeriod", "hourly").request(mediaType).delete(APIResult.class);
+
+ assertSuccess(dropResult);
+
+
+ // Verify only hourly partition was dropped
+ partitionsElement = target().path("metastore/facts").path(table).path("storages/S3/partitions")
+ .queryParam("sessionid", lensSessionId)
+ .queryParam("filter", "dt<='" + HOURLY.format(partDate) + "'")
+ .request(mediaType).get(new GenericType<JAXBElement<XPartitionList>>() {
+ });
+
+
+ partitions = partitionsElement.getValue();
+ assertNotNull(partitions);
+ assertEquals(partitionsElement.getValue().getPartition().get(0).getUpdatePeriod(), XUpdatePeriod.DAILY);
+
+
} finally {
setCurrentDatabase(prevDb, mediaType);
dropDatabase(DB, mediaType);
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
index 20243f4..a800e2d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
@@ -32,6 +32,7 @@
import org.apache.lens.server.api.query.comparators.QueryCostComparator;
import org.apache.lens.server.api.query.comparators.QueryPriorityComparator;
import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.cost.StaticQueryCost;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -111,13 +112,15 @@
QueryContext query1 = mock(QueryContext.class);
when(query1.getPriority()).thenReturn(Priority.HIGH);
- QueryCost qcO1 = mock(QueryCost.class);
- when(query1.getSelectedDriverQueryCost()).thenReturn(qcO1);
+
+ QueryCost s1 = new StaticQueryCost(0.0);
+ QueryCost s2 = new StaticQueryCost(0.0);
+
+ when(query1.getSelectedDriverQueryCost()).thenReturn(s1);
QueryContext query2 = mock(QueryContext.class);
when(query2.getPriority()).thenReturn(Priority.HIGH);
- QueryCost qcO2 = mock(QueryCost.class);
- when(query2.getSelectedDriverQueryCost()).thenReturn(qcO2);
+ when(query2.getSelectedDriverQueryCost()).thenReturn(s2);
when(query1.getSubmissionTime()).thenReturn(submitTimeQuery1);
when(query2.getSubmissionTime()).thenReturn(submitTimeQuery2);
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
index 068d46c..5d37f41 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
@@ -217,7 +217,7 @@
};
}
- @Test(dataProvider = "restartDataProvider")
+ @Test(dataProvider = "restartDataProvider", enabled = false)
public void testQueryAliveOnSessionClose(boolean restartBeforeFinish, boolean restartAfterFinish)
throws LensException, InterruptedException {
int numSessions = getSessionsOfFoo().size();
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollectionTest.java b/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollectionTest.java
index 4dde18a..06ac9fb 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollectionTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollectionTest.java
@@ -30,8 +30,8 @@
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.cost.StaticQueryCost;
import org.testng.annotations.Test;
@@ -50,7 +50,7 @@
EstimatedQueryCollection queries = new DefaultEstimatedQueryCollection(mockQueries);
QueryCost actualQueryCost = queries.getTotalQueryCost(MOCK_USER);
- assertEquals(actualQueryCost, new FactPartitionBasedQueryCost(0));
+ assertEquals(actualQueryCost, new StaticQueryCost(0));
}
@Test
@@ -63,19 +63,15 @@
final QueryContext query0 = Iterables.get(mockQueriesSet, 0);
final QueryContext query1 = Iterables.get(mockQueriesSet, 1);
- final QueryCost mockCost0 = mock(QueryCost.class);
- final QueryCost mockCost1 = mock(QueryCost.class);
- final QueryCost mockCost0Plus0 = mock(QueryCost.class);
- final QueryCost mockCost0Plus0Plus1 = mock(QueryCost.class);
+ final QueryCost sCost0 = new StaticQueryCost(1.0);
+ final QueryCost sCost1 = new StaticQueryCost(1.0);
+ final QueryCost expectedCost = new StaticQueryCost(2.0);
- when(query0.getSelectedDriverQueryCost()).thenReturn(mockCost0);
- when(query1.getSelectedDriverQueryCost()).thenReturn(mockCost1);
-
- when(mockCost0.add(mockCost0)).thenReturn(mockCost0Plus0);
- when(mockCost0Plus0.add(mockCost1)).thenReturn(mockCost0Plus0Plus1);
+ when(query0.getSelectedDriverQueryCost()).thenReturn(sCost0);
+ when(query1.getSelectedDriverQueryCost()).thenReturn(sCost1);
QueryCost actualQueryCost = new DefaultEstimatedQueryCollection(mockQueries).getTotalQueryCost(MOCK_USER);
- assertEquals(actualQueryCost, mockCost0Plus0Plus1);
+ assertEquals(actualQueryCost, expectedCost);
}
@Test
diff --git a/lens-ship-jars/pom.xml b/lens-ship-jars/pom.xml
index f6bc69e..67867a4 100644
--- a/lens-ship-jars/pom.xml
+++ b/lens-ship-jars/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-ship-jars</artifactId>
diff --git a/lens-storage-db/pom.xml b/lens-storage-db/pom.xml
index c9629a0..68c6e46 100644
--- a/lens-storage-db/pom.xml
+++ b/lens-storage-db/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-storage-db</artifactId>
diff --git a/lens-ui/pom.xml b/lens-ui/pom.xml
index 4b4552d..1185be9 100644
--- a/lens-ui/pom.xml
+++ b/lens-ui/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</parent>
<artifactId>lens-ui</artifactId>
diff --git a/pom.xml b/pom.xml
index 7010ea8..bd08b0d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -128,7 +128,7 @@
<groupId>org.apache.lens</groupId>
<artifactId>apache-lens</artifactId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
<name>Lens</name>
<packaging>pom</packaging>
<description>Unified Analytics Platform</description>
@@ -740,7 +740,7 @@
<dependency>
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</dependency>
</dependencies>
<executions>
@@ -1854,7 +1854,7 @@
<dependency>
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
- <version>2.7.2-SNAPSHOT</version>
+ <version>2.8.0-SNAPSHOT</version>
</dependency>
</dependencies>
<executions>
diff --git a/src/site/apt/admin/hivedriver-config.apt b/src/site/apt/admin/hivedriver-config.apt
index a9f020d..b1a8431 100644
--- a/src/site/apt/admin/hivedriver-config.apt
+++ b/src/site/apt/admin/hivedriver-config.apt
@@ -42,15 +42,19 @@
*--+--+---+--+
|9|lens.cube.query.replace.timedim|true|Tells whether timedim attribute queried in the time range should be replaced with its corresponding partition column name.|
*--+--+---+--+
-|10|lens.driver.hive.calculate.priority|true|Whether priority should be calculated for hive mr jobs or not|
+|10|lens.driver.cost.query.decider.class|org.apache.lens.server.api.query.cost.RangeBasedQueryCostTypeDecider|Decider class which looks at ranges passed in config and decides the querycosttype|
*--+--+---+--+
-|11|lens.driver.hive.connection.class|org.apache.lens.driver.hive.EmbeddedThriftConnection|The connection class from HiveDriver to HiveServer. The default is an embedded connection which does not require a remote hive server. For connecting to a hiveserver end point, remote connection should be used. The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection and org.apache.lens.driver.hive.RemoteThriftConnection.|
+|11|lens.driver.cost.type.ranges|LOW,0.0,HIGH|Cost based Query type mapping|
*--+--+---+--+
-|12|lens.driver.hive.cost.calculator.class|org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator|Cost calculator class. By default calculating cost through fact partitions.|
+|12|lens.driver.hive.calculate.priority|true|Whether priority should be calculated for hive mr jobs or not|
*--+--+---+--+
-|13|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2|
+|13|lens.driver.hive.connection.class|org.apache.lens.driver.hive.EmbeddedThriftConnection|The connection class from HiveDriver to HiveServer. The default is an embedded connection which does not require a remote hive server. For connecting to a hiveserver end point, remote connection should be used. The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection and org.apache.lens.driver.hive.RemoteThriftConnection.|
*--+--+---+--+
-|14|lens.driver.hive.priority.ranges|VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW|Priority Ranges. The numbers are the costs of the query. \ |
+|14|lens.driver.hive.cost.calculator.class|org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator|Cost calculator class. By default calculating cost through fact partitions.|
+*--+--+---+--+
+|15|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2|
+*--+--+---+--+
+|16|lens.driver.hive.priority.ranges|VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW|Priority Ranges. The numbers are the costs of the query. \ |
| | | |The cost is calculated based on partition weights and fact weights. The interpretation of the default config is: \ |
| | | | \ |
| | | |cost \<= 7\ \ \ \ \ \ \ \ \ \ \ :\ \ \ \ \ Priority = VERY_HIGH \ |
@@ -66,10 +70,10 @@
| | | |One use case in range tuning can be that you never want queries to run with VERY_HIGH, assuming no other changes, you'll modify the value of this param in hivedriver-site.xml to be HIGH,30.0,NORMAL,90,LOW\ |
| | | |via the configs, you can tune both the ranges and partition weights. this would give the end user more control. |
*--+--+---+--+
-|15|lens.driver.hive.query.hook.classes| |The query hook classes for hive driver. By default there are no hooks. To add a hook, you should look at the default implementation and from there it'll be easy to derive what value can be added through a new hook. Multiple hooks can be provided by providing comma seperated name of classes.|
+|17|lens.driver.hive.query.hook.classes| |The query hook classes for hive driver. By default there are no hooks. To add a hook, you should look at the default implementation and from there it'll be easy to derive what value can be added through a new hook. Multiple hooks can be provided by providing comma seperated name of classes.|
*--+--+---+--+
-|16|lens.driver.hive.query.launching.constraint.factories| |Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.|
+|18|lens.driver.hive.query.launching.constraint.factories| |Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.|
*--+--+---+--+
-|17|lens.driver.hive.waiting.queries.selection.policy.factories| |Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
+|19|lens.driver.hive.waiting.queries.selection.policy.factories| |Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
*--+--+---+--+
The configuration parameters and their default values
diff --git a/src/site/apt/admin/jdbcdriver-config.apt b/src/site/apt/admin/jdbcdriver-config.apt
index eb56927..c50872b 100644
--- a/src/site/apt/admin/jdbcdriver-config.apt
+++ b/src/site/apt/admin/jdbcdriver-config.apt
@@ -34,63 +34,71 @@
*--+--+---+--+
|5|lens.cube.query.time.range.writer.class|org.apache.lens.cube.parse.BetweenTimeRangeWriter|The timerange writer class which specifies how the resolved partitions in timeranges should be written in final query. Available writers are org.apache.lens.cube.parse.ORTimeRangeWriter and org.apache.lens.cube.parse.BetweenTimeRangeWriter|
*--+--+---+--+
-|6|lens.driver.jdbc.connection.properties| |Connection properties for jdbc connection.|
+|6|lens.driver.cost.query.decider.class|org.apache.lens.server.api.query.cost.RangeBasedQueryCostTypeDecider|Decider class which looks at ranges passed in config and decides the querycosttype|
*--+--+---+--+
-|7|lens.driver.jdbc.connection.provider| |A contract for obtaining JDBC connections|
+|7|lens.driver.cost.type.ranges|LOW,0.0,HIGH|Cost based Query type mapping|
*--+--+---+--+
-|8|lens.driver.jdbc.db.password| |The database user's password|
+|8|lens.driver.jdbc.connection.properties| |Connection properties for jdbc connection.|
*--+--+---+--+
-|9|lens.driver.jdbc.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname|
+|9|lens.driver.jdbc.connection.provider| |A contract for obtaining JDBC connections|
*--+--+---+--+
-|10|lens.driver.jdbc.db.user| |The database user on whose behalf the connection is being made|
+|10|lens.driver.jdbc.cost.calculator.class|org.apache.lens.cube.query.cost.StaticCostCalculator|Cost calculator class. By default calculating cost through static values|
*--+--+---+--+
-|11|lens.driver.jdbc.driver.class|com.mysql.jdbc.Driver|Type of JDBC driver used to connect backend database|
+|11|lens.driver.jdbc.db.password| |The database user's password|
*--+--+---+--+
-|12|lens.driver.jdbc.enable.resultset.streaming.retrieval|false|Flag to enable row by row retrieval of result set from the database server. This is used to enable streaming result sets for MySQL. This is set to false by default.|
+|12|lens.driver.jdbc.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname|
*--+--+---+--+
-|13|lens.driver.jdbc.estimate.connection.properties| |Connection properties for jdbc estimate connection.|
+|13|lens.driver.jdbc.db.user| |The database user on whose behalf the connection is being made|
*--+--+---+--+
-|14|lens.driver.jdbc.estimate.db.password| |The database user's password, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.password would be used. Override this property to tune estimate connection pool|
+|14|lens.driver.jdbc.driver.class|com.mysql.jdbc.Driver|Type of JDBC driver used to connect backend database|
*--+--+---+--+
-|15|lens.driver.jdbc.estimate.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.uri will be used.|
+|15|lens.driver.jdbc.enable.resultset.streaming.retrieval|false|Flag to enable row by row retrieval of result set from the database server. This is used to enable streaming result sets for MySQL. This is set to false by default.|
*--+--+---+--+
-|16|lens.driver.jdbc.estimate.db.user| |The database user on whose behalf the connection is being made, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.user would be used. Override this property to tune estimate connection pool|
+|16|lens.driver.jdbc.estimate.connection.properties| |Connection properties for jdbc estimate connection.|
*--+--+---+--+
-|17|lens.driver.jdbc.estimate.driver.class| |Type of JDBC driver used to connect backend database for estimate queries. If This property is not specified, value for lens.driver.jdbc.driver.class will be used. Override this property to tune estimate connection pool|
+|17|lens.driver.jdbc.estimate.db.password| |The database user's password, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.password would be used. Override this property to tune estimate connection pool|
*--+--+---+--+
-|18|lens.driver.jdbc.estimate.get.connection.timeout| |Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket , for estimate queries. If this property is not specified, value for lens.driver.jdbc.get.connection.timeout would be used. Override this property to tune estimate connection pool.|
+|18|lens.driver.jdbc.estimate.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.uri will be used.|
*--+--+---+--+
-|19|lens.driver.jdbc.estimate.pool.idle.time| |Maximum idle time in sec before a connection is closed, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.idle.time would be used. Override this property to tune estimate connection pool.|
+|19|lens.driver.jdbc.estimate.db.user| |The database user on whose behalf the connection is being made, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.user would be used. Override this property to tune estimate connection pool|
*--+--+---+--+
-|20|lens.driver.jdbc.estimate.pool.max.size| |Maximum number of concurrent connections allowed in pool, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.pool.max.size would be used. Override this property to tune estimate connection pool|
+|20|lens.driver.jdbc.estimate.driver.class| |Type of JDBC driver used to connect backend database for estimate queries. If This property is not specified, value for lens.driver.jdbc.driver.class will be used. Override this property to tune estimate connection pool|
*--+--+---+--+
-|21|lens.driver.jdbc.estimate.pool.max.statements| |Maximum number of prepared statements to cache per connection, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.max.statements would be used.|
+|21|lens.driver.jdbc.estimate.get.connection.timeout| |Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket , for estimate queries. If this property is not specified, value for lens.driver.jdbc.get.connection.timeout would be used. Override this property to tune estimate connection pool.|
*--+--+---+--+
-|22|lens.driver.jdbc.explain.keyword|Explain|Explain keyword used to get the query plan of underlying database|
+|22|lens.driver.jdbc.estimate.pool.idle.time| |Maximum idle time in sec before a connection is closed, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.idle.time would be used. Override this property to tune estimate connection pool.|
*--+--+---+--+
-|23|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set|
+|23|lens.driver.jdbc.estimate.pool.max.size| |Maximum number of concurrent connections allowed in pool, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.pool.max.size would be used. Override this property to tune estimate connection pool|
*--+--+---+--+
-|24|lens.driver.jdbc.get.connection.timeout|10000|The number of milliseconds a client calling getConnection() will wait for a Connection to be checked-in or acquired when the pool is exhausted. Zero means wait indefinitely. Setting any positive value will cause the getConnection () call to time-out and break with an SQLException after the specified number of milliseconds. The default value of this property is 10 secs.|
+|24|lens.driver.jdbc.estimate.pool.max.statements| |Maximum number of prepared statements to cache per connection, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.max.statements would be used.|
*--+--+---+--+
-|25|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed|
+|25|lens.driver.jdbc.explain.keyword|Explain|Explain keyword used to get the query plan of underlying database|
*--+--+---+--+
-|26|lens.driver.jdbc.pool.max.size|15|Maximum number of concurrent connections allowed in pool|
+|26|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set|
*--+--+---+--+
-|27|lens.driver.jdbc.pool.max.statements|20|Maximum number of prepared statements to cache per connection|
+|27|lens.driver.jdbc.get.connection.timeout|10000|The number of milliseconds a client calling getConnection() will wait for a Connection to be checked-in or acquired when the pool is exhausted. Zero means wait indefinitely. Setting any positive value will cause the getConnection () call to time-out and break with an SQLException after the specified number of milliseconds. The default value of this property is 10 secs.|
*--+--+---+--+
-|28|lens.driver.jdbc.query.launching.constraint.factories|org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory,
+|28|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed|
+*--+--+---+--+
+|29|lens.driver.jdbc.pool.max.size|15|Maximum number of concurrent connections allowed in pool|
+*--+--+---+--+
+|30|lens.driver.jdbc.pool.max.statements|20|Maximum number of prepared statements to cache per connection|
+*--+--+---+--+
+|31|lens.driver.jdbc.query.launching.constraint.factories|org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory,
org.apache.lens.driver.jdbc.MaxJDBCConnectionCheckConstraintFactory|Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.|
*--+--+---+--+
-|29|lens.driver.jdbc.query.rewriter|org.apache.lens.driver.jdbc.ColumnarSQLRewriter|Rewriting the HQL to optimized sql queries|
+|32|lens.driver.jdbc.query.rewriter|org.apache.lens.driver.jdbc.ColumnarSQLRewriter|Rewriting the HQL to optimized sql queries|
*--+--+---+--+
-|30|lens.driver.jdbc.regex.replacement.values|to_date=date, format_number=format, date_sub\((.*?)\,\s*([0-9]+\s*)\)=date_sub($1\, interval $2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day)|Rewriting the HQL to optimized sql queries|
+|33|lens.driver.jdbc.regex.replacement.values|to_date=date, format_number=format, date_sub\((.*?)\,\s*([0-9]+\s*)\)=date_sub($1\, interval $2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day)|Rewriting the HQL to optimized sql queries|
*--+--+---+--+
-|31|lens.driver.jdbc.statement.cancel.supported|true|Flag to indicate Whether cancel on JDBC statement is supported. If not supported, framework wont call cancel on JDBC statement.|
+|34|lens.driver.jdbc.statement.cancel.supported|true|Flag to indicate Whether cancel on JDBC statement is supported. If not supported, framework wont call cancel on JDBC statement.|
*--+--+---+--+
-|32|lens.driver.jdbc.validate.through.prepare|true|Flag to enable query syntactic and semantic validation using prepared statement.|
+|35|lens.driver.jdbc.validate.through.prepare|true|Flag to enable query syntactic and semantic validation using prepared statement.|
*--+--+---+--+
-|33|lens.driver.jdbc.waiting.queries.selection.policy.factories|org.apache.lens.server.api.query.collect.DriverSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
+|36|lens.driver.jdbc.waiting.queries.selection.policy.factories|org.apache.lens.server.api.query.collect.DriverSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
*--+--+---+--+
-|34|lens.query.timeout.millis|3600000|The runtime(millis) of the query after which query will be timedout and cancelled. Default is 1 hour for jdbc queries.|
+|37|lens.driver.query.cost|0.0|Jdbc driver static cost value|
+*--+--+---+--+
+|38|lens.query.timeout.millis|3600000|The runtime(millis) of the query after which query will be timedout and cancelled. Default is 1 hour for jdbc queries.|
*--+--+---+--+
The configuration parameters and their default values
diff --git a/tools/conf-pseudo-distr/server/drivers/hive/hive1/hivedriver-site.xml b/tools/conf-pseudo-distr/server/drivers/hive/hive1/hivedriver-site.xml
index c21020a..cb5fa8a 100644
--- a/tools/conf-pseudo-distr/server/drivers/hive/hive1/hivedriver-site.xml
+++ b/tools/conf-pseudo-distr/server/drivers/hive/hive1/hivedriver-site.xml
@@ -52,6 +52,6 @@
</property>
<property>
<name>hive.aux.jars.path</name>
- <value>file:///opt/lens/lens-ml-dist/target/apache-lens-2.7.0-SNAPSHOT-ml/lib/lens-ml-lib-2.7.0-SNAPSHOT.jar,file:///usr/local/spark-1.3.0-bin-hadoop2.4/lib/spark-assembly-1.3.0-hadoop2.4.0.jar</value>
+ <value>file:///opt/lens/lens-ml-dist/target/apache-lens-2.8.0-SNAPSHOT-ml/lib/lens-ml-lib-2.8.0-SNAPSHOT.jar,file:///usr/local/spark-1.3.0-bin-hadoop2.4/lib/spark-assembly-1.3.0-hadoop2.4.0.jar</value>
</property>
</configuration>
diff --git a/tools/scripts/generate-site-public.sh b/tools/scripts/generate-site-public.sh
index 574caf5..0fe6d63 100755
--- a/tools/scripts/generate-site-public.sh
+++ b/tools/scripts/generate-site-public.sh
@@ -92,4 +92,4 @@
cp -r $SITE_BACKUP/site/publish/versions $SVN_TARGET/site/publish/
cp -r $TMP/ $SVN_TARGET/site/publish
cd $SVN_TARGET
-echo "Generated site."
+echo "Generated site."
\ No newline at end of file
