TAJO-1991: Tablespace::getVolume should take filter predication.
Closes #901
diff --git a/CHANGES b/CHANGES
index ec607d4..008b987 100644
--- a/CHANGES
+++ b/CHANGES
@@ -7,6 +7,9 @@
IMPROVEMENT
+ TAJO-1991: Tablespace::getVolume should take filter predication.
+ (hyunsik)
+
TAJO-2007: By default, Optimizer should use the table volume in TableStat.
(hyunsik)
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
index bb8494c..c179ed9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
@@ -30,11 +30,8 @@
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
-import org.apache.tajo.plan.expr.*;
import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.plan.expr.AlgebraicUtil;
-import org.apache.tajo.plan.expr.EvalNode;
-import org.apache.tajo.plan.expr.EvalTreeUtil;
+import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.joinorder.*;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteEngine;
@@ -42,7 +39,6 @@
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleProvider;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
-import org.apache.tajo.storage.StorageService;
import org.apache.tajo.util.ReflectionUtil;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.util.graph.DirectedGraphCursor;
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 2bdec36..4b0caa0 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -48,7 +48,6 @@
import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule;
import org.apache.tajo.plan.util.ExprFinder;
import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.storage.StorageService;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.StringUtils;
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
similarity index 86%
rename from tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
rename to tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
index 0c3c031..f27aeff 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
@@ -16,9 +16,11 @@
* limitations under the License.
*/
-package org.apache.tajo.storage;
+package org.apache.tajo.plan;
+import com.google.common.base.Optional;
import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.expr.EvalNode;
import javax.annotation.Nullable;
import java.net.URI;
@@ -38,5 +40,5 @@
*/
URI getTableURI(@Nullable String spaceName, String databaseName, String tableName);
- long getTableVolumn(URI uri) throws UnsupportedException;
+ long getTableVolumn(URI uri, Optional<EvalNode> filter) throws UnsupportedException;
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
index 565b14b..0362001 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
@@ -21,7 +21,7 @@
import org.apache.tajo.OverridableConf;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.storage.StorageService;
+import org.apache.tajo.plan.StorageService;
public class LogicalPlanRewriteRuleContext {
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
index 9c6f285..a1e9a6d 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
@@ -18,6 +18,7 @@
package org.apache.tajo.plan.rewrite;
+import com.google.common.base.Optional;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.OverridableConf;
@@ -27,10 +28,11 @@
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.StorageService;
+import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
-import org.apache.tajo.storage.StorageService;
import java.util.Stack;
@@ -75,15 +77,15 @@
final TableDesc table = scanNode.getTableDesc();
if (!isVirtual(table)) {
-
final TableStats stats = getTableStat(table);
final long tableSize = stats.getNumBytes();
+ final Optional<EvalNode> filter = scanNode.hasQual() ? Optional.of(scanNode.getQual()) : Optional.<EvalNode>absent();
// If USE_TABLE_VOLUME is set, we will update the table volume through a storage handler.
// In addition, if the table size is zero, we will update too.
// It is a good workaround to avoid suboptimal join orders without cheap cost.
if (conf.getBool(SessionVars.USE_TABLE_VOLUME) || tableSize == 0) {
- table.getStats().setNumBytes(getTableVolume(table));
+ table.getStats().setNumBytes(getTableVolume(table, filter));
}
}
@@ -105,10 +107,10 @@
return table.getMeta().getDataFormat().equals("SYSTEM");
}
- private long getTableVolume(TableDesc table) {
+ private long getTableVolume(TableDesc table, Optional<EvalNode> filter) {
try {
if (table.getStats() != null) {
- return storage.getTableVolumn(table.getUri());
+ return storage.getTableVolumn(table.getUri(), filter);
}
} catch (UnsupportedException t) {
LOG.warn(table.getName() + " does not support Tablespace::getTableVolume()");
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
index 81fbbc0..01ef9de 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage;
+import com.google.common.base.Optional;
import net.minidev.json.JSONObject;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.ExecutionBlockId;
@@ -98,7 +99,7 @@
return name + "=" + uri.toString();
}
- public abstract long getTableVolume(URI uri) throws UnsupportedException;
+ public abstract long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException;
/**
* if {@link StorageProperty#isArbitraryPathAllowed} is true,
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
index 668b6c6..33cd7a3 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
@@ -37,6 +37,8 @@
import org.apache.tajo.exception.UndefinedTablespaceException;
import org.apache.tajo.exception.UndefinedTablespaceHandlerException;
import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.StorageService;
+import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.Pair;
@@ -437,8 +439,9 @@
}
@Override
- public long getTableVolumn(URI tableUri) throws UnsupportedException {
- return get(tableUri).getTableVolume(tableUri);
+ public long getTableVolumn(URI tableUri, Optional<EvalNode> filter)
+ throws UnsupportedException {
+ return get(tableUri).getTableVolume(tableUri, filter);
}
public static Iterable<Tablespace> getAllTablespaces() {
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
index acdb6ec..8aa51e6 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
@@ -18,7 +18,7 @@
package org.apache.tajo.storage.hbase;
-import com.google.common.base.Preconditions;
+import com.google.common.base.Optional;
import com.google.common.collect.Sets;
import net.minidev.json.JSONObject;
import org.apache.commons.logging.Log;
@@ -51,7 +51,6 @@
import org.apache.tajo.plan.logical.CreateTableNode;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.NodeType;
-import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.plan.verifier.SyntaxErrorUtil;
import org.apache.tajo.storage.*;
@@ -100,7 +99,7 @@
}
@Override
- public long getTableVolume(URI uri) throws UnsupportedException {
+ public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException {
throw new UnsupportedException();
}
@@ -176,7 +175,7 @@
throw new UnavailableTableLocationException(hbaseTableName, "the table does not exist");
}
HTableDescriptor hTableDescriptor = hAdmin.getTableDescriptor(hTableName);
- Set<String> tableColumnFamilies = new HashSet<String>();
+ Set<String> tableColumnFamilies = new HashSet<>();
for (HColumnDescriptor eachColumn : hTableDescriptor.getColumnFamilies()) {
tableColumnFamilies.add(eachColumn.getNameAsString());
}
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
index 9b949d6..dc4502c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
@@ -127,7 +127,7 @@
}
@Override
- public long getTableVolume(URI uri) throws UnsupportedException {
+ public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException {
Path path = new Path(uri);
ContentSummary summary;
try {
diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml
index 0b76f4d..ead7f36 100644
--- a/tajo-storage/tajo-storage-jdbc/pom.xml
+++ b/tajo-storage/tajo-storage-jdbc/pom.xml
@@ -196,7 +196,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
-
</dependencies>
<profiles>
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
index 7489307..da0b5a7 100644
--- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage.jdbc;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import net.minidev.json.JSONObject;
@@ -109,7 +110,7 @@
}
@Override
- public long getTableVolume(URI uri) throws UnsupportedException {
+ public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException {
throw new UnsupportedException();
}
diff --git a/tajo-storage/tajo-storage-pgsql/pom.xml b/tajo-storage/tajo-storage-pgsql/pom.xml
index d67abce..012d5cb 100644
--- a/tajo-storage/tajo-storage-pgsql/pom.xml
+++ b/tajo-storage/tajo-storage-pgsql/pom.xml
@@ -215,7 +215,7 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>15.0</version>
- <scope>test</scope>
+ <!--<scope>test</scope>-->
</dependency>
</dependencies>
