TAJO-1926: Disable partition pruning using catalog temporarily.
Closes #824
diff --git a/CHANGES b/CHANGES
index 04d4ab5..8863224 100644
--- a/CHANGES
+++ b/CHANGES
@@ -285,9 +285,10 @@
BUG FIXES
+ TAJO-1926: Disable partition pruning using catalog temporarily. (jaehwa)
+
TAJO-1924: Repair partition need to calculate partition volume. (jaehwa)
-
TAJO-1923: Selecting on information_schema.table_stats throws an internal
error. (jihoon)
@@ -722,8 +723,6 @@
TAJO-1887: Disable the alter table add partition statement temporarily.
(jaehwa)
- TAJO-1493: Make partition pruning based on catalog informations. (jaehwa)
-
TAJO-1673: Implement recover partitions. (jaehwa)
TAJO-1465: Add ORCFileAppender to write into ORCFile table.
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index ba4f37e..7af110c 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@ -407,29 +407,6 @@
}
@Override
- public boolean existPartitions(String databaseName, String tableName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException {
-
- try {
- final BlockingInterface stub = getStub();
- final TableIdentifierProto request = buildTableIdentifier(databaseName, tableName);
- final ReturnState state = stub.existsPartitions(null, request);
-
- if (isThisError(state, UNDEFINED_PARTITIONS)) {
- return false;
- }
- throwsIfThisError(state, UndefinedDatabaseException.class);
- throwsIfThisError(state, UndefinedTableException.class);
- throwsIfThisError(state, UndefinedPartitionMethodException.class);
- ensureOk(state);
- return true;
-
- } catch (ServiceException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
public final PartitionDescProto getPartition(final String databaseName, final String tableName,
final String partitionName)
throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException,
@@ -457,16 +434,15 @@
}
@Override
- public final List<PartitionDescProto> getPartitionsOfTable(final String databaseName, final String tableName) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException {
+ public final List<PartitionDescProto> getPartitions(final String databaseName, final String tableName) {
try {
final BlockingInterface stub = getStub();
- final TableIdentifierProto request = buildTableIdentifier(databaseName, tableName);
+ final PartitionIdentifierProto request = PartitionIdentifierProto.newBuilder()
+ .setDatabaseName(databaseName)
+ .setTableName(tableName)
+ .build();
final GetPartitionsResponse response = stub.getPartitionsByTableName(null, request);
- throwsIfThisError(response.getState(), UndefinedDatabaseException.class);
- throwsIfThisError(response.getState(), UndefinedTableException.class);
- throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class);
ensureOk(response.getState());
return response.getPartitionList();
@@ -476,26 +452,6 @@
}
@Override
- public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UnsupportedException {
- try {
- final BlockingInterface stub = getStub();
- GetPartitionsResponse response = stub.getPartitionsByAlgebra(null, request);
-
- throwsIfThisError(response.getState(), UndefinedDatabaseException.class);
- throwsIfThisError(response.getState(), UndefinedTableException.class);
- throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class);
- throwsIfThisError(response.getState(), UnsupportedException.class);
- ensureOk(response.getState());
- return response.getPartitionList();
- } catch (ServiceException e) {
- LOG.error(e.getMessage(), e);
- return null;
- }
- }
-
- @Override
public List<TablePartitionProto> getAllPartitions() {
try {
final BlockingInterface stub = getStub();
diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
index a3a904b..8cc8e2f 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
+++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
@@ -120,12 +120,10 @@
rpc getPartitionMethodByTableName(TableIdentifierProto) returns (GetPartitionMethodResponse);
rpc existPartitionMethod(TableIdentifierProto) returns (ReturnState);
- rpc existsPartitions(TableIdentifierProto) returns (ReturnState);
rpc getPartitionByPartitionName(PartitionIdentifierProto) returns (GetPartitionDescResponse);
- rpc getPartitionsByTableName(TableIdentifierProto) returns (GetPartitionsResponse);
+ rpc getPartitionsByTableName(PartitionIdentifierProto) returns (GetPartitionsResponse);
rpc getAllPartitions(NullProto) returns (GetTablePartitionsResponse);
rpc addPartitions(AddPartitionsProto) returns (ReturnState);
- rpc getPartitionsByAlgebra(PartitionsByAlgebraProto) returns (GetPartitionsResponse);
rpc createIndex(IndexDescProto) returns (ReturnState);
rpc dropIndex(IndexNameProto) returns (ReturnState);
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
index f2acf98..721bcf1 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
@@ -59,7 +59,6 @@
public static final String COL_PARTITIONS_PK = "PARTITION_ID";
public static final String COL_COLUMN_NAME = "COLUMN_NAME";
public static final String COL_PARTITION_VALUE = "PARTITION_VALUE";
- public static final String COL_PARTITION_BYTES = "NUM_BYTES";
-
+
public static final String INFORMATION_SCHEMA_DB_NAME = "information_schema";
}
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
index 54d22e7..9173dc1 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
@@ -168,19 +168,11 @@
boolean existPartitionMethod(String databaseName, String tableName) throws UndefinedTableException,
UndefinedDatabaseException;
- boolean existPartitions(String databaseName, String tableName) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException;
-
PartitionDescProto getPartition(String databaseName, String tableName, String partitionName)
throws UndefinedPartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException,
UndefinedTableException;
- List<PartitionDescProto> getPartitionsOfTable(String databaseName, String tableName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException;
-
- List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UnsupportedException;
+ List<PartitionDescProto> getPartitions(String databaseName, String tableName);
List<TablePartitionProto> getAllPartitions();
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 12fb23d..d4ff69d 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -811,33 +811,16 @@
/**
* Converts passed parameters to a AlterTableDesc. This method would be called when adding a partition or dropping
* a table. This creates AlterTableDesc that is a wrapper class for protocol buffer.
- * *
- * @param tableName
- * @param columns
- * @param values
- * @param path
- * @param alterTableType
- * @return
- */
- public static AlterTableDesc addOrDropPartition(String tableName, String[] columns, String[] values, @Nullable
- String path, AlterTableType alterTableType) {
- return addOrDropPartition(tableName, columns, values, path, alterTableType, 0L);
- }
- /**
- * Converts passed parameters to a AlterTableDesc. This method would be called when adding a partition or dropping
- * a table. This creates AlterTableDesc that is a wrapper class for protocol buffer.
*
* @param tableName table name
* @param columns partition column names
* @param values partition values
- * @param path partition directory path
+ * @param location partition location
* @param alterTableType ADD_PARTITION or DROP_PARTITION
- * @param numBytes contents length
* @return AlterTableDesc
*/
- public static AlterTableDesc addOrDropPartition(String tableName, String[] columns, String[] values,
- @Nullable String path, AlterTableType alterTableType, long numBytes) {
-
+ public static AlterTableDesc addOrDropPartition(String tableName, String[] columns,
+ String[] values, String location, AlterTableType alterTableType) {
final AlterTableDesc alterTableDesc = new AlterTableDesc();
alterTableDesc.setTableName(tableName);
@@ -847,11 +830,8 @@
partitionDesc.setPartitionKeys(pair.getFirst());
partitionDesc.setPartitionName(pair.getSecond());
- if (alterTableType.equals(AlterTableType.ADD_PARTITION)) {
- if (path != null) {
- partitionDesc.setPath(path);
- }
- partitionDesc.setNumBytes(numBytes);
+ if (alterTableType.equals(AlterTableType.ADD_PARTITION) && location != null) {
+ partitionDesc.setPath(location);
}
alterTableDesc.setPartitionDesc(partitionDesc);
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index 7159aae..4b56e8c 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -18,11 +18,14 @@
package org.apache.tajo.catalog;
+import com.google.common.base.Function;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.StringUtils;
-import java.io.File;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -115,19 +118,31 @@
sb.append(" USING " + CatalogUtil.getBackwardCompitableDataFormat(meta.getDataFormat()));
}
- private static void buildWithClause(StringBuilder sb, TableMeta meta) {
+ private static void buildWithClause(final StringBuilder sb, TableMeta meta) {
KeyValueSet options = meta.getOptions();
if (options != null && options.size() > 0) {
- boolean first = true;
+
sb.append(" WITH (");
- for (Map.Entry<String, String> entry : meta.getOptions().getAllKeyValus().entrySet()) {
- if (first) {
- first = false;
- } else {
- sb.append(", ");
+
+ // sort table properties in an lexicographic order of the property keys.
+ Map.Entry<String, String>[] entries = meta.getOptions().getAllKeyValus().entrySet().toArray(
+ new Map.Entry[meta.getOptions().size()]);
+
+ Arrays.sort(entries, new Comparator<Map.Entry<String, String>>() {
+ @Override
+ public int compare(Map.Entry<String, String> o1, Map.Entry<String, String> o2) {
+ return o1.getKey().compareTo(o2.getKey());
}
- sb.append("'").append(entry.getKey()).append("'='").append(entry.getValue()).append("'");
- }
+ });
+
+ // Join all properties by comma (',')
+ sb.append(StringUtils.join(entries, ", ", new Function<Map.Entry<String, String>, String>() {
+ @Override
+ public String apply(Map.Entry<String, String> e) {
+ return "'" + e.getKey() + "'='" + e.getValue() + "'";
+ }
+ }));
+
sb.append(")");
}
}
@@ -167,7 +182,8 @@
List<Column> colums = table.getPartitionMethod().getExpressionSchema().getAllColumns();
- String[] splitPartitionName = partition.getPartitionName().split(File.separator);
+ String[] splitPartitionName = partition.getPartitionName().split("/");
+
for(int i = 0; i < splitPartitionName.length; i++) {
String[] partitionColumnValue = splitPartitionName[i].split("=");
if (i > 0) {
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
index e41ac853..7287fce 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
@@ -27,7 +27,6 @@
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.json.GsonObject;
import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto;
-import org.apache.tajo.util.TUtil;
import java.util.List;
@@ -57,7 +56,6 @@
@Expose protected String partitionName;
@Expose protected List<PartitionKeyProto> partitionKeys;
@Expose protected String path; //optional
- @Expose private Long numBytes = null; // optional
private CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder();
@@ -85,30 +83,27 @@
this.partitionKeys = partitionKeys;
}
- public Long getNumBytes() {
- return numBytes;
- }
-
- public void setNumBytes(Long numBytes) {
- this.numBytes = numBytes;
- }
-
public int hashCode() {
- return Objects.hashCode(partitionName, partitionKeys, path, numBytes);
+ return Objects.hashCode(partitionName, partitionKeys, path);
}
public boolean equals(Object o) {
if (o instanceof PartitionDesc) {
PartitionDesc another = (PartitionDesc) o;
- boolean eq = this.partitionName.equals(another.partitionName);
- eq = eq && this.partitionKeys.equals(another.partitionKeys);
- eq = eq && this.path.equals(another.path);
- eq = eq && TUtil.checkEquals(this.numBytes, another.numBytes);
+ boolean eq = ((partitionName != null && another.partitionName != null
+ && partitionName.equals(another.partitionName)) ||
+ (partitionName == null && another.partitionName == null));
+ eq = eq && ((partitionKeys != null && another.partitionKeys != null
+ && partitionKeys.equals(another.partitionKeys))
+ || (partitionKeys == null && another.partitionKeys == null));
+ eq = eq && ((path != null && another.path != null && path.equals(another.path)) ||
+ (path == null && another.path == null));
return eq;
}
return false;
}
+
@Override
public CatalogProtos.PartitionDescProto getProto() {
if (builder == null) {
@@ -130,10 +125,6 @@
builder.setPath(this.path);
}
- if(this.numBytes != null) {
- builder.setNumBytes(this.numBytes);
- }
-
return builder.build();
}
@@ -158,7 +149,6 @@
desc.partitionName = partitionName;
desc.partitionKeys = partitionKeys;
desc.path = path;
- desc.numBytes = numBytes;
return desc;
}
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 9a7b66a..e89c672 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -248,12 +248,12 @@
repeated PartitionKeyProto partitionKeys = 2;
optional string path = 3;
optional int32 id = 4;
- optional int64 numBytes = 5;
}
message PartitionKeyProto {
required string columnName = 1;
- required string partitionValue = 2;
+ optional string parentColumnName = 2;
+ required string partitionValue = 3;
}
message PartitionIdentifierProto {
@@ -262,18 +262,6 @@
optional string partitionName = 3;
}
-message PartitionsByAlgebraProto {
- required string databaseName = 1;
- required string tableName = 2;
- required string algebra = 3; // json object which contains algebra expressions
-}
-
-message PartitionsByFilterProto {
- required string databaseName = 1;
- required string tableName = 2;
- required string filter = 3; // filter string: (col1 ='1' or col1 = '100') and col3 > 20
-}
-
message TablespaceProto {
required string spaceName = 1;
required string uri = 2;
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index e377502..139e650 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -25,7 +25,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -35,9 +34,6 @@
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.TajoConstants;
-import org.apache.tajo.algebra.Expr;
-import org.apache.tajo.algebra.IsNullPredicate;
-import org.apache.tajo.algebra.JsonHelper;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -46,14 +42,11 @@
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.*;
-import org.apache.tajo.plan.expr.AlgebraicUtil;
-import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
import org.apache.thrift.TException;
-import java.io.File;
import java.io.IOException;
import java.util.*;
@@ -700,7 +693,7 @@
Table table = client.getHiveClient().getTable(databaseName, tableName);
List<FieldSchema> columns = table.getSd().getCols();
columns.add(new FieldSchema(columnProto.getName(),
- HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), ""));
+ HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), ""));
client.getHiveClient().alter_table(databaseName, tableName, table);
@@ -725,10 +718,6 @@
partition.setDbName(databaseName);
partition.setTableName(tableName);
- Map<String, String> params = TUtil.newHashMap();
- params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(partitionDescProto.getNumBytes()));
- partition.setParameters(params);
-
List<String> values = Lists.newArrayList();
for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) {
values.add(keyProto.getPartitionValue());
@@ -856,204 +845,11 @@
}
@Override
- public boolean existPartitions(String databaseName, String tableName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException {
-
- HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
- boolean result = false;
-
- try {
- client = clientPool.getClient();
- List<Partition> partitions = client.getHiveClient().listPartitionsByFilter(databaseName, tableName,
- "", (short) -1);
-
- if (partitions.size() > 0) {
- result = true;
- }
- } catch (Exception e) {
- throw new TajoInternalError(e);
- } finally {
- if (client != null) {
- client.release();
- }
- }
-
- return result;
+ public List<CatalogProtos.PartitionDescProto> getPartitions(String databaseName,
+ String tableName) {
+ throw new UnsupportedOperationException();
}
- @Override
- public List<CatalogProtos.PartitionDescProto> getPartitionsOfTable(String databaseName, String tableName)
- throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException {
- List<PartitionDescProto> list = null;
-
- try {
- if (!existDatabase(databaseName)) {
- throw new UndefinedDatabaseException(tableName);
- }
-
- if (!existTable(databaseName, tableName)) {
- throw new UndefinedTableException(tableName);
- }
-
- if (!existPartitionMethod(databaseName, tableName)) {
- throw new UndefinedPartitionMethodException(tableName);
- }
-
- list = getPartitionsFromHiveMetaStore(databaseName, tableName, "");
- } catch (Exception se) {
- throw new TajoInternalError(se);
- }
-
- return list;
- }
-
- @Override
- public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, UnsupportedException {
-
- List<PartitionDescProto> list = null;
-
- try {
- String databaseName = request.getDatabaseName();
- String tableName = request.getTableName();
-
- if (!existDatabase(databaseName)) {
- throw new UndefinedDatabaseException(tableName);
- }
-
- if (!existTable(databaseName, tableName)) {
- throw new UndefinedTableException(tableName);
- }
-
- if (!existPartitionMethod(databaseName, tableName)) {
- throw new UndefinedPartitionMethodException(tableName);
- }
-
- TableDescProto tableDesc = getTable(databaseName, tableName);
- String filter = getFilter(databaseName, tableName, tableDesc.getPartition().getExpressionSchema().getFieldsList()
- , request.getAlgebra());
- list = getPartitionsFromHiveMetaStore(databaseName, tableName, filter);
- } catch (UnsupportedException ue) {
- throw ue;
- } catch (Exception se) {
- throw new TajoInternalError(se);
- }
-
- return list;
- }
-
- private String getFilter(String databaseName, String tableName, List<ColumnProto> partitionColumns
- , String json) throws TajoException {
-
- Expr[] exprs = null;
-
- if (json != null && !json.isEmpty()) {
- Expr algebra = JsonHelper.fromJson(json, Expr.class);
- exprs = AlgebraicUtil.toConjunctiveNormalFormArray(algebra);
- }
-
- PartitionFilterAlgebraVisitor visitor = new PartitionFilterAlgebraVisitor();
- visitor.setIsHiveCatalog(true);
-
- Expr[] filters = AlgebraicUtil.getRearrangedCNFExpressions(databaseName + "." + tableName, partitionColumns, exprs);
-
- StringBuffer sb = new StringBuffer();
-
- // Write join clause from second column to last column.
- Column target;
-
- String result;
- for (int i = 0; i < partitionColumns.size(); i++) {
- target = new Column(partitionColumns.get(i));
-
- if (!(filters[i] instanceof IsNullPredicate)) {
- visitor.setColumn(target);
- visitor.visit(null, new Stack<Expr>(), filters[i]);
- result = visitor.getResult();
-
- // If visitor build filter successfully, add filter to be used for executing hive api.
- if (result.length() > 0) {
- if (sb.length() > 0) {
- sb.append(" AND ");
- }
- sb.append(" ( ").append(result).append(" ) ");
- } else {
- throw new TajoInternalError("Filter does not exist : " + filters[i].toJson());
- }
- }
- }
-
- return sb.toString();
- }
-
- /**
- * Get list of partitions matching specified filter.
- *
- * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
- * Assume that an user want to give a condition WHERE (col1 ='1' or col1 = '100') and col3 > 20 .
- *
- * Then, the filter string would be written as following:
- * (col1 =\"1\" or col1 = \"100\") and col3 > 20
- *
- *
- * @param databaseName
- * @param tableName
- * @param filter
- * @return
- */
- private List<PartitionDescProto> getPartitionsFromHiveMetaStore(String databaseName, String tableName,
- String filter) {
- HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
- List<PartitionDescProto> partitions = null;
- TableDescProto tableDesc = null;
- List<ColumnProto> parititonColumns = null;
-
- try {
- partitions = TUtil.newList();
- client = clientPool.getClient();
-
- List<Partition> hivePartitions = client.getHiveClient().listPartitionsByFilter(databaseName, tableName
- , filter, (short) -1);
-
- tableDesc = getTable(databaseName, tableName);
- parititonColumns = tableDesc.getPartition().getExpressionSchema().getFieldsList();
-
- StringBuilder partitionName = new StringBuilder();
- for (Partition hivePartition : hivePartitions) {
- CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder();
- builder.setPath(hivePartition.getSd().getLocation());
-
- partitionName.delete(0, partitionName.length());
- for (int i = 0; i < parititonColumns.size(); i++) {
- if (i > 0) {
- partitionName.append(File.separator);
- }
- partitionName.append(CatalogUtil.extractSimpleName(parititonColumns.get(i).getName()));
- partitionName.append("=");
- partitionName.append(hivePartition.getValues().get(i));
- }
-
- builder.setPartitionName(partitionName.toString());
-
- Map<String, String> params = hivePartition.getParameters();
- if (params != null) {
- if (params.get(StatsSetupConst.TOTAL_SIZE) != null) {
- builder.setNumBytes(Long.parseLong(params.get(StatsSetupConst.TOTAL_SIZE)));
- }
- }
-
- partitions.add(builder.build());
- }
- } catch (Exception e) {
- throw new TajoInternalError(e);
- } finally {
- if (client != null) {
- client.release();
- }
- }
-
- return partitions;
- }
@Override
public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName,
@@ -1079,14 +875,6 @@
keyBuilder.setColumnName(columnName);
keyBuilder.setPartitionValue(value);
builder.addPartitionKeys(keyBuilder);
-
- Map<String, String> params = partition.getParameters();
- if (params != null) {
- if (params.get(StatsSetupConst.TOTAL_SIZE) != null) {
- builder.setNumBytes(Long.parseLong(params.get(StatsSetupConst.TOTAL_SIZE)));
- }
- }
-
}
} catch (NoSuchObjectException e) {
return null;
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
index c843b21..af1b0b1 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
@@ -34,7 +34,6 @@
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
-import org.apache.tajo.util.TUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -262,27 +261,10 @@
}
testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150101");
- testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150102");
- testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150101");
testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150102");
- testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150101");
- testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150102");
-
- List<String> partitionNames = TUtil.newList();
- partitionNames.add("n_nationkey=40/n_date=20150801");
- partitionNames.add("n_nationkey=40/n_date=20150802");
- partitionNames.add("n_nationkey=50/n_date=20150801");
- partitionNames.add("n_nationkey=50/n_date=20150802");
- testAddPartitions(table1.getUri(), NATION, partitionNames);
-
- testGetPartitionsByAlgebra(DB_NAME, NATION);
testDropPartition(NATION, "n_nationkey=10/n_date=20150101");
- testDropPartition(NATION, "n_nationkey=10/n_date=20150102");
- testDropPartition(NATION, "n_nationkey=20/n_date=20150101");
testDropPartition(NATION, "n_nationkey=20/n_date=20150102");
- testDropPartition(NATION, "n_nationkey=30/n_date=20150101");
- testDropPartition(NATION, "n_nationkey=30/n_date=20150102");
CatalogProtos.PartitionDescProto partition = store.getPartition(DB_NAME, NATION, "n_nationkey=10/n_date=20150101");
assertNull(partition);
@@ -293,87 +275,6 @@
store.dropTable(DB_NAME, NATION);
}
- private void testGetPartitionsByAlgebra(String databaseName, String tableName) throws Exception {
- String qfTableName = databaseName + "." + tableName;
-
- // Equals Operator
- CatalogProtos.PartitionsByAlgebraProto.Builder request = CatalogProtos.PartitionsByAlgebraProto.newBuilder();
- request.setDatabaseName(databaseName);
- request.setTableName(tableName);
-
- String algebra = "{\n" +
- " \"LeftExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"n_nationkey\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"10\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"n_date\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"20150101\",\n" +
- " \"ValueType\": \"String\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"OpType\": \"And\"\n" +
- "}";
-
- request.setAlgebra(algebra);
-
- List<CatalogProtos.PartitionDescProto> partitions = store.getPartitionsByAlgebra(request.build());
- assertNotNull(partitions);
- assertEquals(1, partitions.size());
-
- // OR
- algebra = "{\n" +
- " \"LeftExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"n_nationkey\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"20\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"n_nationkey\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"30\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"OpType\": \"Or\"\n" +
- "}";
-
- request.setAlgebra(algebra);
-
- partitions = store.getPartitionsByAlgebra(request.build());
- assertNotNull(partitions);
- assertEquals(4, partitions.size());
- }
private void testAddPartition(URI uri, String tableName, String partitionName) throws Exception {
AlterTableDesc alterTableDesc = new AlterTableDesc();
@@ -415,45 +316,6 @@
}
}
- private void testAddPartitions(URI uri, String tableName, List<String> partitionNames) throws Exception {
- List<CatalogProtos.PartitionDescProto> partitions = TUtil.newList();
- for (String partitionName : partitionNames) {
- CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder();
- builder.setPartitionName(partitionName);
- Path path = new Path(uri.getPath(), partitionName);
- builder.setPath(path.toString());
-
- List<PartitionKeyProto> partitionKeyList = new ArrayList<PartitionKeyProto>();
- String[] split = partitionName.split("/");
- for(int i = 0; i < split.length; i++) {
- String[] eachPartitionName = split[i].split("=");
-
- PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder();
- keyBuilder.setColumnName(eachPartitionName[0]);
- keyBuilder.setPartitionValue(eachPartitionName[1]);
- partitionKeyList.add(keyBuilder.build());
- }
- builder.addAllPartitionKeys(partitionKeyList);
- partitions.add(builder.build());
- }
-
- store.addPartitions(DB_NAME, tableName, partitions, true);
-
- for (String partitionName : partitionNames) {
- CatalogProtos.PartitionDescProto resultDesc = store.getPartition(DB_NAME, NATION, partitionName);
- assertNotNull(resultDesc);
- assertEquals(resultDesc.getPartitionName(), partitionName);
- assertEquals(resultDesc.getPath(), uri.toString() + "/" + partitionName);
- assertEquals(resultDesc.getPartitionKeysCount(), 2);
-
- String[] split = partitionName.split("/");
- for (int i = 0; i < resultDesc.getPartitionKeysCount(); i++) {
- CatalogProtos.PartitionKeyProto keyProto = resultDesc.getPartitionKeys(i);
- String[] eachName = split[i].split("=");
- assertEquals(keyProto.getPartitionValue(), eachName[1]);
- }
- }
- }
private void testDropPartition(String tableName, String partitionName) throws Exception {
AlterTableDesc alterTableDesc = new AlterTableDesc();
diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml
index 3f93236..eff5f8f 100644
--- a/tajo-catalog/tajo-catalog-server/pom.xml
+++ b/tajo-catalog/tajo-catalog-server/pom.xml
@@ -148,14 +148,6 @@
<artifactId>tajo-rpc-protobuf</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-algebra</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-plan</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index 552168d..5a63bbb 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -47,7 +47,6 @@
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.rpc.BlockingRpcServer;
-import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListResponse;
@@ -933,38 +932,6 @@
}
@Override
- public ReturnState existsPartitions(RpcController controller, TableIdentifierProto request) throws
- ServiceException {
-
- String dbName = request.getDatabaseName();
- String tbName = request.getTableName();
-
- // linked meta data do not support partition.
- // So, the request that wants to get partitions in this db will be failed.
- if (linkedMetadataManager.existsDatabase(dbName)) {
- return errUndefinedPartitionMethod(tbName);
- }
-
- if (metaDictionary.isSystemDatabase(dbName)) {
- return errUndefinedPartitionMethod(tbName);
- } else {
- rlock.lock();
- try {
- if (store.existPartitions(dbName, tbName)) {
- return OK;
- } else {
- return errUndefinedPartitions(tbName);
- }
- } catch (Throwable t) {
- printStackTraceIfError(LOG, t);
- return returnError(t);
- } finally {
- rlock.unlock();
- }
- }
- }
-
- @Override
public GetPartitionDescResponse getPartitionByPartitionName(RpcController controller,
PartitionIdentifierProto request)
throws ServiceException {
@@ -1015,7 +982,7 @@
}
@Override
- public GetPartitionsResponse getPartitionsByTableName(RpcController controller, TableIdentifierProto request)
+ public GetPartitionsResponse getPartitionsByTableName(RpcController controller, PartitionIdentifierProto request)
throws ServiceException {
String dbName = request.getDatabaseName();
String tbName = request.getTableName();
@@ -1038,7 +1005,7 @@
rlock.lock();
try {
- List<PartitionDescProto> partitions = store.getPartitionsOfTable(dbName, tbName);
+ List<PartitionDescProto> partitions = store.getPartitions(dbName, tbName);
GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder();
for (PartitionDescProto partition : partitions) {
@@ -1085,41 +1052,6 @@
}
@Override
- public GetPartitionsResponse getPartitionsByAlgebra(RpcController controller,
- PartitionsByAlgebraProto request) throws ServiceException {
- String dbName = request.getDatabaseName();
- String tbName = request.getTableName();
-
- // linked meta data do not support partition.
- // So, the request that wants to get partitions in this db will be failed.
- if (linkedMetadataManager.existsDatabase(dbName)) {
- return GetPartitionsResponse.newBuilder().setState(errUndefinedPartitionMethod(tbName)).build();
- }
-
- if (metaDictionary.isSystemDatabase(dbName)) {
- return GetPartitionsResponse.newBuilder().setState(errUndefinedPartitionMethod(tbName)).build();
- } else {
- rlock.lock();
- try {
- GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder();
- List<PartitionDescProto> partitions = store.getPartitionsByAlgebra(request);
- builder.addAllPartition(partitions);
- builder.setState(OK);
- return builder.build();
- } catch (Throwable t) {
- printStackTraceIfError(LOG, t);
-
- return GetPartitionsResponse.newBuilder()
- .setState(returnError(t))
- .build();
-
- } finally {
- rlock.unlock();
- }
- }
- }
-
- @Override
public ReturnState addPartitions(RpcController controller, AddPartitionsProto request) {
TableIdentifierProto identifier = request.getTableIdentifier();
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index f9b04ef..4038c9d 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -25,27 +25,25 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tajo.algebra.Expr;
-import org.apache.tajo.algebra.JsonHelper;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.common.TajoDataTypes.*;
+import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.*;
import org.apache.tajo.util.JavaResourceUtil;
-import org.apache.tajo.plan.expr.*;
-import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import java.io.IOException;
import java.net.URI;
import java.sql.*;
-import java.sql.Date;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto;
@@ -59,8 +57,7 @@
protected final String catalogUri;
protected final String insertPartitionSql = "INSERT INTO " + TB_PARTTIONS
- + "(" + COL_TABLES_PK + ", PARTITION_NAME, PATH, " + COL_PARTITION_BYTES
- + ") VALUES (?, ? , ?, ?)";
+ + "(" + COL_TABLES_PK + ", PARTITION_NAME, PATH) VALUES (?, ? , ?)";
protected final String insertPartitionKeysSql = "INSERT INTO " + TB_PARTTION_KEYS + "("
+ COL_PARTITIONS_PK + ", " + COL_TABLES_PK + ", "
@@ -1383,7 +1380,6 @@
pstmt1.setInt(1, tableId);
pstmt1.setString(2, partition.getPartitionName());
pstmt1.setString(3, partition.getPath());
- pstmt1.setLong(4, partition.getNumBytes());
pstmt1.executeUpdate();
pstmt2 = conn.prepareStatement(insertPartitionKeysSql);
@@ -2137,7 +2133,7 @@
PartitionDescProto.Builder builder = null;
try {
- String sql = "SELECT PATH, " + COL_PARTITIONS_PK + ", " + COL_PARTITION_BYTES + " FROM " + TB_PARTTIONS +
+ String sql = "SELECT PATH, " + COL_PARTITIONS_PK + " FROM " + TB_PARTTIONS +
" WHERE " + COL_TABLES_PK + " = ? AND PARTITION_NAME = ? ";
if (LOG.isDebugEnabled()) {
@@ -2155,7 +2151,6 @@
builder.setId(res.getInt(COL_PARTITIONS_PK));
builder.setPath(res.getString("PATH"));
builder.setPartitionName(partitionName);
- builder.setNumBytes(res.getLong(COL_PARTITION_BYTES));
setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder);
} else {
throw new UndefinedPartitionException(partitionName);
@@ -2196,8 +2191,9 @@
}
@Override
- public List<PartitionDescProto> getPartitionsOfTable(String databaseName, String tableName)
+ public List<PartitionDescProto> getPartitions(String databaseName, String tableName)
throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException {
+
Connection conn = null;
ResultSet res = null;
PreparedStatement pstmt = null;
@@ -2209,8 +2205,8 @@
ensurePartitionTable(tableName, tableId);
try {
- String sql = "SELECT PATH, PARTITION_NAME, " + COL_PARTITIONS_PK + ", " + COL_PARTITION_BYTES
- + " FROM " + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? ";
+ String sql = "SELECT PATH, PARTITION_NAME, " + COL_PARTITIONS_PK + " FROM "
+ + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? ";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
@@ -2225,7 +2221,6 @@
builder = PartitionDescProto.newBuilder();
builder.setPath(res.getString("PATH"));
builder.setPartitionName(res.getString("PARTITION_NAME"));
- builder.setNumBytes(res.getLong(COL_PARTITION_BYTES));
setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder);
partitions.add(builder.build());
}
@@ -2238,256 +2233,6 @@
}
@Override
- public boolean existPartitions(String databaseName, String tableName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException {
-
- String sql = null;
- Connection conn = null;
- ResultSet res = null;
- PreparedStatement pstmt = null;
- boolean result = false;
-
- final int databaseId = getDatabaseId(databaseName);
- final int tableId = getTableId(databaseId, databaseName, tableName);
- ensurePartitionTable(tableName, tableId);
-
- try {
- if (this instanceof DerbyStore) {
- sql = "SELECT 1 FROM " + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? FETCH FIRST ROW ONLY ";
- } else {
- sql = "SELECT 1 FROM " + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? LIMIT 1 ";
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(sql);
- }
-
- conn = getConnection();
- pstmt = conn.prepareStatement(sql);
- pstmt.setInt(1, tableId);
- res = pstmt.executeQuery();
-
- if (res.next()) {
- result = true;
- }
- } catch (SQLException se) {
- throw new TajoInternalError(se);
- } finally {
- CatalogUtil.closeQuietly(pstmt, res);
- }
- return result;
- }
-
- @Override
- public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UnsupportedException {
- Connection conn = null;
- PreparedStatement pstmt = null;
- ResultSet res = null;
- int currentIndex = 1;
- String selectStatement = null;
- Pair<String, List<PartitionFilterSet>> pair = null;
-
- List<PartitionDescProto> partitions = TUtil.newList();
- List<PartitionFilterSet> filterSets = null;
-
- int databaseId = getDatabaseId(request.getDatabaseName());
- int tableId = getTableId(databaseId, request.getDatabaseName(), request.getTableName());
- if (!existPartitionMethod(request.getDatabaseName(), request.getTableName())) {
- throw new UndefinedPartitionMethodException(request.getTableName());
- }
-
- try {
- TableDescProto tableDesc = getTable(request.getDatabaseName(), request.getTableName());
-
- pair = getSelectStatementAndPartitionFilterSet(tableDesc.getTableName(), tableDesc.getPartition()
- .getExpressionSchema().getFieldsList(), request.getAlgebra());
-
- selectStatement = pair.getFirst();
- filterSets = pair.getSecond();
-
- conn = getConnection();
- pstmt = conn.prepareStatement(selectStatement);
-
- // Set table id by force because first parameter of all direct sql is table id
- pstmt.setInt(currentIndex, tableId);
- currentIndex++;
-
- for (PartitionFilterSet filter : filterSets) {
- // Set table id by force because all filters have table id as first parameter.
- pstmt.setInt(currentIndex, tableId);
- currentIndex++;
-
- for (Pair<Type, Object> parameter : filter.getParameters()) {
- switch (parameter.getFirst()) {
- case BOOLEAN:
- pstmt.setBoolean(currentIndex, (Boolean)parameter.getSecond());
- break;
- case INT8:
- pstmt.setLong(currentIndex, (Long) parameter.getSecond());
- break;
- case FLOAT8:
- pstmt.setDouble(currentIndex, (Double) parameter.getSecond());
- break;
- case DATE:
- pstmt.setDate(currentIndex, (Date) parameter.getSecond());
- break;
- case TIMESTAMP:
- pstmt.setTimestamp(currentIndex, (Timestamp) parameter.getSecond());
- break;
- case TIME:
- pstmt.setTime(currentIndex, (Time) parameter.getSecond());
- break;
- default:
- pstmt.setString(currentIndex, (String) parameter.getSecond());
- break;
- }
- currentIndex++;
- }
- }
-
- res = pstmt.executeQuery();
-
- while (res.next()) {
- PartitionDescProto.Builder builder = PartitionDescProto.newBuilder();
-
- builder.setId(res.getInt(COL_PARTITIONS_PK));
- builder.setPartitionName(res.getString("PARTITION_NAME"));
- builder.setPath(res.getString("PATH"));
- builder.setNumBytes(res.getLong(COL_PARTITION_BYTES));
-
- partitions.add(builder.build());
- }
- } catch (SQLException se) {
- throw new TajoInternalError(se);
- } finally {
- CatalogUtil.closeQuietly(pstmt, res);
- }
-
- return partitions;
- }
-
- /**
- * Create a select statement and parameters for querying partitions and partition keys in CatalogStore.
- *
- * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
- * Assume that an user gives a condition WHERE (col1 ='1' or col1 = '100') and col3 > 20.
- * There is no filter condition corresponding to col2.
- *
- * Then, the sql would be generated as following:
- *
- * SELECT A.PARTITION_ID, A.PARTITION_NAME, A.PATH FROM PARTITIONS A
- * WHERE A.TID = ?
- * AND A.PARTITION_ID IN (
- * SELECT T1.PARTITION_ID FROM PARTITION_KEYS T1
- * JOIN PARTITION_KEYS T2 ON T1.TID=T2.TID AND T1.PARTITION_ID = T2.PARTITION_ID AND T2.TID = ?
- * AND ( T2.COLUMN_NAME = 'col2' AND T2.PARTITION_VALUE IS NOT NULL )
- * JOIN PARTITION_KEYS T3 ON T1.TID=T3.TID AND T1.PARTITION_ID = T3.PARTITION_ID AND T3.TID = ?
- * AND ( T3.COLUMN_NAME = 'col3' AND T3.PARTITION_VALUE > ? )
- * WHERE T1.TID = ? AND ( T1.COLUMN_NAME = 'col1' AND T1.PARTITION_VALUE = ? )
- * OR ( T1.COLUMN_NAME = 'col1' AND T1.PARTITION_VALUE = ? )
- )
- *
- * @param tableName the table name
- * @param partitionColumns list of partition column
- * @param json the algebra expression
- * @return the select statement and partition filter sets
- */
- private Pair<String, List<PartitionFilterSet>> getSelectStatementAndPartitionFilterSet(String tableName,
- List<ColumnProto> partitionColumns, String json) {
-
- Pair<String, List<PartitionFilterSet>> result = null;
- Expr[] exprs = null;
-
- try {
- List<PartitionFilterSet> filterSets = TUtil.newList();
-
- if (json != null && !json.isEmpty()) {
- Expr algebra = JsonHelper.fromJson(json, Expr.class);
- exprs = AlgebraicUtil.toConjunctiveNormalFormArray(algebra);
- }
-
- // Write table alias for all levels
- String tableAlias;
-
- PartitionFilterAlgebraVisitor visitor = new PartitionFilterAlgebraVisitor();
- visitor.setIsHiveCatalog(false);
-
- Expr[] filters = AlgebraicUtil.getRearrangedCNFExpressions(tableName, partitionColumns, exprs);
-
- StringBuffer sb = new StringBuffer();
- sb.append("\n SELECT A.").append(CatalogConstants.COL_PARTITIONS_PK)
- .append(", A.PARTITION_NAME, A.PATH ").append(", ").append(COL_PARTITION_BYTES)
- .append(" FROM ").append(CatalogConstants.TB_PARTTIONS).append(" A ")
- .append("\n WHERE A.").append(CatalogConstants.COL_TABLES_PK).append(" = ? ")
- .append("\n AND A.").append(CatalogConstants.COL_PARTITIONS_PK).append(" IN (")
- .append("\n SELECT T1.").append(CatalogConstants.COL_PARTITIONS_PK)
- .append(" FROM ").append(CatalogConstants.TB_PARTTION_KEYS).append(" T1 ");
-
- // Write join clause from second column to last column.
- Column target;
-
- for (int i = 1; i < partitionColumns.size(); i++) {
- target = new Column(partitionColumns.get(i));
- tableAlias = "T" + (i+1);
-
- visitor.setColumn(target);
- visitor.setTableAlias(tableAlias);
- visitor.visit(null, new Stack<Expr>(), filters[i]);
-
- sb.append("\n JOIN ").append(CatalogConstants.TB_PARTTION_KEYS).append(" ").append(tableAlias)
- .append(" ON T1.").append(CatalogConstants.COL_TABLES_PK).append("=")
- .append(tableAlias).append(".").append(CatalogConstants.COL_TABLES_PK)
- .append(" AND T1.").append(CatalogConstants.COL_PARTITIONS_PK)
- .append(" = ").append(tableAlias).append(".").append(CatalogConstants.COL_PARTITIONS_PK)
- .append(" AND ").append(tableAlias).append(".").append(CatalogConstants.COL_TABLES_PK).append(" = ? AND ");
- sb.append(visitor.getResult());
-
- // Set parameters for executing PrepareStament
- PartitionFilterSet filterSet = new PartitionFilterSet();
- filterSet.setColumnName(target.getSimpleName());
-
- List<Pair<Type, Object>> list = TUtil.newList();
- list.addAll(visitor.getParameters());
- filterSet.addParameters(list);
-
- filterSets.add(filterSet);
- visitor.clearParameters();
- }
-
- // Write where clause for first column
- target = new Column(partitionColumns.get(0));
- tableAlias = "T1";
- visitor.setColumn(target);
- visitor.setTableAlias(tableAlias);
- visitor.visit(null, new Stack<Expr>(), filters[0]);
-
- sb.append("\n WHERE T1.").append(CatalogConstants.COL_TABLES_PK).append(" = ? AND ");
- sb.append(visitor.getResult())
- .append("\n )");
- sb.append("\n ORDER BY A.PARTITION_NAME");
-
- // Set parameters for executing PrepareStament
- PartitionFilterSet filterSet = new PartitionFilterSet();
- filterSet.setColumnName(target.getSimpleName());
-
- List<Pair<Type, Object>> list = TUtil.newList();
- list.addAll(visitor.getParameters());
- filterSet.addParameters(list);
-
- filterSets.add(filterSet);
-
- result = new Pair<>(sb.toString(), filterSets);
- } catch (TajoException e) {
- throw new TajoInternalError(e);
- }
-
- return result;
- }
-
-
- @Override
public List<TablePartitionProto> getAllPartitions() {
Connection conn = null;
Statement stmt = null;
@@ -2578,7 +2323,6 @@
pstmt3.setInt(1, tableId);
pstmt3.setString(2, partition.getPartitionName());
pstmt3.setString(3, partition.getPath());
- pstmt3.setLong(4, partition.getNumBytes());
pstmt3.addBatch();
pstmt3.clearParameters();
@@ -3207,33 +2951,4 @@
return exist;
}
-
- class PartitionFilterSet {
- private String columnName;
- private List<Pair<Type, Object>> parameters;
-
- public PartitionFilterSet() {
- parameters = TUtil.newList();
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
- public List<Pair<Type, Object>> getParameters() {
- return parameters;
- }
-
- public void setParameters(List<Pair<Type, Object>> parameters) {
- this.parameters = parameters;
- }
-
- public void addParameters(List<Pair<Type, Object>> parameters) {
- this.parameters.addAll(parameters);
- }
- }
}
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
index 5288979..a067a53 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
@@ -92,96 +92,18 @@
/************************** PARTITIONS *****************************/
/**
- * Check if list of partitions exist on catalog.
- *
- * @param databaseName
- * @param tableName
- * @return
- * @throws UndefinedDatabaseException
- * @throws UndefinedTableException
- * @throws UndefinedPartitionMethodException
- */
- boolean existPartitions(String databaseName, String tableName) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException;
-
- /**
* Get all partitions of a table
* @param tableName the table name
* @return
* @throws TajoException
*/
- List<CatalogProtos.PartitionDescProto> getPartitionsOfTable(String databaseName, String tableName) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UnsupportedException;
+ List<CatalogProtos.PartitionDescProto> getPartitions(String databaseName, String tableName) throws
+ UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException;
CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName,
- String partitionName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException;
-
- /**
- * Get list of partitions matching specified algrbra expression.
- *
- * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
- * Assume that an user want to give a condition WHERE (col1 ='1' or col1 = '100') and col3 > 20 .
- *
- * Then, the algebra expression would be written as following:
- *
- * {
- * "LeftExpr": {
- * "LeftExpr": {
- * "Qualifier": "default.table1",
- * "ColumnName": "col3",
- * "OpType": "Column"
- * },
- * "RightExpr": {
- * "Value": "20.0",
- * "ValueType": "Unsigned_Integer",
- * "OpType": "Literal"
- * },
- * "OpType": "GreaterThan"
- * },
- * "RightExpr": {
- * "LeftExpr": {
- * "LeftExpr": {
- * "Qualifier": "default.table1",
- * "ColumnName": "col1",
- * "OpType": "Column"
- * },
- * "RightExpr": {
- * "Value": "1",
- * "ValueType": "String",
- * "OpType": "Literal"
- * },
- * "OpType": "Equals"
- * },
- * "RightExpr": {
- * "LeftExpr": {
- * "Qualifier": "default.table1",
- * "ColumnName": "col1",
- * "OpType": "Column"
- * },
- * "RightExpr": {
- * "Value": "100",
- * "ValueType": "String",
- * "OpType": "Literal"
- * },
- * "OpType": "Equals"
- * },
- * "OpType": "Or"
- * },
- * "OpType": "And"
- * }
- *
- * @param request the database name, the table name, the algebra expression
- * @return list of PartitionDescProto
- * @throws UndefinedDatabaseException
- * @throws UndefinedTableException
- * @throws UndefinedPartitionMethodException
- * @throws UndefinedOperatorException
- */
- List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UndefinedOperatorException, UnsupportedException;
+ String partitionName)
+ throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException,
+ UndefinedPartitionMethodException;
List<TablePartitionProto> getAllPartitions();
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
index 96100e8..2ae85d1 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -32,7 +33,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -177,7 +178,6 @@
TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE,
PARTITION_NAME VARCHAR(767),
PATH VARCHAR(1024),
- NUM_BYTES BIGINT,
CONSTRAINT C_PARTITIONS_PK PRIMARY KEY (PARTITION_ID),
CONSTRAINT C_PARTITIONS_UNIQ UNIQUE (TID, PARTITION_NAME)
)]]>
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
index a94489d..6fb2610 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -32,7 +33,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -154,7 +155,6 @@
TID INT NOT NULL,
PARTITION_NAME VARCHAR(255) BINARY,
PATH VARCHAR(4096) BINARY,
- NUM_BYTES BIGINT,
UNIQUE INDEX PARTITION_UNIQUE_IDX (TID, PARTITION_NAME),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)]]>
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
index c0dadaa..d4ad970 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -32,7 +33,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -158,7 +159,6 @@
TID INT NOT NULL,
PARTITION_NAME VARCHAR(255) BINARY,
PATH VARCHAR(4096) BINARY,
- NUM_BYTES BIGINT,
UNIQUE INDEX PARTITION_UNIQUE_IDX (TID, PARTITION_NAME),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)]]>
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
index 190270c..8f23514 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -32,7 +33,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object order="0" type="table" name="meta">
<tns:sql><![CDATA[
@@ -211,7 +212,6 @@
TID INT NOT NULL,
PARTITION_NAME VARCHAR2(767),
PATH VARCHAR2(4000),
- NUM_BYTES NUMBER(38),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
CONSTRAINT C_PARTITIONS_UNIQ UNIQUE (TID, PARTITION_NAME)
)]]>
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
index 33a1fd2..6c0f26a 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
@@ -21,6 +21,7 @@
xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -35,7 +36,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object name="META" type="table" order="0">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -174,7 +175,6 @@
PARTITION_NAME VARCHAR(128),
PARTITION_VALUE VARCHAR(1024),
PATH VARCHAR(4096),
- NUM_BYTES BIGINT,
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
UNIQUE (TID, PARTITION_NAME)
)]]>
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index cbddab8..1671f57 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -59,7 +59,7 @@
static CatalogServer server;
static CatalogService catalog;
- @BeforeClass
+ @BeforeClass
public static void setUp() throws Exception {
server = new MiniCatalogServer();
@@ -590,7 +590,7 @@
assertEquals(retrived.getFunctionName(),"test2");
assertEquals(retrived.getLegacyFuncClass(),TestFunc1.class);
- assertEquals(retrived.getFuncType(), FunctionType.UDF);
+ assertEquals(retrived.getFuncType(),FunctionType.UDF);
}
@Test
@@ -785,8 +785,7 @@
.addColumn("age", Type.INT4)
.addColumn("score", Type.FLOAT8);
- String simpleTableName = "addedtable";
- String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName);
+ String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
@@ -801,7 +800,7 @@
TableDesc desc =
new TableDesc(tableName, schema, meta,
- new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri());
+ new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionMethodDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
@@ -816,17 +815,14 @@
testAddPartition(tableName, "id=10/name=aaa");
testAddPartition(tableName, "id=20/name=bbb");
- List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
+ List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable");
assertNotNull(partitions);
assertEquals(partitions.size(), 2);
- assertEquals(partitions.get(0).getNumBytes(), 0L);
-
- testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName);
testDropPartition(tableName, "id=10/name=aaa");
testDropPartition(tableName, "id=20/name=bbb");
- partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
+ partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable");
assertNotNull(partitions);
assertEquals(partitions.size(), 0);
@@ -834,99 +830,6 @@
assertFalse(catalog.existsTable(tableName));
}
- private void testGetPartitionsByAlgebra(String databaseName, String tableName) throws Exception {
- String qfTableName = databaseName + "." + tableName;
-
- // Equals Operator
- CatalogProtos.PartitionsByAlgebraProto.Builder request = CatalogProtos.PartitionsByAlgebraProto.newBuilder();
- request.setDatabaseName(databaseName);
- request.setTableName(tableName);
-
- String algebra = "{\n" +
- " \"LeftExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"id\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"10\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"name\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"aaa\",\n" +
- " \"ValueType\": \"String\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"OpType\": \"And\"\n" +
- "}";
-
- request.setAlgebra(algebra);
-
- List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsByAlgebra(request.build());
- assertNotNull(partitions);
- assertEquals(1, partitions.size());
-
- // GreaterThan Operator and InPredicate Operatior
- algebra = "{\n" +
- " \"LeftExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"id\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"0\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"GreaterThan\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"IsNot\": false,\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"name\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Values\": [\n" +
- " {\n" +
- " \"Value\": \"aaa\",\n" +
- " \"ValueType\": \"String\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " {\n" +
- " \"Value\": \"bbb\",\n" +
- " \"ValueType\": \"String\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " }\n" +
- " ],\n" +
- " \"OpType\": \"ValueList\"\n" +
- " },\n" +
- " \"OpType\": \"InPredicate\"\n" +
- " },\n" +
- " \"OpType\": \"And\"\n" +
- "}";
-
- request.setAlgebra(algebra);
-
- partitions = catalog.getPartitionsByAlgebra(request.build());
- assertNotNull(partitions);
- assertEquals(2, partitions.size());
- }
-
private void testAddPartition(String tableName, String partitionName) throws Exception {
AlterTableDesc alterTableDesc = new AlterTableDesc();
alterTableDesc.setTableName(tableName);
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
index 99a7b43..a8b3feb 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
@@ -212,7 +212,7 @@
// Test get partitions of a table
//////////////////////////////////////////////////////////////////////////////
- List<PartitionDescProto> partitionDescs = catalog.getPartitionsOfTable("TestDatabase1", "TestPartition1");
+ List<PartitionDescProto> partitionDescs = catalog.getPartitions("TestDatabase1", "TestPartition1");
assertEquals(2, partitionDescs.size());
Map<String, PartitionDescProto> tablePartitionMap = new HashMap<>();
for (PartitionDescProto eachPartition : partitionDescs) {
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
index 8652478..0c946ba 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
@@ -20,6 +20,8 @@
import com.google.protobuf.ServiceException;
import org.apache.commons.cli.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.tajo.auth.UserRoleInfo;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -42,6 +44,7 @@
import java.util.List;
public class TajoDump {
+ private static final Log LOG = LogFactory.getLog(TajoDump.class);
private static final org.apache.commons.cli.Options options;
static {
@@ -180,25 +183,24 @@
if (table.getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")) {
continue;
}
-
+
if (table.isExternal()) {
writer.write(DDLBuilder.buildDDLForExternalTable(table));
} else {
writer.write(DDLBuilder.buildDDLForBaseTable(table));
}
+
+ // TODO: This should be improved at TAJO-1891
if (table.hasPartition()) {
writer.write("\n\n");
writer.write("--\n");
writer.write(String.format("-- Table Partitions: %s%n", tableName));
+ writer.write("-- Partition dump and restore are not supported yet\n");
writer.write("--\n");
- // TODO: This should be improved at TAJO-1891
// List<PartitionDescProto> partitionProtos = client.getPartitionsOfTable(fqName);
// for (PartitionDescProto eachPartitionProto : partitionProtos) {
// writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto));
// }
- writer.write(String.format("ALTER TABLE %s REPAIR PARTITION;",
- CatalogUtil.denormalizeIdentifier(databaseName) + "." + CatalogUtil.denormalizeIdentifier(tableName)));
-
writer.write("\n\n");
}
@@ -218,6 +220,7 @@
}
}
+
private static String toDateString() {
DateFormat df = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
java.util.Date today = Calendar.getInstance().getTime();
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
index 0e4c1b8..22f3a09 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
@@ -144,7 +144,7 @@
* @param tableName The table name to get. This name is case sensitive.
* @return lists of partitions
*/
- List<PartitionDescProto> getPartitionsOfTable(final String tableName) throws UndefinedDatabaseException,
+ List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException,
UndefinedTableException, UndefinedPartitionMethodException;
List<CatalogProtos.FunctionDescProto> getFunctions(final String functionName);
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
index 97f28a6..cd48c5f 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
@@ -245,7 +245,7 @@
}
@Override
- public List<PartitionDescProto> getPartitionsOfTable(final String tableName) throws UndefinedDatabaseException,
+ public List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException,
UndefinedTableException, UndefinedPartitionMethodException {
final BlockingInterface stub = conn.getTMStub();
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
index cbc194f..c685609 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
@@ -35,7 +35,6 @@
import org.apache.tajo.jdbc.TajoMemoryResultSet;
import org.apache.tajo.service.ServiceTracker;
import org.apache.tajo.util.KeyValueSet;
-import org.apache.tajo.util.TUtil;
import java.net.InetSocketAddress;
import java.net.URI;
@@ -246,9 +245,9 @@
return catalogClient.getFunctions(functionName);
}
- public List<PartitionDescProto> getPartitionsOfTable(final String tableName) throws UndefinedDatabaseException,
+ public List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException,
UndefinedTableException, UndefinedPartitionMethodException {
- return catalogClient.getPartitionsOfTable(tableName);
+ return catalogClient.getAllPartitions(tableName);
}
@Override
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java
index f526f62..ffb5eb3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java
@@ -65,7 +65,6 @@
ADD_MESSAGE(UNDEFINED_FUNCTION, "function does not exist: %s", 1);
ADD_MESSAGE(UNDEFINED_PARTITION_METHOD, "table '%s' is not a partitioned table", 1);
ADD_MESSAGE(UNDEFINED_PARTITION, "partition '%s' does not exist", 1);
- ADD_MESSAGE(UNDEFINED_PARTITIONS, "No partitions for table '%s'", 1);
ADD_MESSAGE(UNDEFINED_PARTITION_KEY, "'%s' column is not a partition key", 1);
ADD_MESSAGE(UNDEFINED_OPERATOR, "operator does not exist: '%s'", 1);
ADD_MESSAGE(UNDEFINED_INDEX_FOR_TABLE, "index ''%s' does not exist", 1);
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
index adfb0c4..65e6b9b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
@@ -67,7 +67,6 @@
ADD_EXCEPTION(UNDEFINED_TABLE, UndefinedTableException.class);
ADD_EXCEPTION(UNDEFINED_COLUMN, UndefinedColumnException.class);
ADD_EXCEPTION(UNDEFINED_FUNCTION, UndefinedFunctionException.class);
- ADD_EXCEPTION(UNDEFINED_PARTITION_METHOD, UndefinedPartitionMethodException.class);
ADD_EXCEPTION(UNDEFINED_PARTITION, UndefinedPartitionException.class);
ADD_EXCEPTION(UNDEFINED_PARTITION_KEY, UndefinedPartitionKeyException.class);
ADD_EXCEPTION(UNDEFINED_OPERATOR, UndefinedOperatorException.class);
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java
index 3257f46..01845f4 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java
@@ -161,10 +161,6 @@
return returnError(ResultCode.UNDEFINED_PARTITION, partitionName);
}
- public static ReturnState errUndefinedPartitions(String tbName) {
- return returnError(ResultCode.UNDEFINED_PARTITIONS, tbName);
- }
-
public static ReturnState errUndefinedPartitionMethod(String tbName) {
return returnError(ResultCode.UNDEFINED_PARTITION_METHOD, tbName);
}
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java b/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java
index ca61c70..459269c 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java
@@ -29,7 +29,7 @@
super(state);
}
- public UndefinedPartitionMethodException(String tableName) {
- super(ResultCode.UNDEFINED_PARTITION_METHOD, tableName);
+ public UndefinedPartitionMethodException(String partitionName) {
+ super(ResultCode.UNDEFINED_PARTITION_METHOD, partitionName);
}
}
diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto
index 7585a83..fc69c4e 100644
--- a/tajo-common/src/main/proto/errors.proto
+++ b/tajo-common/src/main/proto/errors.proto
@@ -116,7 +116,6 @@
UNDEFINED_OPERATOR = 522; // SQLState: 42883 (=UNDEFINED_FUNCTION)
UNDEFINED_PARTITION_KEY = 523; // ?
UNDEFINED_TABLESPACE_HANDLER = 524; // SQLState: 42T11 - No Tablespace Handler for the URI scheme
- UNDEFINED_PARTITIONS = 525; // ?
DUPLICATE_TABLESPACE = 531;
DUPLICATE_DATABASE = 532; // SQLState: 42P04
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java
deleted file mode 100644
index 2baa79a..0000000
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.engine.planner;
-
-import org.apache.tajo.LocalTajoTestingUtility;
-import org.apache.tajo.QueryVars;
-import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.algebra.*;
-import org.apache.tajo.benchmark.TPCH;
-import org.apache.tajo.catalog.*;
-import org.apache.tajo.engine.function.FunctionLoader;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.parser.sql.SQLAnalyzer;
-import org.apache.tajo.plan.LogicalOptimizer;
-import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.plan.LogicalPlanner;
-import org.apache.tajo.plan.expr.AlgebraicUtil;
-import org.apache.tajo.plan.expr.EvalNode;
-import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.plan.logical.NodeType;
-import org.apache.tajo.plan.logical.ScanNode;
-import org.apache.tajo.plan.util.EvalNodeToExprConverter;
-import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.session.Session;
-import org.apache.tajo.storage.TablespaceManager;
-import org.apache.tajo.util.CommonTestingUtil;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.Stack;
-
-import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
-import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.junit.Assert.*;
-
-public class TestEvalNodeToExprConverter {
- private static TajoTestingCluster util;
- private static CatalogService catalog;
- private static SQLAnalyzer sqlAnalyzer;
- private static LogicalPlanner planner;
- private static TPCH tpch;
- private static Session session = LocalTajoTestingUtility.createDummySession();
-
- @BeforeClass
- public static void setUp() throws Exception {
- util = new TajoTestingCluster();
- util.startCatalogCluster();
- catalog = util.getCatalogService();
- catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234");
- catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
-
- for (FunctionDesc funcDesc : FunctionLoader.findLegacyFunctions()) {
- catalog.createFunction(funcDesc);
- }
-
- // TPC-H Schema for Complex Queries
- String [] tpchTables = {
- "part", "supplier", "partsupp", "nation", "region", "lineitem"
- };
- tpch = new TPCH();
- tpch.loadSchemas();
- tpch.loadOutSchema();
- for (String table : tpchTables) {
- TableMeta m = CatalogUtil.newTableMeta("TEXT");
- TableDesc d = CatalogUtil.newTableDesc(
- CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m,
- CommonTestingUtil.getTestDir());
- catalog.createTable(d);
- }
-
- sqlAnalyzer = new SQLAnalyzer();
- planner = new LogicalPlanner(catalog, TablespaceManager.getInstance());
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- util.shutdownCatalogCluster();
- }
-
- static String[] QUERIES = {
- "select * from lineitem where L_ORDERKEY > 500", //0
- "select * from region where r_name = 'EUROPE'", //1
- "select * from lineitem where L_DISCOUNT >= 0.05 and L_DISCOUNT <= 0.07 OR L_QUANTITY < 24.0 ", //2
- "select * from lineitem where L_DISCOUNT between 0.06 - 0.01 and 0.08 + 0.02 and L_ORDERKEY < 24 ", //3
- "select * from lineitem where (case when L_DISCOUNT > 0.0 then L_DISCOUNT / L_TAX else null end) > 1.2 ", //4
- "select * from part where p_brand = 'Brand#23' and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') " +
- "and p_size between 1 and 10", //5
- };
-
- private static QueryContext createQueryContext() {
- QueryContext qc = new QueryContext(util.getConfiguration(), session);
- qc.put(QueryVars.DEFAULT_SPACE_URI, "file:/");
- qc.put(QueryVars.DEFAULT_SPACE_ROOT_URI, "file:/");
- return qc;
- }
-
- @Test
- public final void testBinaryOperator1() throws CloneNotSupportedException, TajoException {
- QueryContext qc = createQueryContext();
-
- Expr expr = sqlAnalyzer.parse(QUERIES[0]);
-
- LogicalPlan plan = planner.createPlan(qc, expr);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
- optimizer.optimize(plan);
-
- LogicalNode node = plan.getRootBlock().getRoot();
- ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN);
-
- EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName());
- convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>());
-
- Expr resultExpr = convertor.getResult();
-
- BinaryOperator binaryOperator = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThan);
- assertNotNull(binaryOperator);
-
- ColumnReferenceExpr column = binaryOperator.getLeft();
- assertEquals("default.lineitem", column.getQualifier());
- assertEquals("l_orderkey", column.getName());
-
- LiteralValue literalValue = binaryOperator.getRight();
- assertEquals("500", literalValue.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Integer, literalValue.getValueType());
- }
-
- @Test
- public final void testBinaryOperator2() throws CloneNotSupportedException, TajoException {
- QueryContext qc = createQueryContext();
-
- Expr expr = sqlAnalyzer.parse(QUERIES[1]);
-
- LogicalPlan plan = planner.createPlan(qc, expr);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
- optimizer.optimize(plan);
-
- LogicalNode node = plan.getRootBlock().getRoot();
- ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN);
-
- EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName());
- convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>());
-
- Expr resultExpr = convertor.getResult();
- BinaryOperator equals = AlgebraicUtil.findTopExpr(resultExpr, OpType.Equals);
- assertNotNull(equals);
-
- ColumnReferenceExpr column = equals.getLeft();
- assertEquals("default.region", column.getQualifier());
- assertEquals("r_name", column.getName());
-
- LiteralValue literalValue = equals.getRight();
- assertEquals("EUROPE", literalValue.getValue());
- assertEquals(LiteralValue.LiteralType.String, literalValue.getValueType());
- }
-
- @Test
- public final void testBinaryOperator3() throws CloneNotSupportedException, TajoException {
- QueryContext qc = createQueryContext();
-
- Expr expr = sqlAnalyzer.parse(QUERIES[2]);
-
- LogicalPlan plan = planner.createPlan(qc, expr);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
- optimizer.optimize(plan);
-
- LogicalNode node = plan.getRootBlock().getRoot();
- ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN);
-
- EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName());
- convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>());
-
- Expr resultExpr = convertor.getResult();
-
- BinaryOperator greaterThanOrEquals = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThanOrEquals);
- assertNotNull(greaterThanOrEquals);
-
- ColumnReferenceExpr greaterThanOrEqualsLeft = greaterThanOrEquals.getLeft();
- assertEquals("default.lineitem", greaterThanOrEqualsLeft.getQualifier());
- assertEquals("l_discount", greaterThanOrEqualsLeft.getName());
-
- LiteralValue greaterThanOrEqualsRight = greaterThanOrEquals.getRight();
- assertEquals("0.05", greaterThanOrEqualsRight.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, greaterThanOrEqualsRight.getValueType());
-
- BinaryOperator lessThanOrEquals = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThanOrEquals);
- assertNotNull(lessThanOrEquals);
-
- ColumnReferenceExpr lessThanOrEqualsLeft = lessThanOrEquals.getLeft();
- assertEquals("default.lineitem", lessThanOrEqualsLeft.getQualifier());
- assertEquals("l_discount", lessThanOrEqualsLeft.getName());
-
- LiteralValue lessThanOrEqualsRight = lessThanOrEquals.getRight();
- assertEquals("0.07", lessThanOrEqualsRight.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, lessThanOrEqualsRight.getValueType());
-
- BinaryOperator lessThan = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThan);
- assertNotNull(lessThan);
-
- ColumnReferenceExpr lessThanLeft = lessThan.getLeft();
- assertEquals("default.lineitem", lessThanLeft.getQualifier());
- assertEquals("l_quantity", lessThanLeft.getName());
-
- LiteralValue lessThanRight = lessThan.getRight();
- assertEquals("24.0", lessThanRight.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, lessThanRight.getValueType());
-
- BinaryOperator leftExpr = new BinaryOperator(OpType.And, greaterThanOrEquals, lessThanOrEquals);
-
- BinaryOperator topExpr = AlgebraicUtil.findTopExpr(resultExpr, OpType.Or);
- assertEquals(leftExpr, topExpr.getLeft());
- assertEquals(lessThan, topExpr.getRight());
- }
-
- @Test
- public final void testBetweenPredicate() throws CloneNotSupportedException, TajoException {
- QueryContext qc = createQueryContext();
-
- Expr expr = sqlAnalyzer.parse(QUERIES[3]);
-
- LogicalPlan plan = planner.createPlan(qc, expr);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
- optimizer.optimize(plan);
-
- LogicalNode node = plan.getRootBlock().getRoot();
- ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN);
-
- EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName());
- convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>());
-
- Expr resultExpr = convertor.getResult();
-
- BinaryOperator binaryOperator = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThan);
- assertNotNull(binaryOperator);
- ColumnReferenceExpr column = binaryOperator.getLeft();
- assertEquals("default.lineitem", column.getQualifier());
- assertEquals("l_orderkey", column.getName());
-
- LiteralValue literalValue = binaryOperator.getRight();
- assertEquals("24", literalValue.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Integer, literalValue.getValueType());
-
- BetweenPredicate between = AlgebraicUtil.findTopExpr(resultExpr, OpType.Between);
- assertFalse(between.isNot());
- assertFalse(between.isSymmetric());
-
- ColumnReferenceExpr predicand = (ColumnReferenceExpr)between.predicand();
- assertEquals("default.lineitem", predicand.getQualifier());
- assertEquals("l_discount", predicand.getName());
-
- BinaryOperator begin = (BinaryOperator)between.begin();
- assertEquals(OpType.Minus, begin.getType());
- LiteralValue left = begin.getLeft();
- assertEquals("0.06", left.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, left.getValueType());
- LiteralValue right = begin.getRight();
- assertEquals("0.01", right.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, right.getValueType());
-
- BinaryOperator end = (BinaryOperator)between.end();
- assertEquals(OpType.Plus, end.getType());
- left = end.getLeft();
- assertEquals("0.08", left.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, left.getValueType());
- right = end.getRight();
- assertEquals("0.02", right.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, right.getValueType());
- }
-
- @Test
- public final void testCaseWhenPredicate() throws CloneNotSupportedException, TajoException {
- QueryContext qc = createQueryContext();
-
- Expr expr = sqlAnalyzer.parse(QUERIES[4]);
-
- LogicalPlan plan = planner.createPlan(qc, expr);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
- optimizer.optimize(plan);
-
- LogicalNode node = plan.getRootBlock().getRoot();
- ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN);
-
- EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName());
- convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>());
-
- Expr resultExpr = convertor.getResult();
-
- CaseWhenPredicate caseWhen = AlgebraicUtil.findTopExpr(resultExpr, OpType.CaseWhen);
- assertNotNull(caseWhen);
-
- CaseWhenPredicate.WhenExpr[] whenExprs = new CaseWhenPredicate.WhenExpr[1];
- caseWhen.getWhens().toArray(whenExprs);
-
- BinaryOperator condition = (BinaryOperator) whenExprs[0].getCondition();
- assertEquals(OpType.GreaterThan, condition.getType());
-
- ColumnReferenceExpr conditionLeft = condition.getLeft();
- assertEquals("default.lineitem", conditionLeft.getQualifier());
- assertEquals("l_discount", conditionLeft.getName());
-
- LiteralValue conditionRight = condition.getRight();
- assertEquals("0.0", conditionRight.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, conditionRight.getValueType());
-
- BinaryOperator result = (BinaryOperator) whenExprs[0].getResult();
- assertEquals(OpType.Divide, result.getType());
- ColumnReferenceExpr resultLeft = result.getLeft();
- assertEquals("default.lineitem", resultLeft.getQualifier());
- assertEquals("l_discount", resultLeft.getName());
-
- ColumnReferenceExpr resultRight = result.getRight();
- assertEquals("default.lineitem", resultRight.getQualifier());
- assertEquals("l_tax", resultRight.getName());
-
- BinaryOperator greaterThan = AlgebraicUtil.findMostBottomExpr(resultExpr, OpType.GreaterThan);
- assertNotNull(greaterThan);
-
- assertEquals(greaterThan.getLeft(), caseWhen);
-
- LiteralValue binaryOperatorRight = greaterThan.getRight();
- assertEquals("1.2", binaryOperatorRight.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Float, conditionRight.getValueType());
- }
-
- @Test
- public final void testThreeFilters() throws CloneNotSupportedException, TajoException {
- QueryContext qc = createQueryContext();
-
- Expr expr = sqlAnalyzer.parse(QUERIES[5]);
-
- LogicalPlan plan = planner.createPlan(qc, expr);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
- optimizer.optimize(plan);
-
- LogicalNode node = plan.getRootBlock().getRoot();
- ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN);
-
- EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName());
- convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>());
-
- Expr resultExpr = convertor.getResult();
-
- BetweenPredicate between = AlgebraicUtil.findTopExpr(resultExpr, OpType.Between);
- assertFalse(between.isNot());
- assertFalse(between.isSymmetric());
-
- ColumnReferenceExpr predicand = (ColumnReferenceExpr)between.predicand();
- assertEquals("default.part", predicand.getQualifier());
- assertEquals("p_size", predicand.getName());
-
- LiteralValue begin = (LiteralValue)between.begin();
- assertEquals("1", begin.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Integer, begin.getValueType());
-
- LiteralValue end = (LiteralValue)between.end();
- assertEquals("10", end.getValue());
- assertEquals(LiteralValue.LiteralType.Unsigned_Integer, end.getValueType());
-
- BinaryOperator equals = AlgebraicUtil.findTopExpr(resultExpr, OpType.Equals);
- assertNotNull(equals);
-
- ColumnReferenceExpr equalsLeft = equals.getLeft();
- assertEquals("default.part", equalsLeft.getQualifier());
- assertEquals("p_brand", equalsLeft.getName());
-
- LiteralValue equalsRight = equals.getRight();
- assertEquals("Brand#23", equalsRight.getValue());
- assertEquals(LiteralValue.LiteralType.String, equalsRight.getValueType());
-
- InPredicate inPredicate = AlgebraicUtil.findTopExpr(resultExpr, OpType.InPredicate);
- assertNotNull(inPredicate);
-
- ValueListExpr valueList = (ValueListExpr)inPredicate.getInValue();
- assertEquals(4, valueList.getValues().length);
- for(int i = 0; i < valueList.getValues().length; i++) {
- LiteralValue literalValue = (LiteralValue) valueList.getValues()[i];
-
- if (i == 0) {
- assertEquals("MED BAG", literalValue.getValue());
- } else if (i == 1) {
- assertEquals("MED BOX", literalValue.getValue());
- } else if (i == 2) {
- assertEquals("MED PKG", literalValue.getValue());
- } else {
- assertEquals("MED PACK", literalValue.getValue());
- }
- }
- }
-}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java
new file mode 100644
index 0000000..e8b1011
--- /dev/null
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java
@@ -0,0 +1,432 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.planner;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.LocalTajoTestingUtility;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.algebra.Expr;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter;
+import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.KeyValueSet;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestPartitionedTableRewriter extends QueryTestCaseBase {
+
+ final static String PARTITION_TABLE_NAME = "tb_partition";
+ final static String MULTIPLE_PARTITION_TABLE_NAME = "tb_multiple_partition";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ FileSystem fs = FileSystem.get(conf);
+ Path rootDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration());
+
+ Schema schema = new Schema();
+ schema.addColumn("n_nationkey", TajoDataTypes.Type.INT8);
+ schema.addColumn("n_name", TajoDataTypes.Type.TEXT);
+ schema.addColumn("n_regionkey", TajoDataTypes.Type.INT8);
+
+ TableMeta meta = CatalogUtil.newTableMeta("TEXT", new KeyValueSet());
+
+ createExternalTableIncludedOnePartitionKeyColumn(fs, rootDir, schema, meta);
+ createExternalTableIncludedMultiplePartitionKeyColumns(fs, rootDir, schema, meta);
+ }
+
+ private static void createExternalTableIncludedOnePartitionKeyColumn(FileSystem fs, Path rootDir, Schema schema,
+ TableMeta meta) throws Exception {
+ Schema partSchema = new Schema();
+ partSchema.addColumn("key", TajoDataTypes.Type.TEXT);
+
+ PartitionMethodDesc partitionMethodDesc =
+ new PartitionMethodDesc("TestPartitionedTableRewriter", PARTITION_TABLE_NAME,
+ CatalogProtos.PartitionType.COLUMN, "key", partSchema);
+
+ Path tablePath = new Path(rootDir, PARTITION_TABLE_NAME);
+ fs.mkdirs(tablePath);
+
+ client.createExternalTable(PARTITION_TABLE_NAME, schema, tablePath.toUri(), meta, partitionMethodDesc);
+
+ TableDesc tableDesc = client.getTableDesc(PARTITION_TABLE_NAME);
+ assertNotNull(tableDesc);
+
+ Path path = new Path(tableDesc.getUri().toString() + "/key=part123");
+ fs.mkdirs(path);
+ FileUtil.writeTextToFile("1|ARGENTINA|1", new Path(path, "data"));
+
+ path = new Path(tableDesc.getUri().toString() + "/key=part456");
+ fs.mkdirs(path);
+ FileUtil.writeTextToFile("2|BRAZIL|1", new Path(path, "data"));
+
+ path = new Path(tableDesc.getUri().toString() + "/key=part789");
+ fs.mkdirs(path);
+ FileUtil.writeTextToFile("3|CANADA|1", new Path(path, "data"));
+ }
+
+ private static void createExternalTableIncludedMultiplePartitionKeyColumns(FileSystem fs, Path rootDir,
+ Schema schema, TableMeta meta) throws Exception {
+ Schema partSchema = new Schema();
+ partSchema.addColumn("key1", TajoDataTypes.Type.TEXT);
+ partSchema.addColumn("key2", TajoDataTypes.Type.TEXT);
+ partSchema.addColumn("key3", TajoDataTypes.Type.INT8);
+
+ PartitionMethodDesc partitionMethodDesc =
+ new PartitionMethodDesc("TestPartitionedTableRewriter", MULTIPLE_PARTITION_TABLE_NAME,
+ CatalogProtos.PartitionType.COLUMN, "key1,key2,key3", partSchema);
+
+ Path tablePath = new Path(rootDir, MULTIPLE_PARTITION_TABLE_NAME);
+ fs.mkdirs(tablePath);
+
+ client.createExternalTable(MULTIPLE_PARTITION_TABLE_NAME, schema, tablePath.toUri(), meta, partitionMethodDesc);
+
+ TableDesc tableDesc = client.getTableDesc(MULTIPLE_PARTITION_TABLE_NAME);
+ assertNotNull(tableDesc);
+
+ Path path = new Path(tableDesc.getUri().toString() + "/key1=part123");
+ fs.mkdirs(path);
+ path = new Path(tableDesc.getUri().toString() + "/key1=part123/key2=supp123");
+ fs.mkdirs(path);
+ path = new Path(tableDesc.getUri().toString() + "/key1=part123/key2=supp123/key3=1");
+ fs.mkdirs(path);
+ FileUtil.writeTextToFile("1|ARGENTINA|1", new Path(path, "data"));
+
+ path = new Path(tableDesc.getUri().toString() + "/key1=part123/key2=supp123/key3=2");
+ fs.mkdirs(path);
+ FileUtil.writeTextToFile("2|BRAZIL|1", new Path(path, "data"));
+
+ path = new Path(tableDesc.getUri().toString() + "/key1=part789");
+ fs.mkdirs(path);
+ path = new Path(tableDesc.getUri().toString() + "/key1=part789/key2=supp789");
+ fs.mkdirs(path);
+ path = new Path(tableDesc.getUri().toString() + "/key1=part789/key2=supp789/key3=3");
+ fs.mkdirs(path);
+ FileUtil.writeTextToFile("3|CANADA|1", new Path(path, "data"));
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ client.executeQuery("DROP TABLE IF EXISTS " + PARTITION_TABLE_NAME + " PURGE;");
+ client.executeQuery("DROP TABLE IF EXISTS " + MULTIPLE_PARTITION_TABLE_NAME + " PURGE;");
+ }
+
+ @Test
+ public void testFilterIncludePartitionKeyColumn() throws Exception {
+ Expr expr = sqlParser.parse("SELECT * FROM " + PARTITION_TABLE_NAME + " WHERE key = 'part456' ORDER BY key");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SORT, projNode.getChild().getType());
+ SortNode sortNode = projNode.getChild();
+
+ assertEquals(NodeType.SELECTION, sortNode.getChild().getType());
+ SelectionNode selNode = sortNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(1, filteredPaths.length);
+ assertEquals("key=part456", filteredPaths[0].getName());
+ }
+
+ @Test
+ public void testWithoutAnyFilters() throws Exception {
+ Expr expr = sqlParser.parse("SELECT * FROM " + PARTITION_TABLE_NAME + " ORDER BY key");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SORT, projNode.getChild().getType());
+ SortNode sortNode = projNode.getChild();
+
+ assertEquals(NodeType.SCAN, sortNode.getChild().getType());
+ ScanNode scanNode = sortNode.getChild();
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(3, filteredPaths.length);
+ assertEquals("key=part123", filteredPaths[0].getName());
+ assertEquals("key=part456", filteredPaths[1].getName());
+ assertEquals("key=part789", filteredPaths[2].getName());
+ }
+
+ @Test
+ public void testFilterIncludeNonExistingPartitionValue() throws Exception {
+ Expr expr = sqlParser.parse("SELECT * FROM " + PARTITION_TABLE_NAME + " WHERE key = 'part123456789'");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SELECTION, projNode.getChild().getType());
+ SelectionNode selNode = projNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(0, filteredPaths.length);
+ }
+
+ @Test
+ public void testFilterIncludeNonPartitionKeyColumn() throws Exception {
+ String sql = "SELECT * FROM " + PARTITION_TABLE_NAME + " WHERE n_nationkey = 1";
+ Expr expr = sqlParser.parse(sql);
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SELECTION, projNode.getChild().getType());
+ SelectionNode selNode = projNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(3, filteredPaths.length);
+ assertEquals("key=part123", filteredPaths[0].getName());
+ assertEquals("key=part456", filteredPaths[1].getName());
+ assertEquals("key=part789", filteredPaths[2].getName());
+ }
+
+ @Test
+ public void testFilterIncludeEveryPartitionKeyColumn() throws Exception {
+ Expr expr = sqlParser.parse("SELECT * FROM " + MULTIPLE_PARTITION_TABLE_NAME
+ + " WHERE key1 = 'part789' and key2 = 'supp789' and key3=3");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SELECTION, projNode.getChild().getType());
+ SelectionNode selNode = projNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(1, filteredPaths.length);
+ assertEquals("key3=3", filteredPaths[0].getName());
+ assertEquals("key2=supp789", filteredPaths[0].getParent().getName());
+ assertEquals("key1=part789", filteredPaths[0].getParent().getParent().getName());
+ }
+
+ @Test
+ public void testFilterIncludeSomeOfPartitionKeyColumns() throws Exception {
+ Expr expr = sqlParser.parse("SELECT * FROM " + MULTIPLE_PARTITION_TABLE_NAME
+ + " WHERE key1 = 'part123' and key2 = 'supp123' order by n_nationkey");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SORT, projNode.getChild().getType());
+ SortNode sortNode = projNode.getChild();
+
+ assertEquals(NodeType.SELECTION, sortNode.getChild().getType());
+ SelectionNode selNode = sortNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(2, filteredPaths.length);
+
+ assertEquals("key3=1", filteredPaths[0].getName());
+ assertEquals("key2=supp123", filteredPaths[0].getParent().getName());
+ assertEquals("key1=part123", filteredPaths[0].getParent().getParent().getName());
+
+ assertEquals("key3=2", filteredPaths[1].getName());
+ assertEquals("key2=supp123", filteredPaths[1].getParent().getName());
+ assertEquals("key1=part123", filteredPaths[1].getParent().getParent().getName());
+ }
+
+ @Test
+ public void testFilterIncludeNonPartitionKeyColumns() throws Exception {
+ Expr expr = sqlParser.parse("SELECT * FROM " + MULTIPLE_PARTITION_TABLE_NAME
+ + " WHERE key1 = 'part123' and n_nationkey >= 2 order by n_nationkey");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SORT, projNode.getChild().getType());
+ SortNode sortNode = projNode.getChild();
+
+ assertEquals(NodeType.SELECTION, sortNode.getChild().getType());
+ SelectionNode selNode = sortNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(2, filteredPaths.length);
+
+ assertEquals("key3=1", filteredPaths[0].getName());
+ assertEquals("key2=supp123", filteredPaths[0].getParent().getName());
+ assertEquals("key1=part123", filteredPaths[0].getParent().getParent().getName());
+
+ assertEquals("key3=2", filteredPaths[1].getName());
+ assertEquals("key2=supp123", filteredPaths[1].getParent().getName());
+ assertEquals("key1=part123", filteredPaths[1].getParent().getParent().getName());
+ }
+
+ @Test
+ public final void testPartitionPruningWitCTAS() throws Exception {
+ String tableName = "testPartitionPruningUsingDirectories".toLowerCase();
+ String canonicalTableName = CatalogUtil.getCanonicalTableName("\"TestPartitionedTableRewriter\"", tableName);
+
+ executeString(
+ "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) "
+ + " as select l_orderkey, l_partkey, l_quantity from default.lineitem");
+
+ TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), tableName);
+ assertNotNull(tableDesc);
+
+ // With a filter which checks a partition key column
+ Expr expr = sqlParser.parse("SELECT * FROM " + canonicalTableName + " WHERE key <= 40.0 ORDER BY key");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SORT, projNode.getChild().getType());
+ SortNode sortNode = projNode.getChild();
+
+ assertEquals(NodeType.SELECTION, sortNode.getChild().getType());
+ SelectionNode selNode = sortNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(3, filteredPaths.length);
+ assertEquals("key=17.0", filteredPaths[0].getName());
+ assertEquals("key=36.0", filteredPaths[1].getName());
+ assertEquals("key=38.0", filteredPaths[2].getName());
+
+ executeString("DROP TABLE " + canonicalTableName + " PURGE").close();
+ }
+}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java
index 4c9f367..bb7f2b9 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java
@@ -105,7 +105,7 @@
executeDDL("alter_table_add_partition1.sql", null);
executeDDL("alter_table_add_partition2.sql", null);
- List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table");
+ List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions("TestAlterTable", "partitioned_table");
assertNotNull(partitions);
assertEquals(partitions.size(), 1);
assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2");
@@ -123,7 +123,7 @@
executeDDL("alter_table_drop_partition1.sql", null);
executeDDL("alter_table_drop_partition2.sql", null);
- partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table");
+ partitions = catalog.getPartitions("TestAlterTable", "partitioned_table");
assertNotNull(partitions);
assertEquals(partitions.size(), 0);
assertFalse(fs.exists(partitionPath));
@@ -478,12 +478,6 @@
verifyPartitionCount(databaseName, tableName, 5);
- // Check the volume of partition
- List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(databaseName, tableName);
- for (CatalogProtos.PartitionDescProto eachPartition : partitions) {
- assertTrue(eachPartition.getNumBytes() > 0L);
- }
-
// Remove all partitions
dropPartitions(databaseName, tableName, tableDesc.getPartitionMethod().getExpressionSchema().getAllColumns());
@@ -505,7 +499,7 @@
private void verifyPartitionCount(String databaseName, String tableName, int expectedCount)
throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
UndefinedPartitionException {
- List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(databaseName, tableName);
+ List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions(databaseName, tableName);
assertNotNull(partitions);
assertEquals(partitions.size(), expectedCount);
}
@@ -513,7 +507,7 @@
private void dropPartitions(String databaseName, String tableName, List<Column> colums)
throws Exception {
String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName);
- List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(databaseName, tableName);
+ List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions(databaseName, tableName);
StringBuilder sb = new StringBuilder();
for (CatalogProtos.PartitionDescProto partition : partitions) {
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
index 6081d78..569d2a7 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
@@ -433,14 +433,6 @@
assertEquals(resultRows2.get(res.getDouble(4))[1], res.getInt(3));
}
- res = executeString("select * from " + tableName + " WHERE (col1 ='1' or col1 = '100') and col3 > 20");
- String result = resultSetToString(res);
- String expectedResult = "col4,col1,col2,col3\n" +
- "-------------------------------\n" +
- "N,1,1,36.0\n";
- res.close();
- assertEquals(expectedResult, result);
-
res = executeString("SELECT col1, col2, col3 FROM " + tableName);
res.close();
@@ -1249,8 +1241,6 @@
private void verifyPartitionDirectoryFromCatalog(String databaseName, String tableName,
String[] partitionColumns, Long numRows) throws Exception {
int rowCount = 0;
- FileSystem fs = FileSystem.get(conf);
- Path partitionPath = null;
// Get all partition column values
StringBuilder query = new StringBuilder();
@@ -1262,7 +1252,7 @@
}
query.append(" ").append(partitionColumn);
}
- query.append(" FROM ").append(databaseName).append(".").append(tableName);
+ query.append(" FROM ").append(tableName);
ResultSet res = executeString(query.toString());
StringBuilder partitionName = new StringBuilder();
@@ -1283,10 +1273,6 @@
assertNotNull(partitionDescProto);
assertTrue(partitionDescProto.getPath().indexOf(tableName + "/" + partitionName.toString()) > 0);
- partitionPath = new Path(partitionDescProto.getPath());
- ContentSummary cs = fs.getContentSummary(partitionPath);
-
- assertEquals(cs.getLength(), partitionDescProto.getNumBytes());
rowCount++;
}
@@ -1326,7 +1312,7 @@
// If duplicated partitions had been removed, partitions just will contain 'KEY=N' partition and 'KEY=R'
// partition. In previous Query and Stage, duplicated partitions were not deleted because they had been in List.
// If you want to verify duplicated partitions, you need to use List instead of Set with DerbyStore.
- List<PartitionDescProto> partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, tableName);
+ List<PartitionDescProto> partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, tableName);
assertEquals(2, partitions.size());
PartitionDescProto firstPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=N");
@@ -1338,503 +1324,4 @@
executeString("DROP TABLE " + tableName + " PURGE");
}
}
-
- @Test
- public final void testPatternMatchingPredicatesAndStringFunctions() throws Exception {
- ResultSet res = null;
- String tableName = CatalogUtil.normalizeIdentifier("testPatternMatchingPredicatesAndStringFunctions");
- String expectedResult;
-
- if (nodeType == NodeType.INSERT) {
- executeString("create table " + tableName
- + " (col1 int4, col2 int4) partition by column(l_shipdate text, l_returnflag text) ").close();
-
- assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
- assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size());
- assertEquals(4, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
-
- executeString(
- "insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_shipdate, l_returnflag from lineitem");
- } else {
- executeString(
- "create table " + tableName + "(col1 int4, col2 int4) partition by column(l_shipdate text, l_returnflag text) "
- + " as select l_orderkey, l_partkey, l_shipdate, l_returnflag from lineitem");
- }
-
- assertTrue(client.existTable(tableName));
-
- // Like
- res = executeString("SELECT * FROM " + tableName
- + " WHERE l_shipdate LIKE '1996%' and l_returnflag = 'N' order by l_shipdate");
-
- expectedResult = "col1,col2,l_shipdate,l_returnflag\n" +
- "-------------------------------\n" +
- "1,1,1996-03-13,N\n" +
- "1,1,1996-04-12,N\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Not like
- res = executeString("SELECT * FROM " + tableName
- + " WHERE l_shipdate NOT LIKE '1996%' and l_returnflag IN ('R') order by l_shipdate");
-
- expectedResult = "col1,col2,l_shipdate,l_returnflag\n" +
- "-------------------------------\n" +
- "3,3,1993-11-09,R\n" +
- "3,2,1994-02-02,R\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // In
- res = executeString("SELECT * FROM " + tableName
- + " WHERE l_shipdate IN ('1993-11-09', '1994-02-02', '1997-01-28') AND l_returnflag = 'R' order by l_shipdate");
-
- expectedResult = "col1,col2,l_shipdate,l_returnflag\n" +
- "-------------------------------\n" +
- "3,3,1993-11-09,R\n" +
- "3,2,1994-02-02,R\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Similar to
- res = executeString("SELECT * FROM " + tableName + " WHERE l_shipdate similar to '1993%' order by l_shipdate");
-
- expectedResult = "col1,col2,l_shipdate,l_returnflag\n" +
- "-------------------------------\n" +
- "3,3,1993-11-09,R\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Regular expression
- res = executeString("SELECT * FROM " + tableName
- + " WHERE l_shipdate regexp '[1-2][0-9][0-9][3-9]-[0-1][0-9]-[0-3][0-9]' "
- + " AND l_returnflag <> 'N' ORDER BY l_shipdate");
-
- expectedResult = "col1,col2,l_shipdate,l_returnflag\n" +
- "-------------------------------\n" +
- "3,3,1993-11-09,R\n" +
- "3,2,1994-02-02,R\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Concatenate
- res = executeString("SELECT * FROM " + tableName
- + " WHERE l_shipdate = ( '1996' || '-' || '03' || '-' || '13' ) order by l_shipdate");
-
- expectedResult = "col1,col2,l_shipdate,l_returnflag\n" +
- "-------------------------------\n" +
- "1,1,1996-03-13,N\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- executeString("DROP TABLE " + tableName + " PURGE").close();
- res.close();
- }
-
- @Test
- public final void testDatePartitionColumn() throws Exception {
- ResultSet res = null;
- String tableName = CatalogUtil.normalizeIdentifier("testDatePartitionColumn");
- String expectedResult;
-
- if (nodeType == NodeType.INSERT) {
- executeString("create table " + tableName + " (col1 int4, col2 int4) partition by column(key date) ").close();
-
- assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
- assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size());
- assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
-
- executeString(
- "insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_shipdate from lineitem");
- } else {
- executeString(
- "create table " + tableName + "(col1 int4, col2 int4) partition by column(key date) "
- + " as select l_orderkey, l_partkey, l_shipdate::date from lineitem");
- }
-
- assertTrue(client.existTable(tableName));
-
- // LessThanOrEquals
- res = executeString("SELECT * FROM " + tableName + " WHERE key <= date '1995-09-01' order by col1, col2, key");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "3,2,1994-02-02\n" +
- "3,3,1993-11-09\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // LessThan and GreaterThan
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key > to_date('1993-01-01', 'YYYY-MM-DD') " +
- " and key < to_date('1996-01-01', 'YYYY-MM-DD') order by col1, col2, key desc");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "3,2,1994-02-02\n" +
- "3,3,1993-11-09\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Between
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key between date '1993-01-01' and date '1997-01-01' order by col1, col2, key desc");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,1996-04-12\n" +
- "1,1,1996-03-13\n" +
- "3,2,1994-02-02\n" +
- "3,3,1993-11-09\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Cast
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key > '1993-01-01'::date " +
- " and key < '1997-01-01'::timestamp order by col1, col2, key ");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,1996-03-13\n" +
- "1,1,1996-04-12\n" +
- "3,2,1994-02-02\n" +
- "3,3,1993-11-09\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Interval
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key > '1993-01-01'::date " +
- " and key < date '1994-01-01' + interval '1 year' order by col1, col2, key ");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "3,2,1994-02-02\n" +
- "3,3,1993-11-09\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // DateTime Function #1
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key > '1993-01-01'::date " +
- " and key < add_months(date '1994-01-01', 12) order by col1, col2, key ");
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // DateTime Function #2
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key > '1993-01-01'::date " +
- " and key < add_months('1994-01-01'::timestamp, 12) order by col1, col2, key ");
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- executeString("DROP TABLE " + tableName + " PURGE").close();
- res.close();
- }
-
- @Test
- public final void testTimestampPartitionColumn() throws Exception {
- ResultSet res = null;
- String tableName = CatalogUtil.normalizeIdentifier("testTimestampPartitionColumn");
- String expectedResult;
-
- if (nodeType == NodeType.INSERT) {
- executeString("create table " + tableName
- + " (col1 int4, col2 int4) partition by column(key timestamp) ").close();
-
- assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
- assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size());
- assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
-
- executeString(
- "insert overwrite into " + tableName
- + " select l_orderkey, l_partkey, to_timestamp(l_shipdate, 'YYYY-MM-DD') from lineitem");
- } else {
- executeString(
- "create table " + tableName + "(col1 int4, col2 int4) partition by column(key timestamp) "
- + " as select l_orderkey, l_partkey, to_timestamp(l_shipdate, 'YYYY-MM-DD') from lineitem");
- }
-
- assertTrue(client.existTable(tableName));
-
- // LessThanOrEquals
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key <= to_timestamp('1995-09-01', 'YYYY-MM-DD') order by col1, col2, key");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "3,2,1994-02-02 00:00:00\n" +
- "3,3,1993-11-09 00:00:00\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // LessThan and GreaterThan
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key > to_timestamp('1993-01-01', 'YYYY-MM-DD') and " +
- "key < to_timestamp('1996-01-01', 'YYYY-MM-DD') order by col1, col2, key desc");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "3,2,1994-02-02 00:00:00\n" +
- "3,3,1993-11-09 00:00:00\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Between
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key between to_timestamp('1993-01-01', 'YYYY-MM-DD') " +
- "and to_timestamp('1997-01-01', 'YYYY-MM-DD') order by col1, col2, key desc");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,1996-04-12 00:00:00\n" +
- "1,1,1996-03-13 00:00:00\n" +
- "3,2,1994-02-02 00:00:00\n" +
- "3,3,1993-11-09 00:00:00\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- executeString("DROP TABLE " + tableName + " PURGE").close();
- res.close();
- }
-
- @Test
- public final void testTimePartitionColumn() throws Exception {
- ResultSet res = null;
- String tableName = CatalogUtil.normalizeIdentifier("testTimePartitionColumn");
- String expectedResult;
-
- if (nodeType == NodeType.INSERT) {
- executeString("create table " + tableName
- + " (col1 int4, col2 int4) partition by column(key time) ").close();
-
- assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
- assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size());
- assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
-
- executeString(
- "insert overwrite into " + tableName
- + " select l_orderkey, l_partkey " +
- " , CASE l_shipdate WHEN '1996-03-13' THEN cast ('11:20:40' as time) " +
- " WHEN '1997-01-28' THEN cast ('12:10:20' as time) " +
- " WHEN '1994-02-02' THEN cast ('12:10:30' as time) " +
- " ELSE cast ('00:00:00' as time) END " +
- " from lineitem");
- } else {
- executeString(
- "create table " + tableName + "(col1 int4, col2 int4) partition by column(key time) "
- + " as select l_orderkey, l_partkey " +
- " , CASE l_shipdate WHEN '1996-03-13' THEN cast ('11:20:40' as time) " +
- " WHEN '1997-01-28' THEN cast ('12:10:20' as time) " +
- " WHEN '1994-02-02' THEN cast ('12:10:30' as time) " +
- " ELSE cast ('00:00:00' as time) END " +
- " from lineitem");
- }
-
- assertTrue(client.existTable(tableName));
- // LessThanOrEquals
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key <= cast('12:10:20' as time) order by col1, col2, key");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,00:00:00\n" +
- "1,1,11:20:40\n" +
- "2,2,12:10:20\n" +
- "3,3,00:00:00\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // LessThan and GreaterThan
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key > cast('00:00:00' as time) and " +
- "key < cast('12:10:00' as time) order by col1, col2, key desc");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,11:20:40\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- // Between
- res = executeString("SELECT * FROM " + tableName
- + " WHERE key between cast('11:00:00' as time) " +
- "and cast('13:00:00' as time) order by col1, col2, key desc");
-
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,11:20:40\n" +
- "2,2,12:10:20\n" +
- "3,2,12:10:30\n";
-
- assertEquals(expectedResult, resultSetToString(res));
- res.close();
-
- executeString("DROP TABLE " + tableName + " PURGE").close();
- res.close();
- }
-
- @Test
- public final void testDatabaseNameIncludeTableName() throws Exception {
- executeString("create database test_partition").close();
-
- String databaseName = "test_partition";
- String tableName = CatalogUtil.normalizeIdentifier("part");
-
- if (nodeType == NodeType.INSERT) {
- executeString(
- "create table " + databaseName + "." + tableName + " (col1 int4, col2 int4) partition by column(key float8) ");
-
- assertTrue(catalog.existsTable(databaseName, tableName));
- assertEquals(2, catalog.getTableDesc(databaseName, tableName).getSchema().size());
- assertEquals(3, catalog.getTableDesc(databaseName, tableName).getLogicalSchema().size());
-
- executeString(
- "insert overwrite into " + databaseName + "." + tableName + " select l_orderkey, l_partkey, " +
- "l_quantity from lineitem");
- } else {
- executeString(
- "create table "+ databaseName + "." + tableName + "(col1 int4, col2 int4) partition by column(key float8) "
- + " as select l_orderkey, l_partkey, l_quantity from lineitem");
- }
-
- TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName);
- verifyPartitionDirectoryFromCatalog(databaseName, tableName, new String[]{"key"},
- tableDesc.getStats().getNumRows());
-
- ResultSet res = executeString("select * from " + databaseName + "." + tableName + " ORDER BY key");
-
- String result = resultSetToString(res);
- String expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,17.0\n" +
- "1,1,36.0\n" +
- "2,2,38.0\n" +
- "3,2,45.0\n" +
- "3,3,49.0\n";
- res.close();
- assertEquals(expectedResult, result);
-
- executeString("DROP TABLE " + databaseName + "." + tableName + " PURGE").close();
- executeString("DROP database " + databaseName).close();
- }
-
- @Test
- public void testAbnormalDirectories() throws Exception {
- ResultSet res = null;
- FileSystem fs = FileSystem.get(conf);
- Path path = null;
-
- String tableName = CatalogUtil.normalizeIdentifier("testAbnormalDirectories");
- if (nodeType == NodeType.INSERT) {
- executeString(
- "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ").close();
- executeString(
- "insert overwrite into " + tableName + " select l_orderkey, l_partkey, " +
- "l_quantity from lineitem").close();
- } else {
- executeString(
- "create table " + tableName + "(col1 int4, col2 int4) partition by column(key float8) "
- + " as select l_orderkey, l_partkey, l_quantity from lineitem").close();
- }
-
- TableDesc tableDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
-
- verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"},
- tableDesc.getStats().getNumRows());
-
- // When partitions only exist on file system without catalog.
- String externalTableName = "testCreateExternalColumnPartitionedTable";
-
- executeString("create external table " + externalTableName + " (col1 int4, col2 int4) " +
- " USING TEXT WITH ('text.delimiter'='|') PARTITION BY COLUMN (key float8) " +
- " location '" + tableDesc.getUri().getPath() + "'").close();
-
- res = executeString("SELECT COUNT(*) AS cnt FROM " + externalTableName);
- String result = resultSetToString(res);
- String expectedResult = "cnt\n" +
- "-------------------------------\n" +
- "5\n";
- res.close();
- assertEquals(expectedResult, result);
-
- // Make abnormal directories
- path = new Path(tableDesc.getUri().getPath(), "key=100.0");
- fs.mkdirs(path);
- path = new Path(tableDesc.getUri().getPath(), "key=");
- fs.mkdirs(path);
- path = new Path(tableDesc.getUri().getPath(), "col1=a");
- fs.mkdirs(path);
- assertEquals(8, fs.listStatus(path.getParent()).length);
-
- res = executeString("SELECT COUNT(*) AS cnt FROM " + externalTableName + " WHERE key > 40.0");
- result = resultSetToString(res);
- expectedResult = "cnt\n" +
- "-------------------------------\n" +
- "2\n";
- res.close();
- assertEquals(expectedResult, result);
-
- // Remove existing partition directory
- path = new Path(tableDesc.getUri().getPath(), "key=36.0");
- fs.delete(path, true);
-
- res = executeString("SELECT * FROM " + tableName + " ORDER BY key");
- result = resultSetToString(res);
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,17.0\n" +
- "2,2,38.0\n" +
- "3,2,45.0\n" +
- "3,3,49.0\n";
- res.close();
- assertEquals(expectedResult, result);
-
- res = executeString("SELECT COUNT(*) AS cnt FROM " + tableName + " WHERE key > 30.0");
- result = resultSetToString(res);
- expectedResult = "cnt\n" +
- "-------------------------------\n" +
- "3\n";
- res.close();
- assertEquals(expectedResult, result);
-
- // Sort
- String sortedTableName = "sortedPartitionTable";
- executeString("create table " + sortedTableName + " AS SELECT * FROM " + tableName
- + " ORDER BY col1, col2 desc").close();
-
- res = executeString("SELECT * FROM " + sortedTableName + " ORDER BY col1, col2 desc;");
- result = resultSetToString(res);
- expectedResult = "col1,col2,key\n" +
- "-------------------------------\n" +
- "1,1,17.0\n" +
- "2,2,38.0\n" +
- "3,3,49.0\n" +
- "3,2,45.0\n";
- res.close();
- assertEquals(expectedResult, result);
-
- executeString("DROP TABLE " + sortedTableName + " PURGE").close();
- executeString("DROP TABLE " + externalTableName).close();
- executeString("DROP TABLE " + tableName + " PURGE").close();
- }
}
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
index 677b5f2..93a80a1 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
@@ -16,8 +16,9 @@
--
-- Table Partitions: TableName3
+-- Partition dump and restore are not supported yet
--
-ALTER TABLE "TestTajoDump"."TableName3" REPAIR PARTITION;
+
@@ -28,5 +29,5 @@
--
-- Table Partitions: TableName4
---
-ALTER TABLE "TestTajoDump"."TableName4" REPAIR PARTITION;
\ No newline at end of file
+-- Partition dump and restore are not supported yet
+--
\ No newline at end of file
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 4b96767..59059c8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -974,7 +974,7 @@
tableName = request.getValue();
}
- List<PartitionDescProto> partitions = catalog.getPartitionsOfTable(databaseName, tableName);
+ List<PartitionDescProto> partitions = catalog.getPartitions(databaseName, tableName);
return PartitionListResponse.newBuilder()
.setState(OK)
.addAllPartition(partitions)
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
index a604f94..086060e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
@@ -20,7 +20,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -510,14 +509,8 @@
throw new AmbiguousPartitionDirectoryExistException(assumedDirectory.toString());
}
- long numBytes = 0L;
- if (fs.exists(partitionPath)) {
- ContentSummary summary = fs.getContentSummary(partitionPath);
- numBytes = summary.getLength();
- }
-
catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(),
- alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION, numBytes));
+ alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION));
// If the partition's path doesn't exist, this would make the directory by force.
if (!fs.exists(partitionPath)) {
@@ -603,15 +596,15 @@
PathFilter[] filters = PartitionedTableRewriter.buildAllAcceptingPathFilters(partitionColumns);
// loop from one to the number of partition columns
- Path [] filteredPaths = toPathArray(fs.listStatus(tablePath, filters[0]));
+ Path [] filteredPaths = PartitionedTableRewriter.toPathArray(fs.listStatus(tablePath, filters[0]));
// Get all file status matched to a ith level path filter.
for (int i = 1; i < partitionColumns.size(); i++) {
- filteredPaths = toPathArray(fs.listStatus(filteredPaths, filters[i]));
+ filteredPaths = PartitionedTableRewriter.toPathArray(fs.listStatus(filteredPaths, filters[i]));
}
// Find missing partitions from filesystem
- List<PartitionDescProto> existingPartitions = catalog.getPartitionsOfTable(databaseName, simpleTableName);
+ List<PartitionDescProto> existingPartitions = catalog.getPartitions(databaseName, simpleTableName);
List<String> existingPartitionNames = TUtil.newList();
Path existingPartitionPath = null;
@@ -682,9 +675,6 @@
builder.setPath(partitionPath.toString());
- ContentSummary contentSummary = fs.getContentSummary(partitionPath);
- builder.setNumBytes(contentSummary.getLength());
-
return builder.build();
}
@@ -721,13 +711,4 @@
final TableDesc tableDesc = catalog.getTableDesc(tableName);
return tableDesc.getSchema().containsByName(columnName);
}
-
- private Path [] toPathArray(FileStatus[] fileStatuses) {
- Path [] paths = new Path[fileStatuses.length];
- for (int i = 0; i < fileStatuses.length; i++) {
- FileStatus fileStatus = fileStatuses[i];
- paths[i] = fileStatus.getPath();
- }
- return paths;
- }
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
index 44be3b0..f06d28c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
@@ -24,7 +24,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -510,11 +509,8 @@
if (queryContext.hasOutputTableUri() && queryContext.hasPartition()) {
List<PartitionDescProto> partitions = query.getPartitions();
if (partitions != null) {
- // Set contents length and file count to PartitionDescProto by listing final output directories.
- List<PartitionDescProto> finalPartitions = getPartitionsWithContentsSummary(query.systemConf,
- finalOutputDir, partitions);
-
String databaseName, simpleTableName;
+
if (CatalogUtil.isFQTableName(tableDesc.getName())) {
String[] split = CatalogUtil.splitFQTableName(tableDesc.getName());
databaseName = split[0];
@@ -525,7 +521,7 @@
}
// Store partitions to CatalogStore using alter table statement.
- catalog.addPartitions(databaseName, simpleTableName, finalPartitions, true);
+ catalog.addPartitions(databaseName, simpleTableName, partitions, true);
LOG.info("Added partitions to catalog (total=" + partitions.size() + ")");
} else {
LOG.info("Can't find partitions for adding.");
@@ -540,21 +536,6 @@
return QueryState.QUERY_SUCCEEDED;
}
- private List<PartitionDescProto> getPartitionsWithContentsSummary(TajoConf conf, Path outputDir,
- List<PartitionDescProto> partitions) throws IOException {
- List<PartitionDescProto> finalPartitions = TUtil.newList();
-
- FileSystem fileSystem = outputDir.getFileSystem(conf);
- for (PartitionDescProto partition : partitions) {
- PartitionDescProto.Builder builder = partition.toBuilder();
- Path partitionPath = new Path(outputDir, partition.getPath());
- ContentSummary contentSummary = fileSystem.getContentSummary(partitionPath);
- builder.setNumBytes(contentSummary.getLength());
- finalPartitions.add(builder.build());
- }
- return finalPartitions;
- }
-
private static interface QueryHook {
boolean isEligible(QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir);
void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query,
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
index 1303540..19d5d16 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
@@ -18,19 +18,15 @@
package org.apache.tajo.plan.expr;
-import com.google.common.base.Preconditions;
import org.apache.tajo.algebra.*;
import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.plan.util.ExprFinder;
import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
-import org.apache.tajo.util.TUtil;
import java.util.*;
public class AlgebraicUtil {
-
+
/**
* Transpose a given comparison expression into the expression
* where the variable corresponding to the target is placed
@@ -497,146 +493,4 @@
return super.visitTimeLiteral(ctx, stack, expr);
}
}
-
- /**
- * Find the top expr matched to type from the given expr
- *
- * @param expr start expr
- * @param type to find
- * @return a found expr
- */
- public static <T extends Expr> T findTopExpr(Expr expr, OpType type) throws TajoException {
- Preconditions.checkNotNull(expr);
- Preconditions.checkNotNull(type);
-
- List<Expr> exprs = ExprFinder.findsInOrder(expr, type);
- if (exprs.size() == 0) {
- return null;
- } else {
- return (T) exprs.get(0);
- }
- }
-
- /**
- * Find the most bottom expr matched to type from the given expr
- *
- * @param expr start expr
- * @param type to find
- * @return a found expr
- */
- public static <T extends Expr> T findMostBottomExpr(Expr expr, OpType type) throws TajoException {
- Preconditions.checkNotNull(expr);
- Preconditions.checkNotNull(type);
-
- List<Expr> exprs = ExprFinder.findsInOrder(expr, type);
- if (exprs.size() == 0) {
- return null;
- } else {
- return (T) exprs.get(exprs.size()-1);
- }
- }
-
- /**
- * Transforms an algebra expression to an array of conjunctive normal formed algebra expressions.
- *
- * @param expr The algebra expression to be transformed to an array of CNF-formed expressions.
- * @return An array of CNF-formed algebra expressions
- */
- public static Expr[] toConjunctiveNormalFormArray(Expr expr) {
- List<Expr> list = TUtil.newList();
- toConjunctiveNormalFormArrayRecursive(expr, list);
- return list.toArray(new Expr[list.size()]);
- }
-
- private static void toConjunctiveNormalFormArrayRecursive(Expr node, List<Expr> found) {
- if (node.getType() == OpType.And) {
- toConjunctiveNormalFormArrayRecursive(((BinaryOperator) node).getLeft(), found);
- toConjunctiveNormalFormArrayRecursive(((BinaryOperator) node).getRight(), found);
- } else {
- found.add(node);
- }
- }
-
- /**
- * Build Exprs for all columns with a list of filter conditions.
- *
- * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
- * Then, this methods will create three Exprs for (col1), (col2), (col3).
- *
- * Assume that an user gives a condition WHERE col1 ='A' and col3 = 'C'.
- * There is no filter condition corresponding to col2.
- * Then, the path filter conditions are corresponding to the followings:
- *
- * The first Expr: col1 = 'A'
- * The second Expr: col2 IS NOT NULL
- * The third Expr: col3 = 'C'
- *
- * 'IS NOT NULL' predicate is always true against the partition path.
- *
- *
- * @param partitionColumns
- * @param conjunctiveForms
- * @return
- */
- public static Expr[] getRearrangedCNFExpressions(String tableName,
- List<CatalogProtos.ColumnProto> partitionColumns, Expr[] conjunctiveForms) {
- Expr[] filters = new Expr[partitionColumns.size()];
- Column target;
-
- for (int i = 0; i < partitionColumns.size(); i++) {
- List<Expr> accumulatedFilters = TUtil.newList();
- target = new Column(partitionColumns.get(i));
- ColumnReferenceExpr columnReference = new ColumnReferenceExpr(tableName, target.getSimpleName());
-
- if (conjunctiveForms == null) {
- accumulatedFilters.add(new IsNullPredicate(true, columnReference));
- } else {
- for (Expr expr : conjunctiveForms) {
- Set<ColumnReferenceExpr> columnSet = ExprFinder.finds(expr, OpType.Column);
- if (columnSet.contains(columnReference)) {
- // Accumulate one qual per level
- accumulatedFilters.add(expr);
- }
- }
-
- if (accumulatedFilters.size() == 0) {
- accumulatedFilters.add(new IsNullPredicate(true, columnReference));
- }
- }
-
- Expr filterPerLevel = AlgebraicUtil.createSingletonExprFromCNFByExpr(
- accumulatedFilters.toArray(new Expr[accumulatedFilters.size()]));
- filters[i] = filterPerLevel;
- }
-
- return filters;
- }
-
- /**
- * Convert a list of conjunctive normal forms into a singleton expression.
- *
- * @param cnfExprs
- * @return The EvalNode object that merges all CNF-formed expressions.
- */
- public static Expr createSingletonExprFromCNFByExpr(Expr... cnfExprs) {
- if (cnfExprs.length == 1) {
- return cnfExprs[0];
- }
-
- return createSingletonExprFromCNFRecursiveByExpr(cnfExprs, 0);
- }
-
- private static Expr createSingletonExprFromCNFRecursiveByExpr(Expr[] exprs, int idx) {
- if (idx >= exprs.length) {
- throw new ArrayIndexOutOfBoundsException("index " + idx + " is exceeded the maximum length ("+
- exprs.length+") of EvalNode");
- }
-
- if (idx == exprs.length - 2) {
- return new BinaryOperator(OpType.And, exprs[idx], exprs[idx + 1]);
- } else {
- return new BinaryOperator(OpType.And, exprs[idx], createSingletonExprFromCNFRecursiveByExpr(exprs, idx + 1));
- }
- }
-
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
index fc0b1bb..79cdaa3 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
@@ -18,25 +18,26 @@
package org.apache.tajo.plan.rewrite.rules;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.*;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
-import org.apache.tajo.catalog.proto.CatalogProtos.PartitionsByAlgebraProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
-import org.apache.tajo.exception.*;
+import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.exception.TajoInternalError;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
-import org.apache.tajo.plan.util.EvalNodeToExprConverter;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
import org.apache.tajo.storage.Tuple;
@@ -44,12 +45,11 @@
import org.apache.tajo.util.StringUtils;
import java.io.IOException;
-import java.util.*;
+import java.util.List;
+import java.util.Set;
+import java.util.Stack;
public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
- private CatalogService catalog;
- private long totalVolume;
-
private static final Log LOG = LogFactory.getLog(PartitionedTableRewriter.class);
private static final String NAME = "Partitioned Table Rewriter";
@@ -79,7 +79,6 @@
public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException {
LogicalPlan plan = context.getPlan();
LogicalPlan.QueryBlock rootBlock = plan.getRootBlock();
- this.catalog = context.getCatalog();
rewriter.visit(context.getQueryContext(), plan, rootBlock, rootBlock.getRoot(), new Stack<LogicalNode>());
return plan;
}
@@ -110,13 +109,6 @@
}
}
- private Path [] findFilteredPaths(OverridableConf queryContext, String tableName,
- Schema partitionColumns, EvalNode [] conjunctiveForms, Path tablePath)
- throws IOException, UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UndefinedOperatorException, UnsupportedException {
- return findFilteredPaths(queryContext, tableName, partitionColumns, conjunctiveForms, tablePath, null);
- }
-
/**
* It assumes that each conjunctive form corresponds to one column.
*
@@ -127,82 +119,13 @@
* @return
* @throws IOException
*/
- private Path [] findFilteredPaths(OverridableConf queryContext, String tableName,
- Schema partitionColumns, EvalNode [] conjunctiveForms, Path tablePath, ScanNode scanNode)
- throws IOException, UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UndefinedOperatorException, UnsupportedException {
+ private Path [] findFilteredPaths(OverridableConf queryContext, Schema partitionColumns, EvalNode [] conjunctiveForms,
+ Path tablePath)
+ throws IOException {
- Path [] filteredPaths = null;
FileSystem fs = tablePath.getFileSystem(queryContext.getConf());
- String [] splits = CatalogUtil.splitFQTableName(tableName);
- List<PartitionDescProto> partitions = null;
- try {
- if (conjunctiveForms == null) {
- partitions = catalog.getPartitionsOfTable(splits[0], splits[1]);
- if (partitions.isEmpty()) {
- filteredPaths = findFilteredPathsFromFileSystem(partitionColumns, conjunctiveForms, fs, tablePath);
- } else {
- filteredPaths = findFilteredPathsByPartitionDesc(partitions);
- }
- } else {
- if (catalog.existPartitions(splits[0], splits[1])) {
- PartitionsByAlgebraProto request = getPartitionsAlgebraProto(splits[0], splits[1], conjunctiveForms);
- partitions = catalog.getPartitionsByAlgebra(request);
- filteredPaths = findFilteredPathsByPartitionDesc(partitions);
- } else {
- filteredPaths = findFilteredPathsFromFileSystem(partitionColumns, conjunctiveForms, fs, tablePath);
- }
- }
- } catch (UnsupportedException ue) {
- // Partial catalog might not allow some filter conditions. For example, HiveMetastore doesn't In statement,
- // regexp statement and so on. Above case, Tajo need to build filtered path by listing hdfs directories.
- LOG.warn(ue.getMessage());
- partitions = catalog.getPartitionsOfTable(splits[0], splits[1]);
- if (partitions.isEmpty()) {
- filteredPaths = findFilteredPathsFromFileSystem(partitionColumns, conjunctiveForms, fs, tablePath);
- } else {
- filteredPaths = findFilteredPathsByPartitionDesc(partitions);
- }
- scanNode.setQual(AlgebraicUtil.createSingletonExprFromCNF(conjunctiveForms));
- }
-
- LOG.info("Filtered directory or files: " + filteredPaths.length);
- return filteredPaths;
- }
-
- /**
- * Build list of partition path by PartitionDescProto which is generated from CatalogStore.
- *
- * @param partitions
- * @return
- */
- private Path[] findFilteredPathsByPartitionDesc(List<PartitionDescProto> partitions) {
- Path [] filteredPaths = new Path[partitions.size()];
- for (int i = 0; i < partitions.size(); i++) {
- PartitionDescProto partition = partitions.get(i);
- filteredPaths[i] = new Path(partition.getPath());
- totalVolume += partition.getNumBytes();
- }
- return filteredPaths;
- }
-
- /**
- * Build list of partition path by filtering directories in the given table path.
- *
- *
- * @param partitionColumns
- * @param conjunctiveForms
- * @param fs
- * @param tablePath
- * @return
- * @throws IOException
- */
- private Path [] findFilteredPathsFromFileSystem(Schema partitionColumns, EvalNode [] conjunctiveForms,
- FileSystem fs, Path tablePath) throws IOException{
- Path [] filteredPaths = null;
PathFilter [] filters;
-
if (conjunctiveForms == null) {
filters = buildAllAcceptingPathFilters(partitionColumns);
} else {
@@ -210,43 +133,18 @@
}
// loop from one to the number of partition columns
- filteredPaths = toPathArray(fs.listStatus(tablePath, filters[0]));
+ Path [] filteredPaths = toPathArray(fs.listStatus(tablePath, filters[0]));
for (int i = 1; i < partitionColumns.size(); i++) {
// Get all file status matched to a ith level path filter.
filteredPaths = toPathArray(fs.listStatus(filteredPaths, filters[i]));
}
+
+ LOG.info("Filtered directory or files: " + filteredPaths.length);
return filteredPaths;
}
/**
- * Build algebra expressions for querying partitions and partition keys by using EvalNodeToExprConverter.
- *
- * @param databaseName the database name
- * @param tableName the table name
- * @param conjunctiveForms EvalNode which contains filter conditions
- * @return
- */
- public static PartitionsByAlgebraProto getPartitionsAlgebraProto(
- String databaseName, String tableName, EvalNode [] conjunctiveForms) {
-
- PartitionsByAlgebraProto.Builder request = PartitionsByAlgebraProto.newBuilder();
- request.setDatabaseName(databaseName);
- request.setTableName(tableName);
-
- if (conjunctiveForms != null) {
- EvalNode evalNode = AlgebraicUtil.createSingletonExprFromCNF(conjunctiveForms);
- EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(databaseName + "." + tableName);
- convertor.visit(null, evalNode, new Stack<EvalNode>());
- request.setAlgebra(convertor.getResult().toJson());
- } else {
- request.setAlgebra("");
- }
-
- return request.build();
- }
-
- /**
* Build path filters for all levels with a list of filter conditions.
*
* For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
@@ -294,7 +192,6 @@
accumulatedFilters.toArray(new EvalNode[accumulatedFilters.size()]));
filters[i] = new PartitionPathFilter(partitionColumns, filterPerLevel);
}
-
return filters;
}
@@ -318,19 +215,16 @@
return filters;
}
- private Path [] toPathArray(FileStatus[] fileStatuses) {
+ public static Path [] toPathArray(FileStatus[] fileStatuses) {
Path [] paths = new Path[fileStatuses.length];
- for (int i = 0; i < fileStatuses.length; i++) {
- FileStatus fileStatus = fileStatuses[i];
- paths[i] = fileStatus.getPath();
- totalVolume += fileStatus.getLen();
+ for (int j = 0; j < fileStatuses.length; j++) {
+ paths[j] = fileStatuses[j].getPath();
}
return paths;
}
- public Path [] findFilteredPartitionPaths(OverridableConf queryContext, ScanNode scanNode) throws IOException,
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UndefinedOperatorException, UnsupportedException {
+ @VisibleForTesting
+ public Path [] findFilteredPartitionPaths(OverridableConf queryContext, ScanNode scanNode) throws IOException {
TableDesc table = scanNode.getTableDesc();
PartitionMethodDesc partitionDesc = scanNode.getTableDesc().getPartitionMethod();
@@ -369,10 +263,10 @@
}
if (indexablePredicateSet.size() > 0) { // There are at least one indexable predicates
- return findFilteredPaths(queryContext, table.getName(), paritionValuesSchema,
- indexablePredicateSet.toArray(new EvalNode[indexablePredicateSet.size()]), new Path(table.getUri()), scanNode);
+ return findFilteredPaths(queryContext, paritionValuesSchema,
+ indexablePredicateSet.toArray(new EvalNode[indexablePredicateSet.size()]), new Path(table.getUri()));
} else { // otherwise, we will get all partition paths.
- return findFilteredPaths(queryContext, table.getName(), paritionValuesSchema, null, new Path(table.getUri()));
+ return findFilteredPaths(queryContext, paritionValuesSchema, null, new Path(table.getUri()));
}
}
@@ -421,6 +315,24 @@
}
}
+ private void updateTableStat(OverridableConf queryContext, PartitionedTableScanNode scanNode)
+ throws TajoException {
+ if (scanNode.getInputPaths().length > 0) {
+ try {
+ FileSystem fs = scanNode.getInputPaths()[0].getFileSystem(queryContext.getConf());
+ long totalVolume = 0;
+
+ for (Path input : scanNode.getInputPaths()) {
+ ContentSummary summary = fs.getContentSummary(input);
+ totalVolume += summary.getLength();
+ }
+ scanNode.getTableDesc().getStats().setNumBytes(totalVolume);
+ } catch (Throwable e) {
+ throw new TajoInternalError(e);
+ }
+ }
+ }
+
/**
* Take a look at a column partition path. A partition path consists
* of a table path part and column values part. This method transforms
@@ -500,7 +412,7 @@
plan.addHistory("PartitionTableRewriter chooses " + filteredPaths.length + " of partitions");
PartitionedTableScanNode rewrittenScanNode = plan.createNode(PartitionedTableScanNode.class);
rewrittenScanNode.init(scanNode, filteredPaths);
- rewrittenScanNode.getTableDesc().getStats().setNumBytes(totalVolume);
+ updateTableStat(queryContext, rewrittenScanNode);
// if it is topmost node, set it as the rootnode of this block.
if (stack.empty() || block.getRoot().equals(scanNode)) {
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java
deleted file mode 100644
index c656671..0000000
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.plan.util;
-
-import org.apache.tajo.algebra.*;
-import org.apache.tajo.datum.DateDatum;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.TimeDatum;
-import org.apache.tajo.datum.TimestampDatum;
-import org.apache.tajo.plan.expr.*;
-
-import java.util.Stack;
-
-/**
- * This converts EvalNode tree to Expr tree.
- *
- */
-public class EvalNodeToExprConverter extends SimpleEvalNodeVisitor<Object> {
- private Stack<Expr> exprs = new Stack<Expr>();
-
- private String tableName;
-
- public EvalNodeToExprConverter(String tableName) {
- this.tableName = tableName;
- }
-
- public Expr getResult() {
- return exprs.pop();
- }
-
- @Override
- protected EvalNode visitBinaryEval(Object o, Stack<EvalNode> stack, BinaryEval binaryEval) {
- stack.push(binaryEval);
- visit(o, binaryEval.getLeftExpr(), stack);
- Expr left = exprs.pop();
-
- visit(o, binaryEval.getRightExpr(), stack);
- Expr right = exprs.pop();
-
- Expr expr = null;
- switch (binaryEval.getType()) {
- // Arithmetic expression
- case PLUS:
- expr = new BinaryOperator(OpType.Plus, left, right);
- break;
- case MINUS:
- expr = new BinaryOperator(OpType.Minus, left, right);
- break;
- case MULTIPLY:
- expr = new BinaryOperator(OpType.Multiply, left, right);
- break;
- case DIVIDE:
- expr = new BinaryOperator(OpType.Divide, left, right);
- break;
- case MODULAR:
- expr = new BinaryOperator(OpType.Modular, left, right);
- break;
-
- // Logical Predicates
- case AND:
- expr = new BinaryOperator(OpType.And, left, right);
- break;
- case OR:
- expr = new BinaryOperator(OpType.Or, left, right);
- break;
- case NOT:
- expr = new BinaryOperator(OpType.Not, left, right);
- break;
-
- // Comparison Predicates
- case EQUAL:
- expr = new BinaryOperator(OpType.Equals, left, right);
- break;
- case NOT_EQUAL:
- expr = new BinaryOperator(OpType.NotEquals, left, right);
- break;
- case LTH:
- expr = new BinaryOperator(OpType.LessThan, left, right);
- break;
- case LEQ:
- expr = new BinaryOperator(OpType.LessThanOrEquals, left, right);
- break;
- case GTH:
- expr = new BinaryOperator(OpType.GreaterThan, left, right);
- break;
- case GEQ:
- expr = new BinaryOperator(OpType.GreaterThanOrEquals, left, right);
- break;
-
- // SQL standard predicates
- case IS_NULL:
- expr = new BinaryOperator(OpType.IsNullPredicate, left, right);
- break;
- case CASE:
- expr = new BinaryOperator(OpType.CaseWhen, left, right);
- break;
- case IN:
- InEval inEval = (InEval) binaryEval;
- expr = new InPredicate(left, right, inEval.isNot());
- break;
-
- // String operators and Pattern match predicates
- case LIKE:
- LikePredicateEval likePredicateEval = (LikePredicateEval) binaryEval;
- expr = new PatternMatchPredicate(OpType.LikePredicate, likePredicateEval.isNot(), left, right);
- break;
- case SIMILAR_TO:
- SimilarToPredicateEval similarToPredicateEval = (SimilarToPredicateEval) binaryEval;
- expr = new PatternMatchPredicate(OpType.SimilarToPredicate, similarToPredicateEval.isNot(), left, right);
- break;
- case REGEX:
- RegexPredicateEval regexPredicateEval = (RegexPredicateEval) binaryEval;
- expr = new PatternMatchPredicate(OpType.Regexp, regexPredicateEval.isNot(), left, right);
- break;
- case CONCATENATE:
- default:
- throw new RuntimeException("Unsupported type: " + binaryEval.getType().name());
- }
-
- if (expr != null) {
- exprs.push(expr);
- }
-
- stack.pop();
- return null;
- }
-
- @Override
- protected EvalNode visitConst(Object o, ConstEval evalNode, Stack<EvalNode> stack) {
- Expr value = null;
- DateValue dateValue;
- TimeValue timeValue;
-
- switch (evalNode.getValueType().getType()) {
- case NULL_TYPE:
- value = new NullLiteral();
- break;
- case BOOLEAN:
- value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.Boolean);
- break;
- case INT1:
- case INT2:
- case INT4:
- value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.Unsigned_Integer);
- break;
- case INT8:
- value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.Unsigned_Large_Integer);
- break;
- case FLOAT4:
- case FLOAT8:
- value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.Unsigned_Float);
- break;
- case TEXT:
- value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.String);
- break;
- case DATE:
- DateDatum dateDatum = (DateDatum) evalNode.getValue();
-
- dateValue = new DateValue(""+dateDatum.getYear(),
- ""+dateDatum.getMonthOfYear(), ""+dateDatum.getDayOfMonth());
- value = new DateLiteral(dateValue);
-
- break;
- case TIMESTAMP:
- TimestampDatum timestampDatum = (TimestampDatum) evalNode.getValue();
-
- dateValue = new DateValue(""+timestampDatum.getYear(),
- ""+timestampDatum.getMonthOfYear(), ""+timestampDatum.getDayOfMonth());
-
- timeValue = new TimeValue(""+timestampDatum.getHourOfDay()
- , ""+timestampDatum.getMinuteOfHour(), ""+timestampDatum.getSecondOfMinute());
-
- value = new TimestampLiteral(dateValue, timeValue);
- break;
- case TIME:
- TimeDatum timeDatum = (TimeDatum) evalNode.getValue();
- timeValue = new TimeValue(""+timeDatum.getHourOfDay()
- , ""+timeDatum.getMinuteOfHour(), ""+timeDatum.getSecondOfMinute());
-
- value = new TimeLiteral(timeValue);
- break;
- default:
- throw new RuntimeException("Unsupported type: " + evalNode.getValueType().getType().name());
- }
- exprs.push(value);
-
- return super.visitConst(o, evalNode, stack);
- }
-
- @Override
- protected EvalNode visitRowConstant(Object o, RowConstantEval evalNode, Stack<EvalNode> stack) {
- Expr[] values = new Expr[evalNode.getValues().length];
- for (int i = 0; i < evalNode.getValues().length; i++) {
- Datum datum = evalNode.getValues()[i];
- LiteralValue value;
- switch (datum.type()) {
- case BOOLEAN:
- value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Boolean);
- break;
- case TEXT:
- value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.String);
- break;
- case INT1:
- case INT2:
- case INT4:
- value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Integer);
- break;
- case INT8:
- value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Large_Integer);
- break;
- case FLOAT4:
- case FLOAT8:
- value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Float);
- break;
- default:
- throw new RuntimeException("Unsupported type: " + datum.type().name());
- }
- values[i] = value;
- }
- ValueListExpr expr = new ValueListExpr(values);
- exprs.push(expr);
-
- return super.visitRowConstant(o, evalNode, stack);
- }
-
- @Override
- protected EvalNode visitField(Object o, FieldEval evalNode, Stack<EvalNode> stack) {
- ColumnReferenceExpr expr = new ColumnReferenceExpr(tableName, evalNode.getColumnName());
- exprs.push(expr);
- return super.visitField(o, evalNode, stack);
- }
-
- @Override
- protected EvalNode visitBetween(Object o, BetweenPredicateEval evalNode, Stack<EvalNode> stack) {
- stack.push(evalNode);
-
- visit(o, evalNode.getPredicand(), stack);
- Expr predicand = exprs.pop();
-
- visit(o, evalNode.getBegin(), stack);
- Expr begin = exprs.pop();
-
- visit(o, evalNode.getEnd(), stack);
- Expr end = exprs.pop();
-
- Expr expr = new BetweenPredicate(evalNode.isNot(), evalNode.isSymmetric(), predicand, begin, end);
- exprs.push(expr);
-
- stack.pop();
-
- return null;
- }
-
- @Override
- protected EvalNode visitCaseWhen(Object o, CaseWhenEval evalNode, Stack<EvalNode> stack) {
- stack.push(evalNode);
-
- CaseWhenPredicate caseWhenPredicate = new CaseWhenPredicate();
-
- for (CaseWhenEval.IfThenEval ifThenEval : evalNode.getIfThenEvals()) {
- visit(o, ifThenEval.getCondition(), stack);
- Expr condition = exprs.pop();
- visit(o, ifThenEval.getResult(), stack);
- Expr result = exprs.pop();
-
- caseWhenPredicate.addWhen(condition, result);
- }
-
- if (evalNode.hasElse()) {
- visit(o, evalNode.getElse(), stack);
- Expr elseResult = exprs.pop();
- caseWhenPredicate.setElseResult(elseResult);
- }
-
- exprs.push(caseWhenPredicate);
-
- stack.pop();
-
- return null;
- }
-}
\ No newline at end of file
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PartitionFilterAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PartitionFilterAlgebraVisitor.java
deleted file mode 100644
index 72fd939..0000000
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PartitionFilterAlgebraVisitor.java
+++ /dev/null
@@ -1,573 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.tajo.plan.util;
-
-import org.apache.tajo.algebra.*;
-import org.apache.tajo.catalog.CatalogConstants;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.common.TajoDataTypes.*;
-import org.apache.tajo.datum.TimeDatum;
-import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.exception.UnsupportedException;
-import org.apache.tajo.plan.ExprAnnotator;
-import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
-import org.apache.tajo.util.Pair;
-import org.apache.tajo.util.TUtil;
-import org.apache.tajo.util.datetime.DateTimeUtil;
-import org.apache.tajo.util.datetime.TimeMeta;
-
-import java.sql.Date;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.util.List;
-import java.util.Stack;
-import java.util.TimeZone;
-
-/**
- * This build SQL statements for getting partitions informs on CatalogStore with algebra expressions.
- * This visitor assumes that all columns of algebra expressions are reserved for one table.
- *
- */
-public class PartitionFilterAlgebraVisitor extends SimpleAlgebraVisitor<Object, Expr> {
- private String tableAlias;
- private Column column;
- private boolean isHiveCatalog = false;
-
- private Stack<String> queries = new Stack();
- private List<Pair<Type, Object>> parameters = TUtil.newList();
-
- public String getTableAlias() {
- return tableAlias;
- }
-
- public void setTableAlias(String tableAlias) {
- this.tableAlias = tableAlias;
- }
-
- public Column getColumn() {
- return column;
- }
-
- public void setColumn(Column column) {
- this.column = column;
- }
-
- public boolean isHiveCatalog() {
- return isHiveCatalog;
- }
-
- public void setIsHiveCatalog(boolean isHiveCatalog) {
- this.isHiveCatalog = isHiveCatalog;
- }
-
- public List<Pair<Type, Object>> getParameters() {
- return parameters;
- }
-
- public void setParameters(List<Pair<Type, Object>> parameters) {
- this.parameters = parameters;
- }
-
- public void clearParameters() {
- this.parameters.clear();
- }
-
- public String getResult() {
- return queries.pop();
- }
-
- @Override
- public Expr visit(Object ctx, Stack<Expr> stack, Expr expr) throws TajoException {
- if (expr.getType() == OpType.LikePredicate) {
- return visitLikePredicate(ctx, stack, (PatternMatchPredicate) expr);
- } else if (expr.getType() == OpType.SimilarToPredicate) {
- return visitSimilarToPredicate(ctx, stack, (PatternMatchPredicate) expr);
- } else if (expr.getType() == OpType.Regexp) {
- return visitRegexpPredicate(ctx, stack, (PatternMatchPredicate) expr);
- }
- return super.visit(ctx, stack, expr);
- }
-
- @Override
- public Expr visitDateLiteral(Object ctx, Stack<Expr> stack, DateLiteral expr) throws TajoException {
- StringBuilder sb = new StringBuilder();
-
- if (!isHiveCatalog) {
- sb.append("?").append(" )");
- parameters.add(new Pair(Type.DATE, Date.valueOf(expr.toString())));
- } else {
- sb.append("\"").append(expr.toString()).append("\"");
- }
- queries.push(sb.toString());
- return expr;
- }
-
- @Override
- public Expr visitTimestampLiteral(Object ctx, Stack<Expr> stack, TimestampLiteral expr) throws TajoException {
- StringBuilder sb = new StringBuilder();
-
- if (!isHiveCatalog) {
- DateValue dateValue = expr.getDate();
- TimeValue timeValue = expr.getTime();
-
- int [] dates = ExprAnnotator.dateToIntArray(dateValue.getYears(),
- dateValue.getMonths(),
- dateValue.getDays());
- int [] times = ExprAnnotator.timeToIntArray(timeValue.getHours(),
- timeValue.getMinutes(),
- timeValue.getSeconds(),
- timeValue.getSecondsFraction());
-
- long julianTimestamp;
- if (timeValue.hasSecondsFraction()) {
- julianTimestamp = DateTimeUtil.toJulianTimestamp(dates[0], dates[1], dates[2], times[0], times[1], times[2],
- times[3] * 1000);
- } else {
- julianTimestamp = DateTimeUtil.toJulianTimestamp(dates[0], dates[1], dates[2], times[0], times[1], times[2], 0);
- }
-
- TimeMeta tm = new TimeMeta();
- DateTimeUtil.toJulianTimeMeta(julianTimestamp, tm);
-
- TimeZone tz = TimeZone.getDefault();
- DateTimeUtil.toUTCTimezone(tm, tz);
-
- sb.append("?").append(" )");
- Timestamp timestamp = new Timestamp(DateTimeUtil.julianTimeToJavaTime(DateTimeUtil.toJulianTimestamp(tm)));
- parameters.add(new Pair(Type.TIMESTAMP, timestamp));
- } else {
- sb.append("\"").append(expr.toString()).append("\"");
- }
- queries.push(sb.toString());
-
- return expr;
- }
-
- @Override
- public Expr visitTimeLiteral(Object ctx, Stack<Expr> stack, TimeLiteral expr) throws TajoException {
- StringBuilder sb = new StringBuilder();
-
- if (!isHiveCatalog) {
- TimeValue timeValue = expr.getTime();
-
- int [] times = ExprAnnotator.timeToIntArray(timeValue.getHours(),
- timeValue.getMinutes(),
- timeValue.getSeconds(),
- timeValue.getSecondsFraction());
-
- long time;
- if (timeValue.hasSecondsFraction()) {
- time = DateTimeUtil.toTime(times[0], times[1], times[2], times[3] * 1000);
- } else {
- time = DateTimeUtil.toTime(times[0], times[1], times[2], 0);
- }
- TimeDatum timeDatum = new TimeDatum(time);
- TimeMeta tm = timeDatum.asTimeMeta();
-
- TimeZone tz = TimeZone.getDefault();
- DateTimeUtil.toUTCTimezone(tm, tz);
-
- sb.append("?").append(" )");
- parameters.add(new Pair(Type.TIME, new Time(DateTimeUtil.toJavaTime(tm.hours, tm.minutes, tm.secs, tm.fsecs))));
- } else {
- sb.append("\"").append(expr.toString()).append("\"");
- }
- queries.push(sb.toString());
-
- return expr;
- }
-
- @Override
- public Expr visitLiteral(Object ctx, Stack<Expr> stack, LiteralValue expr) throws TajoException {
- StringBuilder sb = new StringBuilder();
-
- if (!isHiveCatalog) {
- sb.append("?").append(" )");
- switch (expr.getValueType()) {
- case Boolean:
- parameters.add(new Pair(Type.BOOLEAN, Boolean.valueOf(expr.getValue())));
- break;
- case Unsigned_Float:
- parameters.add(new Pair(Type.FLOAT8, Double.valueOf(expr.getValue())));
- break;
- case String:
- parameters.add(new Pair(Type.TEXT, expr.getValue()));
- break;
- default:
- parameters.add(new Pair(Type.INT8, Long.valueOf(expr.getValue())));
- break;
- }
- } else {
- switch (expr.getValueType()) {
- case String:
- sb.append("\"").append(expr.getValue()).append("\"");
- break;
- default:
- sb.append(expr.getValue());
- break;
- }
- }
- queries.push(sb.toString());
-
- return expr;
- }
-
- @Override
- public Expr visitValueListExpr(Object ctx, Stack<Expr> stack, ValueListExpr expr) throws TajoException {
- StringBuilder sb = new StringBuilder();
-
- if (!isHiveCatalog) {
- sb.append("(");
- for(int i = 0; i < expr.getValues().length; i++) {
- if (i > 0) {
- sb.append(", ");
- }
- sb.append("?");
- stack.push(expr.getValues()[i]);
- visit(ctx, stack, expr.getValues()[i]);
- stack.pop();
- }
- sb.append(")");
- sb.append(" )");
- queries.push(sb.toString());
- } else {
- throw new UnsupportedException("IN Operator");
- }
-
- return expr;
- }
-
- @Override
- public Expr visitColumnReference(Object ctx, Stack<Expr> stack, ColumnReferenceExpr expr) throws TajoException {
- StringBuilder sb = new StringBuilder();
-
- if (!isHiveCatalog) {
- sb.append("( ").append(tableAlias).append(".").append(CatalogConstants.COL_COLUMN_NAME)
- .append(" = '").append(expr.getName()).append("'")
- .append(" AND ").append(tableAlias).append(".").append(CatalogConstants.COL_PARTITION_VALUE);
- } else {
- sb.append(expr.getName());
- }
- queries.push(sb.toString());
- return expr;
- }
-
-
- @Override
- public Expr visitUnaryOperator(Object ctx, Stack<Expr> stack, UnaryOperator expr) throws TajoException {
- stack.push(expr);
- Expr child = visit(ctx, stack, expr.getChild());
- stack.pop();
-
- if (child.getType() == OpType.Literal) {
- return new NullLiteral();
- }
-
- String childSql = queries.pop();
-
- StringBuilder sb = new StringBuilder();
- if (expr.getType() == OpType.IsNullPredicate) {
- IsNullPredicate isNullPredicate = (IsNullPredicate) expr;
- sb.append(childSql);
- sb.append(" IS ");
- if (isNullPredicate.isNot()) {
- sb.append("NOT NULL");
- } else {
- sb.append("NULL");
- }
- }
-
- if (!isHiveCatalog) {
- sb.append(" )");
- }
- queries.push(sb.toString());
-
- return expr;
- }
-
- @Override
- public Expr visitBetween(Object ctx, Stack<Expr> stack, BetweenPredicate expr) throws TajoException {
- stack.push(expr);
-
- visit(ctx, stack, expr.predicand());
- String predicandSql = queries.pop();
-
- visit(ctx, stack, expr.begin());
- String beginSql= queries.pop();
- if (!isHiveCatalog && beginSql.endsWith(")")) {
- beginSql = beginSql.substring(0, beginSql.length()-1);
- }
-
- visit(ctx, stack, expr.end());
- String endSql = queries.pop();
- if (!isHiveCatalog && endSql.endsWith(")")) {
- endSql = beginSql.substring(0, endSql.length()-1);
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append(predicandSql);
- sb.append(" BETWEEN ");
- sb.append(beginSql);
- sb.append(" AND ");
- sb.append(endSql);
-
- if (!isHiveCatalog) {
- sb.append(")");
- }
-
- queries.push(sb.toString());
-
- return expr;
- }
-
- @Override
- public Expr visitCaseWhen(Object ctx, Stack<Expr> stack, CaseWhenPredicate expr) throws TajoException {
- stack.push(expr);
-
- StringBuilder sb = new StringBuilder();
- sb.append("CASE ");
-
- String condition, result;
-
- for (CaseWhenPredicate.WhenExpr when : expr.getWhens()) {
- visit(ctx, stack, when.getCondition());
- condition = queries.pop();
- visit(ctx, stack, when.getResult());
- result = queries.pop();
-
- String whenSql = condition + " " + result;
- if (!isHiveCatalog && whenSql.endsWith(")")) {
- whenSql = whenSql.substring(0, whenSql.length()-1);
- }
-
- sb.append(whenSql).append(" ");
- }
-
- if (expr.hasElseResult()) {
- visit(ctx, stack, expr.getElseResult());
- String elseSql = queries.pop();
- if (!isHiveCatalog && elseSql.endsWith(")")) {
- elseSql = elseSql.substring(0, elseSql.length()-1);
- }
-
- sb.append("ELSE ").append(elseSql).append(" END");
- }
-
- if (!isHiveCatalog) {
- sb.append(")");
- }
-
- queries.push(sb.toString());
-
- stack.pop();
- return expr;
- }
-
- @Override
- public Expr visitBinaryOperator(Object ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException {
- stack.push(expr);
- Expr lhs = visit(ctx, stack, expr.getLeft());
- String leftSql = queries.pop();
- Expr rhs = visit(ctx, stack, expr.getRight());
- String rightSql = queries.pop();
- stack.pop();
-
- if (!expr.getLeft().equals(lhs)) {
- expr.setLeft(lhs);
- }
- if (!expr.getRight().equals(rhs)) {
- expr.setRight(rhs);
- }
-
- if (lhs.getType() == OpType.Literal && rhs.getType() == OpType.Literal) {
- return new NullLiteral();
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append(leftSql);
- sb.append(" ").append(getOperator(expr.getType())).append(" ");
- sb.append(rightSql);
- queries.push(sb.toString());
-
- return expr;
- }
-
- private String getOperator(OpType type) {
- String operator;
- switch (type) {
- case Not:
- operator = "!";
- break;
- case And:
- operator = "AND";
- break;
- case Or:
- operator = "OR";
- break;
- case Equals:
- operator = "=";
- break;
- case IsNullPredicate:
- operator = "IS NULL";
- break;
- case NotEquals:
- operator = "<>";
- break;
- case LessThan:
- operator = "<";
- break;
- case LessThanOrEquals:
- operator = "<=";
- break;
- case GreaterThan:
- operator = ">";
- break;
- case GreaterThanOrEquals:
- operator = ">=";
- break;
- case Plus:
- operator = "+";
- break;
- case Minus:
- operator = "-";
- break;
- case Modular:
- operator = "%";
- break;
- case Multiply:
- operator = "*";
- break;
- case Divide:
- operator = "/";
- break;
- case LikePredicate:
- operator = "LIKE";
- break;
- case SimilarToPredicate:
- operator = "([.])";
- break;
- case InPredicate:
- operator = "IN";
- break;
- case Asterisk:
- operator = "*";
- break;
- //TODO: need to check more types.
- default:
- operator = type.name();
- break;
- }
-
- return operator;
- }
-
- @Override
- public Expr visitLikePredicate(Object ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws TajoException {
- stack.push(expr);
-
- visit(ctx, stack, expr.getPredicand());
- String predicand = queries.pop();
- visit(ctx, stack, expr.getPattern());
- String pattern = queries.pop();
- stack.pop();
-
- if(isHiveCatalog) {
- if (pattern.startsWith("%") || pattern.endsWith("%")) {
- throw new UnsupportedException("LIKE Operator with '%'");
- }
- }
- StringBuilder sb = new StringBuilder();
- sb.append(predicand);
-
- if (expr.isNot()) {
- sb.append(" NOT ");
- }
-
- if (expr.isCaseInsensitive()) {
- sb.append(" ILIKE ");
- } else {
- sb.append(" LIKE ");
- }
-
-
- sb.append(pattern);
- queries.push(sb.toString());
-
- return expr;
- }
-
- @Override
- public Expr visitSimilarToPredicate(Object ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws TajoException {
- if (isHiveCatalog) {
- throw new UnsupportedException("SIMILAR TO Operator");
- }
-
- stack.push(expr);
-
- visit(ctx, stack, expr.getPredicand());
- String predicand = queries.pop();
- visit(ctx, stack, expr.getPattern());
- String pattern = queries.pop();
- stack.pop();
-
- StringBuilder sb = new StringBuilder();
- sb.append(predicand);
-
- if (expr.isNot()) {
- sb.append(" NOT ");
- }
-
- sb.append(" SIMILAR TO ");
-
- sb.append(pattern);
- queries.push(sb.toString());
-
- return expr;
- }
-
- @Override
- public Expr visitRegexpPredicate(Object ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws TajoException {
- if (isHiveCatalog) {
- throw new UnsupportedException("REGEXP Operator");
- }
-
- stack.push(expr);
-
- visit(ctx, stack, expr.getPredicand());
- String predicand = queries.pop();
- visit(ctx, stack, expr.getPattern());
- String pattern = queries.pop();
- stack.pop();
-
- StringBuilder sb = new StringBuilder();
- sb.append(predicand);
-
- if (expr.isNot()) {
- sb.append(" NOT ");
- }
- sb.append(" REGEXP ");
-
- sb.append(pattern);
- queries.push(sb.toString());
-
- return expr;
- }
-
-}
\ No newline at end of file
